<commit_before>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"path"
	"strings"
	"sync"
	"time"

	"github.com/hashicorp/hcl"
)

type Config struct {
	Listen  string // HTTP listen address. ":8084"
	Data    string // Storage location for cached files. "/var/remirror"
	Mirrors []Mirror
}
type Mirror struct {
	// Prefix specifies a path that should be sent
	// to a certain upstream. E.g. "/archlinux/"
	Prefix string

	// Upstream specifies the upstream protocol and host.
	// You may also specify a path, in which case Prefix is
	// stripped from the incoming request, and what is left is
	// appended to the upstream path component.
	//
	// E.g. "https://mirrors.kernel.org"     (/archlinux/somepackage will be preserved)
	// E.g. "http://mirror.cs.umn.edu/arch/" (/archlinux/thing will transform to /arch/thing)
	Upstream string

	// Local should be used instead of Upstream for a locally served folder.
	// Incoming requests will have Prefix stripped off before being sent to Local.
	// E.g. "/home/you/localrepos/archlinux"
	Local string

	// If nil, default match set will be used
	Matches []Match
}
type Match struct {
	Fail   bool
	Prefix string
	Suffix string
}

func (mirror Mirror) String() string {
	s := mirror.Upstream
	if mirror.Local != "" {
		s = mirror.Local
	}
	s += " "
	for i, m := range mirror.Matches {
		ss := m.Prefix + "*" + m.Suffix
		if m.Fail {
			ss += " fail"
		}
		if i+1 < len(mirror.Matches) {
			ss += ", "
		}
		s += ss
	}
	return fmt.Sprintf("%-20s » %s", mirror.Prefix, s)
}

var (
	http_client = http.Client{}

	downloads_mu sync.Mutex
	downloads    = map[string]*Download{}
)

type Download struct {
	resp *http.Response

	tmp_path string
	tmp_done chan struct{} // will be closed when download is done and final bytes written
}

func (mirror Mirror) should_cache(path string) bool {
	// Use custom match rules?
	if len(mirror.Matches) > 0 {
		for _, m := range mirror.Matches {
			if strings.HasPrefix(path, m.Prefix) &&
				strings.HasSuffix(path, m.Suffix) {
				return !m.Fail
			}
		}
		return false
	}

	// Arch has some DB files we don't want to cache even though
	// they have archive suffixes. So we're a little more strict here.
	if strings.HasPrefix(path, "/archlinux/") {
		if strings.HasSuffix(path, ".pkg.tar.xz") {
			return true
		}
		if strings.HasSuffix(path, ".pkg.tar.xz.sig") {
			return true
		}
		return false
	}

	// Same for ubuntu
	if strings.HasSuffix(path, "/Packages.gz") || strings.HasSuffix(path, "/Sources.gz") {
		return false
	}

	// Otherwise cache everything that looks like an archive.
	if strings.HasSuffix(path, ".xz") ||
		strings.HasSuffix(path, ".gz") ||
		strings.HasSuffix(path, ".bz2") ||
		strings.HasSuffix(path, ".zip") ||
		strings.HasSuffix(path, ".tgz") ||
		strings.HasSuffix(path, ".rpm") ||
		strings.HasSuffix(path, "-rpm.bin") ||
		strings.HasSuffix(path, ".deb") ||
		strings.HasSuffix(path, ".xz.sig") {
		return true
	}
	return false
}

func (mirror Mirror) CreateHandler(config *Config, fileserver http.Handler) (http.Handler, error) {

	if mirror.Local != "" {
		return http.StripPrefix(mirror.Prefix, http.FileServer(http.Dir(mirror.Local))), nil
	}

	upstream, err := url.Parse(mirror.Upstream)
	if err != nil {
		return nil, err
	}

	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		log.Println(r.Method + " http://" + r.Host + r.RequestURI)

		err := func() error {

			local_path := ""
			remote_url := upstream.Scheme + "://" + upstream.Host

			if upstream.Path == "" {
				remote_url += path.Clean(r.URL.Path)
			} else {
				remote_url += path.Clean(upstream.Path + "/" + strings.TrimPrefix(r.URL.Path, mirror.Prefix))
			}

			if mirror.should_cache(remote_url) {
				local_path = config.Data + path.Clean(r.URL.Path)

				_, err := os.Stat(local_path)
				if err == nil {
					fileserver.ServeHTTP(w, r)
					return nil
				}
			}

			var download *Download
			var ok bool

			downloads_mu.Lock()

			if r.Header.Get("Range") == "" && local_path != "" {
				download, ok = downloads[local_path]
				if ok {
					fh, err := os.Open(download.tmp_path)
					downloads_mu.Unlock()
					if err != nil {
						return err
					}
					return tmp_download(local_path, w, download, fh)
				}
			}

			// downloads_mu is still locked. take care.
			// we need to keep it locked until we have
			// registered a download, opened a temp file,
			// and saved it's path into the tmp_path in
			// the struct.
			// then we need to make sure to release.

			log.Println("-->", remote_url)

			req, err := http.NewRequest("GET", remote_url, nil)
			if err != nil {
				downloads_mu.Unlock()
				return err
			}

			for k, vs := range r.Header {
				if !hopHeaders[k] {
					for _, v := range vs {
						req.Header.Add(k, v)
					}
				}
			}

			resp, err := http_client.Do(req)
			if err != nil {
				downloads_mu.Unlock()
				return err
			}
			defer resp.Body.Close()

			out := io.Writer(w)

			tmp_path := ""

			var tmp_needs_final_close io.Closer

			// We don't want to cache the result if the server
			// returns with a 206.
			if resp.StatusCode == 200 && local_path != "" {
				tmp, err := ioutil.TempFile(config.Data, "remirror_tmp_")
				if err != nil {
					downloads_mu.Unlock()
					return err
				}
				tmp_needs_final_close = tmp
				tmp_path = tmp.Name()
				//fmt.Println("tmp", tmp_path)

				defer tmp.Close()
				defer os.Remove(tmp_path)

				out = io.MultiWriter(out, tmp)

				// at this point we have a "successful" download in
				// progress. save into the struct.
				download = &Download{
					resp:     resp,
					tmp_path: tmp_path,
					tmp_done: make(chan struct{}),
				}
				downloads[local_path] = download
			}
			// release the mutex. if we have a successful download in
			// progress, we have stored it correctly so far. if not,
			// we unlock, leaving the download struct unmodified. the
			// next request to try that URL will retry.
			downloads_mu.Unlock()

			// however we quit, we want to clear the download in progress
			// entry. this deferred func should run before the deferred
			// cleanup funcs above, so the filehandle should still be
			// valid when we clear it out.
			defer func() {
				if download == nil {
					// we didn't end up using the map for some reason.
					// (maybe empty content length, non 200 response, etc)
					return
				}

				// make sure final close has been called. things might still
				// be writing, and we need that to be done before
				// we close tmp_done
				_ = tmp_needs_final_close.Close()

				close(download.tmp_done)

				downloads_mu.Lock()
				delete(downloads, local_path)
				downloads_mu.Unlock()
			}()

			write_resp_headers(w, resp)

			n, err := io.Copy(out, resp.Body)
			if err != nil {
				log.Println(err)
				return nil
			}

			if n != resp.ContentLength && resp.ContentLength != -1 {
				log.Printf("Short data returned from server (Content-Length %d received %d)\n", resp.ContentLength, n)

				// Not really an HTTP error, leave it up to the client.
				// but we aren't going to save our response to the cache.
				return nil
			}

			if tmp_path != "" {
				os.MkdirAll(path.Dir(local_path), 0755)

				err = tmp_needs_final_close.Close()
				if err != nil {
					log.Println(err)
					return nil
				}

				// clear from struct before renaming
				if download != nil {
					close(download.tmp_done)
					downloads_mu.Lock()
					delete(downloads, local_path)
					downloads_mu.Unlock()
					download = nil // so we don't re-close
				}

				err = os.Rename(tmp_path, local_path)
				if err != nil {
					log.Println(err)
					return nil
				}
				log.Println(">:)")
			}

			return nil
		}()

		he, ok := err.(HTTPError)
		if ok {
			http.Error(w, he.Error(), he.Code())
			fmt.Println("\t\t", he.Error())
		} else if err != nil {
			http.Error(w, err.Error(), 500)
			fmt.Println("\t\t500 " + err.Error())
		}
	}), nil
}

func load_configs(config *Config) error {
	try := []string{"remirror.hcl"}
	home := os.Getenv("HOME")
	if home != "" {
		try = append(try, home+"/.remirror.hcl")
	}
	try = append(try, "/etc/remirror.hcl")

	for _, t := range try {
		_, err := os.Stat(t)
		if err == nil {
			log.Printf("Loading configuration from %#v ...\n", t)
			config_bytes, err := ioutil.ReadFile(t)
			if err != nil {
				return err
			}
			if err := hcl.Unmarshal(config_bytes, config); err != nil {
				return err
			}
			return nil
		}
	}
	return fmt.Errorf("No files found: Create one of %s", strings.Join(try, ", "))
}

func main() {
	config := &Config{}

	if err := load_configs(config); err != nil {
		log.Fatalf("Config error: %v", err)
	}

	fileserver := http.FileServer(http.Dir(config.Data))

	for _, mirror := range config.Mirrors {
		handler, err := mirror.CreateHandler(config, fileserver)
		if err == nil {
			log.Println(mirror, " ✓ ")
			http.Handle(mirror.Prefix, handler)
		} else {
			log.Println(mirror, " ✗ Error:", err)
		}
	}

	log.Println("remirror listening on HTTP", config.Listen, "with data cache", config.Data)
	log.Fatal(http.ListenAndServe(config.Listen, nil))
}

func write_resp_headers(w http.ResponseWriter, resp *http.Response) {

	for k, vs := range resp.Header {
		if k == "Accept-Ranges" {
			continue
		}
		for _, v := range vs {
			//fmt.Printf("proxy back header %#v\t%#v\n", k, v)
			w.Header().Add(k, v)
		}
	}

	w.Header().Set("Server", "remirror")
	w.WriteHeader(resp.StatusCode)
}

// return a download in progress started by another request
func tmp_download(local_path string, w http.ResponseWriter, download *Download, tmp io.ReadCloser) error {
	defer tmp.Close()

	write_resp_headers(w, download.resp)

	written := int64(0)
	done := false
	last := time.Now()

	for {
		n, err := io.Copy(w, tmp)

		if n < 0 {
			panic(fmt.Sprintf("io.Copy returned n %d: Not what I expected!", n))
		}

		written += n

		if err != nil && err != io.EOF {
			log.Printf("Error while reading concurrent download %#s from %#s: %v\n",
				local_path, download.tmp_path, err)
			// Not an HTTP error: just return, and the client will hopefully
			// handle a short read correctly.
			return nil
		}

		if n > 0 {
			// cool, try another copy. hopefully the file
			// has more bytes now
			last = time.Now()
			continue
		}

		if done {
			return nil
		}

		// sleep for a bit so the other download has a chance to write
		// more bytes.
		select {
		case <-time.After(time.Second):
			// 60 second timeout for the other goroutine to at least write _something_
			if time.Since(last) > time.Minute {
				log.Println("Timeout while reading concurrent download %#s from %#s\n",
					local_path,
					download.tmp_path)
				// Not an HTTP error: just return, and the client will hopefully
				// handle a short read correctly.
				return nil
			}
			continue
		case <-download.tmp_done:
			done = true
			continue
		}
	}
}
<commit_msg>Improve caching rules<commit_after>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"path"
	"strings"
	"sync"
	"time"

	"github.com/hashicorp/hcl"
)

type Config struct {
	Listen  string // HTTP listen address. ":8084"
	Data    string // Storage location for cached files. "/var/remirror"
	Mirrors []Mirror
}
type Mirror struct {
	// Prefix specifies a path that should be sent
	// to a certain upstream. E.g. "/archlinux/"
	Prefix string

	// Upstream specifies the upstream protocol and host.
	// You may also specify a path, in which case Prefix is
	// stripped from the incoming request, and what is left is
	// appended to the upstream path component.
	//
	// E.g. "https://mirrors.kernel.org"     (/archlinux/somepackage will be preserved)
	// E.g. "http://mirror.cs.umn.edu/arch/" (/archlinux/thing will transform to /arch/thing)
	Upstream string

	// Local should be used instead of Upstream for a locally served folder.
	// Incoming requests will have Prefix stripped off before being sent to Local.
	// E.g. "/home/you/localrepos/archlinux"
	Local string

	// If nil, default match set will be used
	Matches []Match
}
type Match struct {
	Prefix string
	Suffix string
	Skip   bool // skip = true means this is a "don't match" rule
}

func (mirror Mirror) String() string {
	s := mirror.Upstream
	if mirror.Local != "" {
		s = mirror.Local
	}
	s += " "
	for i, m := range mirror.Matches {
		ss := m.Prefix + "*" + m.Suffix
		if m.Skip {
			ss += " skip"
		}
		if i+1 < len(mirror.Matches) {
			ss += ", "
		}
		s += ss
	}
	return fmt.Sprintf("%-20s » %s", mirror.Prefix, s)
}

var (
	http_client = http.Client{}

	downloads_mu sync.Mutex
	downloads    = map[string]*Download{}
)

type Download struct {
	resp *http.Response

	tmp_path string
	tmp_done chan struct{} // will be closed when download is done and final bytes written
}

func (mirror Mirror) should_cache(path string) bool {
	// Special rules for Debian/Ubuntu
	if strings.HasSuffix(path, "/Packages.gz") || strings.HasSuffix(path, "/Sources.gz") {
		return false
	}

	// Special rules for Arch
	if strings.HasSuffix(path, ".abs.tar.gz") ||
		strings.HasSuffix(path, ".db.tar.gz") ||
		strings.HasSuffix(path, ".files.tar.gz") ||
		strings.HasSuffix(path, ".links.tar.gz") {
		return false
	}

	// Use custom match rules?
	if len(mirror.Matches) > 0 {
		for _, m := range mirror.Matches {
			if strings.HasPrefix(path, m.Prefix) &&
				strings.HasSuffix(path, m.Suffix) {
				return !m.Skip
			}
		}
		return false
	}

	// Otherwise cache everything that looks like an archive.
	if strings.HasSuffix(path, ".xz") ||
		strings.HasSuffix(path, ".gz") ||
		strings.HasSuffix(path, ".bz2") ||
		strings.HasSuffix(path, ".zip") ||
		strings.HasSuffix(path, ".tgz") ||
		strings.HasSuffix(path, ".rpm") ||
		strings.HasSuffix(path, "-rpm.bin") ||
		strings.HasSuffix(path, ".deb") ||
		strings.HasSuffix(path, ".jar") ||
		strings.HasSuffix(path, ".xz.sig") {
		return true
	}
	return false
}

func (mirror Mirror) CreateHandler(config *Config, fileserver http.Handler) (http.Handler, error) {

	if mirror.Local != "" {
		return http.StripPrefix(mirror.Prefix, http.FileServer(http.Dir(mirror.Local))), nil
	}

	upstream, err := url.Parse(mirror.Upstream)
	if err != nil {
		return nil, err
	}

	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		log.Println(r.Method + " http://" + r.Host + r.RequestURI)

		err := func() error {

			local_path := ""
			remote_url := upstream.Scheme + "://" + upstream.Host

			if upstream.Path == "" {
				remote_url += path.Clean(r.URL.Path)
			} else {
				remote_url += path.Clean(upstream.Path + "/" + strings.TrimPrefix(r.URL.Path, mirror.Prefix))
			}

			if mirror.should_cache(remote_url) {
				local_path = config.Data + path.Clean(r.URL.Path)

				_, err := os.Stat(local_path)
				if err == nil {
					fileserver.ServeHTTP(w, r)
					return nil
				}
			}

			var download *Download
			var ok bool

			downloads_mu.Lock()

			if r.Header.Get("Range") == "" && local_path != "" {
				download, ok = downloads[local_path]
				if ok {
					fh, err := os.Open(download.tmp_path)
					downloads_mu.Unlock()
					if err != nil {
						return err
					}
					return tmp_download(local_path, w, download, fh)
				}
			}

			// downloads_mu is still locked. take care.
			// we need to keep it locked until we have
			// registered a download, opened a temp file,
			// and saved it's path into the tmp_path in
			// the struct.
			// then we need to make sure to release.

			log.Println("-->", remote_url)

			req, err := http.NewRequest("GET", remote_url, nil)
			if err != nil {
				downloads_mu.Unlock()
				return err
			}

			for k, vs := range r.Header {
				if !hopHeaders[k] {
					for _, v := range vs {
						req.Header.Add(k, v)
					}
				}
			}

			resp, err := http_client.Do(req)
			if err != nil {
				downloads_mu.Unlock()
				return err
			}
			defer resp.Body.Close()

			out := io.Writer(w)

			tmp_path := ""

			var tmp_needs_final_close io.Closer

			// We don't want to cache the result if the server
			// returns with a 206.
			if resp.StatusCode == 200 && local_path != "" {
				tmp, err := ioutil.TempFile(config.Data, "remirror_tmp_")
				if err != nil {
					downloads_mu.Unlock()
					return err
				}
				tmp_needs_final_close = tmp
				tmp_path = tmp.Name()
				//fmt.Println("tmp", tmp_path)

				defer tmp.Close()
				defer os.Remove(tmp_path)

				out = io.MultiWriter(out, tmp)

				// at this point we have a "successful" download in
				// progress. save into the struct.
				download = &Download{
					resp:     resp,
					tmp_path: tmp_path,
					tmp_done: make(chan struct{}),
				}
				downloads[local_path] = download
			}
			// release the mutex. if we have a successful download in
			// progress, we have stored it correctly so far. if not,
			// we unlock, leaving the download struct unmodified. the
			// next request to try that URL will retry.
			downloads_mu.Unlock()

			// however we quit, we want to clear the download in progress
			// entry. this deferred func should run before the deferred
			// cleanup funcs above, so the filehandle should still be
			// valid when we clear it out.
			defer func() {
				if download == nil {
					// we didn't end up using the map for some reason.
					// (maybe empty content length, non 200 response, etc)
					return
				}

				// make sure final close has been called. things might still
				// be writing, and we need that to be done before
				// we close tmp_done
				_ = tmp_needs_final_close.Close()

				close(download.tmp_done)

				downloads_mu.Lock()
				delete(downloads, local_path)
				downloads_mu.Unlock()
			}()

			write_resp_headers(w, resp)

			n, err := io.Copy(out, resp.Body)
			if err != nil {
				log.Println(err)
				return nil
			}

			if n != resp.ContentLength && resp.ContentLength != -1 {
				log.Printf("Short data returned from server (Content-Length %d received %d)\n", resp.ContentLength, n)

				// Not really an HTTP error, leave it up to the client.
				// but we aren't going to save our response to the cache.
				return nil
			}

			if tmp_path != "" {
				os.MkdirAll(path.Dir(local_path), 0755)

				err = tmp_needs_final_close.Close()
				if err != nil {
					log.Println(err)
					return nil
				}

				// clear from struct before renaming
				if download != nil {
					close(download.tmp_done)
					downloads_mu.Lock()
					delete(downloads, local_path)
					downloads_mu.Unlock()
					download = nil // so we don't re-close
				}

				err = os.Rename(tmp_path, local_path)
				if err != nil {
					log.Println(err)
					return nil
				}
				log.Println(">:)")
			}

			return nil
		}()

		he, ok := err.(HTTPError)
		if ok {
			http.Error(w, he.Error(), he.Code())
			fmt.Println("\t\t", he.Error())
		} else if err != nil {
			http.Error(w, err.Error(), 500)
			fmt.Println("\t\t500 " + err.Error())
		}
	}), nil
}

func load_configs(config *Config) error {
	try := []string{"remirror.hcl"}
	home := os.Getenv("HOME")
	if home != "" {
		try = append(try, home+"/.remirror.hcl")
	}
	try = append(try, "/etc/remirror.hcl")

	for _, t := range try {
		_, err := os.Stat(t)
		if err == nil {
			log.Printf("Loading configuration from %#v ...\n", t)
			config_bytes, err := ioutil.ReadFile(t)
			if err != nil {
				return err
			}
			if err := hcl.Unmarshal(config_bytes, config); err != nil {
				return err
			}
			return nil
		}
	}
	return fmt.Errorf("No files found: Create one of %s", strings.Join(try, ", "))
}

func main() {
	config := &Config{}

	if err := load_configs(config); err != nil {
		log.Fatalf("Config error: %v", err)
	}

	fileserver := http.FileServer(http.Dir(config.Data))

	for _, mirror := range config.Mirrors {
		handler, err := mirror.CreateHandler(config, fileserver)
		if err == nil {
			log.Println(mirror, " ✓ ")
			http.Handle(mirror.Prefix, handler)
		} else {
			log.Println(mirror, " ✗ Error:", err)
		}
	}

	log.Println("remirror listening on HTTP", config.Listen, "with data cache", config.Data)
	log.Fatal(http.ListenAndServe(config.Listen, nil))
}

func write_resp_headers(w http.ResponseWriter, resp *http.Response) {

	for k, vs := range resp.Header {
		if k == "Accept-Ranges" {
			continue
		}
		for _, v := range vs {
			//fmt.Printf("proxy back header %#v\t%#v\n", k, v)
			w.Header().Add(k, v)
		}
	}

	w.Header().Set("Server", "remirror")
	w.WriteHeader(resp.StatusCode)
}

// return a download in progress started by another request
func tmp_download(local_path string, w http.ResponseWriter, download *Download, tmp io.ReadCloser) error {
	defer tmp.Close()

	write_resp_headers(w, download.resp)

	written := int64(0)
	done := false
	last := time.Now()

	for {
		n, err := io.Copy(w, tmp)

		if n < 0 {
			panic(fmt.Sprintf("io.Copy returned n %d: Not what I expected!", n))
		}

		written += n

		if err != nil && err != io.EOF {
			log.Printf("Error while reading concurrent download %#s from %#s: %v\n",
				local_path, download.tmp_path, err)
			// Not an HTTP error: just return, and the client will hopefully
			// handle a short read correctly.
			return nil
		}

		if n > 0 {
			// cool, try another copy. hopefully the file
			// has more bytes now
			last = time.Now()
			continue
		}

		if done {
			return nil
		}

		// sleep for a bit so the other download has a chance to write
		// more bytes.
		select {
		case <-time.After(time.Second):
			// 60 second timeout for the other goroutine to at least write _something_
			if time.Since(last) > time.Minute {
				log.Println("Timeout while reading concurrent download %#s from %#s\n",
					local_path,
					download.tmp_path)
				// Not an HTTP error: just return, and the client will hopefully
				// handle a short read correctly.
				return nil
			}
			continue
		case <-download.tmp_done:
			done = true
			continue
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"flag"
	"io/ioutil"
	"net/http"
	"net/http/httputil"
	"os"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/signer/v4"
	"github.com/aws/aws-sdk-go/private/protocol/rest"
)

func init() {
	log.SetFormatter(&log.JSONFormatter{})
	log.SetOutput(os.Stderr)

	level := os.Getenv("LOG_LEVEL")
	if len(level) == 0 {
		level = "info"
	}
	if lvl, err := log.ParseLevel(level); err != nil {
		log.Errorf("Level '%s' is invalid: falling back to INFO", level)
		log.SetLevel(log.InfoLevel)
	} else {
		log.SetLevel(lvl)
	}
}

var esDomain = flag.String("domain", os.Getenv("ES_DOMAIN"), "The elasticsearch domain to proxy")
var listenPort = flag.Int("port", 8080, "Listening port for proxy")
var region = flag.String("region", os.Getenv("AWS_REGION"), "AWS region for credentials")

func main() {
	flag.Parse()

	log.Printf("Connected to %s", *esDomain)
	log.Print("AWS ES cluster available at http://127.0.0.1:9200")
	log.Print("Kibana available at http://127.0.0.1:9200/_plugin/kibana/")
	creds := credentials.NewEnvCredentials()

	if _, err := creds.Get(); err != nil {
		log.Fatalf("Failed to load credentials: %v", err)
	}
	signer := v4.NewSigner(creds)

	director := func(req *http.Request) {
		req.URL.Scheme = "https"
		req.Host = *esDomain
		req.URL.Host = *esDomain
		req.Header.Set("Connection", "close")

		if strings.Contains(req.URL.RawPath, "%2C") {
			req.URL.RawPath = rest.EscapePath(req.URL.RawPath, false)
		}

		log.WithFields(log.Fields{
			"method": req.Method,
			"path":   req.URL.Path,
		}).Debug()
		t := time.Now()
		req.Header.Set("Date", t.Format(time.RFC3339))

		bodyData, err := ioutil.ReadAll(req.Body)
		if err != nil {
			log.WithFields(log.Fields{
				"method": req.Method,
				"path":   req.URL.Path,
			}).Errorf("Failed to consume body %v", err)
			return
		}
		buf := bytes.NewReader(bodyData)

		if _, err := signer.Sign(req, buf, "es", *region, t); err != nil {
			log.WithFields(log.Fields{
				"method": req.Method,
				"path":   req.URL.Path,
			}).Errorf("Failed to sign request %v", err)
		}
	}
	proxy := &httputil.ReverseProxy{Director: director}
	log.Fatal(http.ListenAndServe(":9200", proxy))
}
<commit_msg>Updated port in logging string<commit_after>package main

import (
	"bytes"
	"flag"
	"io/ioutil"
	"net/http"
	"net/http/httputil"
	"os"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/signer/v4"
	"github.com/aws/aws-sdk-go/private/protocol/rest"
)

func init() {
	log.SetFormatter(&log.JSONFormatter{})
	log.SetOutput(os.Stderr)

	level := os.Getenv("LOG_LEVEL")
	if len(level) == 0 {
		level = "info"
	}
	if lvl, err := log.ParseLevel(level); err != nil {
		log.Errorf("Level '%s' is invalid: falling back to INFO", level)
		log.SetLevel(log.InfoLevel)
	} else {
		log.SetLevel(lvl)
	}
}

var esDomain = flag.String("domain", os.Getenv("ES_DOMAIN"), "The elasticsearch domain to proxy")
var listenPort = flag.Int("port", 8080, "Listening port for proxy")
var region = flag.String("region", os.Getenv("AWS_REGION"), "AWS region for credentials")

func main() {
	flag.Parse()

	log.Printf("Connected to %s", *esDomain)
	log.Printf("AWS ES cluster available at http://127.0.0.1:%d", *listenPort)
	log.Printf("Kibana available at http://127.0.0.1:%d/_plugin/kibana/", *listenPort)
	creds := credentials.NewEnvCredentials()

	if _, err := creds.Get(); err != nil {
		log.Fatalf("Failed to load credentials: %v", err)
	}
	signer := v4.NewSigner(creds)

	director := func(req *http.Request) {
		req.URL.Scheme = "https"
		req.Host = *esDomain
		req.URL.Host = *esDomain
		req.Header.Set("Connection", "close")

		if strings.Contains(req.URL.RawPath, "%2C") {
			req.URL.RawPath = rest.EscapePath(req.URL.RawPath, false)
		}

		log.WithFields(log.Fields{
			"method": req.Method,
			"path":   req.URL.Path,
		}).Debug()
		t := time.Now()
		req.Header.Set("Date", t.Format(time.RFC3339))

		bodyData, err := ioutil.ReadAll(req.Body)
		if err != nil {
			log.WithFields(log.Fields{
				"method": req.Method,
				"path":   req.URL.Path,
			}).Errorf("Failed to consume body %v", err)
			return
		}
		buf := bytes.NewReader(bodyData)

		if _, err := signer.Sign(req, buf, "es", *region, t); err != nil {
			log.WithFields(log.Fields{
				"method": req.Method,
				"path":   req.URL.Path,
			}).Errorf("Failed to sign request %v", err)
		}
	}
	proxy := &httputil.ReverseProxy{Director: director}
	log.Fatal(http.ListenAndServe(":9200", proxy))
}
<|endoftext|>


<commit_before>package main

import (
	"os"

	"github.com/MartinSahlen/go-cloud-fn/express-wrapper"
	"github.com/MartinSahlen/go-cloud-fn/router"
	"github.com/gopherjs/gopherjs/js"
)

var googleCloudFunctionName = os.Getenv("GOOGLE_CLOUD_FUNCTION_NAME")

func rootHandler(res express.Response, req express.Request) {
	res.Headers.Write("content-type", "application/json")
	res.Status = 404
	res.Write(req.JSON())
}

func helloHandler(res express.Response, req express.Request) {
	res.Headers.Write("content-type", "application/json")
	res.Status = 200
	res.Write(req.JSON())
}

//EntryPoint is the main handler and entrypoint for the google cloud function
func EntryPoint(req, res *js.Object) {

	r := router.New(rootHandler)
	r.Handle("GET", "/hello/:ergegr", helloHandler)

	r.Serve(express.NewResponse(res), express.NewRequest(req))
}

func main() {
	if googleCloudFunctionName == "" {
		googleCloudFunctionName = "helloGO"
	}
	js.Module.Get("exports").Set(googleCloudFunctionName, EntryPoint)
}
<commit_msg>Test request<commit_after>package main

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"net/http"
	"os"

	"github.com/MartinSahlen/go-cloud-fn/express-wrapper"
	"github.com/MartinSahlen/go-cloud-fn/router"
	"github.com/gopherjs/gopherjs/js"
)

var googleCloudFunctionName = os.Getenv("GOOGLE_CLOUD_FUNCTION_NAME")

func rootHandler(res express.Response, req express.Request) {
	res.Headers.Write("content-type", "application/json")
	res.Status = 404
	res.Write(req.JSON())
}

func helloHandler(res express.Response, req express.Request) {
	res.Headers.Write("content-type", "application/json")
	res.Status = 200
	res.Write(req.JSON())
}

type Website struct {
	URL string `json:"url"`
}

func websiteHandler(res express.Response, req express.Request) {

	var site Website
	err := json.Unmarshal(req.Body, &site)

	res.Headers.Write("content-type", "text/html")

	log.Println(string(req.Body))

	if err != nil {
		log.Println("json error" + err.Error())
		res.Write([]byte(err.Error()))
		return
	}

	log.Println(site)

	go func() {
		r, err := http.DefaultClient.Get(site.URL)

		if err != nil {
			log.Println(err.Error())
			res.Write([]byte(err.Error()))
			return
		}

		byt, err := ioutil.ReadAll(r.Body)

		if err != nil {
			log.Println(err.Error())
			res.Write([]byte(err.Error()))
			return
		}

		res.Status = 200
		res.Write(byt)
	}()

}

//EntryPoint is the main handler and entrypoint for the google cloud function
func EntryPoint(req, res *js.Object) {

	r := router.New(rootHandler)
	r.Handle("GET", "/hello/:ergegr", helloHandler)
	r.Handle("POST", "/site", websiteHandler)

	r.Serve(express.NewResponse(res), express.NewRequest(req))
}

func main() {
	if googleCloudFunctionName == "" {
		googleCloudFunctionName = "helloGO"
	}
	js.Module.Get("exports").Set(googleCloudFunctionName, EntryPoint)
}
<|endoftext|>


<commit_before><commit_msg>Fix issue with regex matching party votes.<commit_after>package main

import (
	"fmt"
	"log"
	"regexp"
	"os"
	"text/template"
	"strings"
	"strconv"

	"github.com/huandu/facebook"
)

type entry struct {
	District string
	Constituency string
	CPP int
	NDP int
	NDC int
	PPP int
	NPP int
	PNC int
	IND int
	Total int
	Rejected int
	InBox int
}

var (
	appID = "<app_id>"
	appSecret = "<app_secret>"
	profileID = "310028155725467" // Electoral Commission Ghana Page
	pattern = "Presidential Provisional Results"
	session *facebook.Session
)

func main() {
	log.Print("Checking for new posts ...")

	fb := facebook.New(appID, appSecret)
	token := fb.AppAccessToken()

	session = fb.Session(token)
	n := readFeed(profileID, token)

	log.Print("Got ", n, " declarations.")
}

func readFeed(profileID, token string) int {
	count := 0

	csvheader := "District,Constituency,CPP,NDP,NDC,PPP,NPP,PNC,IND,Total,Rejected,In-Box"

	fmt.Println(csvheader)

	url := fmt.Sprintf("/%s/feed", profileID)
	res, err := session.Get(url, facebook.Params{
		"access_token": token,
		"since": "1481155200",
		"limit": "100",
	})

	if err != nil {
		log.Fatal(err)
	}

	paging, _ := res.Paging(session)
	items := paging.Data()

	for ;; {
		items = paging.Data()
		for _, item := range items {
			msg := fmt.Sprintf("%s",item["message"])
			match, _ := regexp.MatchString(pattern, msg)
			if match {
				getData(msg)
				count++
			}
		}

		if paging.HasNext() {
			paging.Next()
		} else {
			break
		}
	}

	return count
}

func getData(msg string) {
	row := &entry{}

	const line = `{{.District}},{{.Constituency}},{{.CPP}},{{.NDP}},{{.NDC}},{{.PPP}},{{.NPP}},{{.PNC}},{{.IND}},{{.Total}},{{.Rejected}},{{.InBox}}
`
	tmpl := template.Must(template.New("line").Parse(line))

	re := regexp.MustCompile(`Name of District: (?P<dist>[\w ]*)`)
	match := re.FindStringSubmatch(msg)
	row.District = match[1]

	re = regexp.MustCompile(`Name of Constituency: (?P<const>[\w ]*)`)
	match = re.FindStringSubmatch(msg)
	row.Constituency = match[1]

	re = regexp.MustCompile(`CPP:\s*(?P<cpp>[\d,]*)`)
	match = re.FindStringSubmatch(msg)
	row.CPP, _ = strconv.Atoi(strings.Replace(match[1], ",", "", -1))

	re = regexp.MustCompile(`NDP:\s*(?P<ndp>[\d,]*)`)
	match = re.FindStringSubmatch(msg)
	row.NDP, _ = strconv.Atoi(strings.Replace(match[1], ",", "", -1))

	re = regexp.MustCompile(`NDC:\s*(?P<ndc>[\d,]*)`)
	match = re.FindStringSubmatch(msg)
	row.NDC, _ = strconv.Atoi(strings.Replace(match[1], ",", "", -1))

	re = regexp.MustCompile(`PPP:\s*(?P<ppp>[\d,]*)`)
	match = re.FindStringSubmatch(msg)
	row.PPP, _ = strconv.Atoi(strings.Replace(match[1], ",", "", -1))

	re = regexp.MustCompile(`NPP:\s*(?P<npp>[\d,]*)`)
	match = re.FindStringSubmatch(msg)
	row.NPP, _ = strconv.Atoi(strings.Replace(match[1], ",", "", -1))

	re = regexp.MustCompile(`PNC:\s*(?P<pnc>[\d,]*)`)
	match = re.FindStringSubmatch(msg)
	row.PNC, _ = strconv.Atoi(strings.Replace(match[1], ",", "", -1))

	re = regexp.MustCompile(`IND:\s*(?P<ind>[\d,]*)`)
	match = re.FindStringSubmatch(msg)
	row.IND, _ = strconv.Atoi(strings.Replace(match[1], ",", "", -1))

	re = regexp.MustCompile(`Total Votes:\s*(?P<total>[\d,]*)`)
	match = re.FindStringSubmatch(msg)
	row.Total, _ = strconv.Atoi(strings.Replace(match[1], ",", "", -1))

	re = regexp.MustCompile(`Rejected Votes:\s*(?P<rejected>[\d,]*)`)
	match = re.FindStringSubmatch(msg)
	row.Rejected, _ = strconv.Atoi(strings.Replace(match[1], ",", "", -1))

	re = regexp.MustCompile(`Total Votes in Ballot Box:\s*(?P<inbox>[\d,]*)`)
	match = re.FindStringSubmatch(msg)
	row.InBox, _ = strconv.Atoi(strings.Replace(match[1], ",", "", -1))

	err := tmpl.Execute(os.Stdout, row)
	if err != nil {
		log.Fatal(err)
	}
}

func searchPage(token string) {
	res, _ := facebook.Get("/search", facebook.Params{
		"access_token": token,
		"type":         "page",
		"q":            "Electoral Commission Ghana",
	})

	var items []facebook.Result
	err := res.DecodeField("data", &items)
	if err != nil {
		log.Fatal(err)
	}

	for _, item := range items {
		fmt.Println(item)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"os/exec"
	"strings"
)

func isInGit(wd string) bool {
	cmd := exec.Command("git", "rev-parse")
	cmd.Dir = wd
	output, err := cmd.CombinedOutput()
	return len(output) == 0 && err == nil
}

func main() {
	cwd, _ := os.Getwd()
	if !isInGit(cwd) {
		fmt.Println("This is not a git repo :(")
		return
	}

	message := "Updates..."
	if len(os.Args) > 1 {
		words := os.Args[1:]
		message = strings.Join(words, " ")
	}

	addCmd := exec.Command("git", "add", "-A")
	addCmd.Dir = cwd
	if err := addCmd.Run(); err != nil {
		fmt.Println(err.Error())
		return
	}

	commitCmd := exec.Command("git", "commit", "-am\""+message)
	commitCmd.Dir = cwd
	if err := commitCmd.Run(); err != nil {
		fmt.Println(err.Error())
		return
	}

	pullCmd := exec.Command("git", "pull")
	pullCmd.Dir = cwd
	if err := pullCmd.Run(); err != nil {
		fmt.Println(err.Error())
		return
	}

	pushCmd := exec.Command("git push")
	pushCmd.Dir = cwd
	if err := pushCmd.Run(); err != nil {
		fmt.Println(err.Error())
		return
	}
}
<commit_msg>"Updates...<commit_after>package main

import (
	"fmt"
	"os"
	"os/exec"
	"strings"
)

func isInGit(wd string) bool {
	cmd := exec.Command("git", "rev-parse")
	cmd.Dir = wd
	output, err := cmd.CombinedOutput()
	return len(output) == 0 && err == nil
}

func runCmd(wd, app string, args ...string) bool {
	cmd := exec.Command(app, args...)
	cmd.Dir = wd

	output, err := cmd.CombinedOutput()
	if err != nil {
		fmt.Println("\n==============================\n")
		fmt.Println("\nError :( git had this to say: \n")
		fmt.Println("\n==============================\n")
		fmt.Printf("%s", output)
		fmt.Println("\n==============================\n")
	}
	return true
}

func main() {
	cwd, _ := os.Getwd()
	if !isInGit(cwd) {
		fmt.Println("This is not a git repo :(")
		return
	}

	message := "Updates..."
	if len(os.Args) > 1 {
		words := os.Args[1:]
		message = strings.Join(words, " ")
	}

	if !runCmd(cwd, "git", "-A") {
		return
	}
	if !runCmd(cwd, "git", "commit", "-am\""+message) {
		return
	}
	if !runCmd(cwd, "git", "pull") {
		return
	}
	if !runCmd(cwd, "git", "push") {
		return
	}

}
<|endoftext|>


<commit_before>package robification

import (
	"bytes"
	"encoding/json"
	"errors"
	"io/ioutil"
	"net/http"
)

func Send(p *fdChat) error {
	url := "http://jrobles.net:1337/v1/flowdock/chat"

	req, err := http.NewRequest("POST", url, bytes.NewBuffer([]byte(p.Content)))
	if err != nil {
		panic(err)
	}
	req.Header.Set("Token", p.Flow_Token)
	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		panic(err)
	}
	defer resp.Body.Close()

	body, _ := ioutil.ReadAll(resp.Body)
	res := apiResponse{}
	json.Unmarshal([]byte(body), &res)

	// Hard-coding one response for now...
	if res.Messages[0].Status == "200 OK" {
		return nil
	}
	return errors.New(res.Messages[0].Status)

}

func NewFdChat(flowToken string, content string) *fdChat {
	chat := &fdChat{
		Flow_Token: flowToken,
		Content:    content,
	}
	return chat
}
<commit_msg>robification.io<commit_after>package robification

import (
	"bytes"
	"encoding/json"
	"errors"
	"io/ioutil"
	"net/http"
)

func Send(p *fdChat) error {
	url := "http://api.robification.io:1337/v1/flowdock/chat"

	req, err := http.NewRequest("POST", url, bytes.NewBuffer([]byte(p.Content)))
	if err != nil {
		panic(err)
	}
	req.Header.Set("Token", p.Flow_Token)
	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		panic(err)
	}
	defer resp.Body.Close()

	body, _ := ioutil.ReadAll(resp.Body)
	res := apiResponse{}
	json.Unmarshal([]byte(body), &res)

	// Hard-coding one response for now...
	if res.Messages[0].Status == "200 OK" {
		return nil
	}
	return errors.New(res.Messages[0].Status)

}

func NewFdChat(flowToken string, content string) *fdChat {
	chat := &fdChat{
		Flow_Token: flowToken,
		Content:    content,
	}
	return chat
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/tcnksm/go-gitconfig"
)

//go:generate go-bindata _templates/...

const defaultVCS = "github.com"

func usage() {
	fmt.Fprintf(os.Stderr, `Usage of %s:
	%s new <project name>
	%s gen -d <model directory -o <output directory>`,
		os.Args[0], os.Args[0], os.Args[0])
	os.Exit(1)
}

func main() {

	if len(os.Args) < 2 {
		usage()
	}

	cmd := os.Args[1]

	switch cmd {
	case "gen":
		var (
			modelDir string
			outDir   string
		)

		flag := flag.NewFlagSet(os.Args[0], flag.ExitOnError)

		flag.Usage = func() {
			fmt.Fprintf(os.Stderr, `Usage of %s:
	%s gen -d <model directory> -o <output directory>

Options:
`, os.Args[0], os.Args[0])
			flag.PrintDefaults()
		}

		flag.StringVar(&modelDir, "d", "", "Model directory")
		flag.StringVar(&outDir, "o", "", "Output directory")

		flag.Parse(os.Args[2:])

		if modelDir == "" || outDir == "" {
			flag.Usage()
			os.Exit(1)
		}
		cmdGen(modelDir, outDir)

	case "new":
		var (
			vcs      string
			username string
		)

		flag := flag.NewFlagSet(os.Args[0], flag.ExitOnError)

		flag.Usage = func() {
			fmt.Fprintf(os.Stderr, `Usage of %s:
	%s new <project name>

Options:
`, os.Args[0], os.Args[0])
			flag.PrintDefaults()
		}

		flag.StringVar(&vcs, "v", "", "VCS")
		flag.StringVar(&username, "u", "", "Username")

		if len(os.Args) < 3 {
			flag.Usage()
			os.Exit(1)
		}

		flag.Parse(os.Args[3:])

		if vcs == "" {
			vcs = defaultVCS
		}

		if username == "" {
			var err error
			username, err = gitconfig.GithubUser()

			if err != nil {
				username, err = gitconfig.Username()
				if err != nil {
					msg := "Cannot find `~/.gitcofig` file.\n" +
						"Please use -u option"
					fmt.Println(msg)
					os.Exit(1)
				}
			}
		}

		project := os.Args[2]

		detail := Detail{vcs, username, project}

		cmdNew(detail)

	default:
		usage()
	}

}

func cmdNew(detail Detail) {
	gopath := os.Getenv("GOPATH")

	if gopath == "" {
		fmt.Println("Error: $GOPATH is not found")
		os.Exit(1)
	}

	outDir := filepath.Join(gopath, "src", detail.VCS, detail.User, detail.Project)

	if err := generateSkeleton(detail, outDir); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}

func cmdGen(modelDir, outDir string) {
	if !fileExists(outDir) {
		if err := mkdir(outDir); err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
	}

	files, err := ioutil.ReadDir(modelDir)

	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	var models []*Model

	for _, file := range files {
		if file.IsDir() {
			continue
		}

		if !strings.HasSuffix(file.Name(), ".go") {
			continue
		}

		modelPath := filepath.Join(modelDir, file.Name())
		ms, err := parseFile(modelPath)

		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}

		for _, model := range ms {
			models = append(models, model)
		}
	}

	if err := generateREADME(models, outDir); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	if err := generateRouter(models, outDir); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	for _, model := range models {
		if err := generateController(model, outDir); err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
	}
}
<commit_msg>Fix typo<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/tcnksm/go-gitconfig"
)

//go:generate go-bindata _templates/...

const defaultVCS = "github.com"

func usage() {
	fmt.Fprintf(os.Stderr, `Usage of %s:
	%s new <project name>
	%s gen -d <model directory> -o <output directory>`,
		os.Args[0], os.Args[0], os.Args[0])
	os.Exit(1)
}

func main() {

	if len(os.Args) < 2 {
		usage()
	}

	cmd := os.Args[1]

	switch cmd {
	case "gen":
		var (
			modelDir string
			outDir   string
		)

		flag := flag.NewFlagSet(os.Args[0], flag.ExitOnError)

		flag.Usage = func() {
			fmt.Fprintf(os.Stderr, `Usage of %s:
	%s gen -d <model directory> -o <output directory>

Options:
`, os.Args[0], os.Args[0])
			flag.PrintDefaults()
		}

		flag.StringVar(&modelDir, "d", "", "Model directory")
		flag.StringVar(&outDir, "o", "", "Output directory")

		flag.Parse(os.Args[2:])

		if modelDir == "" || outDir == "" {
			flag.Usage()
			os.Exit(1)
		}
		cmdGen(modelDir, outDir)

	case "new":
		var (
			vcs      string
			username string
		)

		flag := flag.NewFlagSet(os.Args[0], flag.ExitOnError)

		flag.Usage = func() {
			fmt.Fprintf(os.Stderr, `Usage of %s:
	%s new <project name>

Options:
`, os.Args[0], os.Args[0])
			flag.PrintDefaults()
		}

		flag.StringVar(&vcs, "v", "", "VCS")
		flag.StringVar(&username, "u", "", "Username")

		if len(os.Args) < 3 {
			flag.Usage()
			os.Exit(1)
		}

		flag.Parse(os.Args[3:])

		if vcs == "" {
			vcs = defaultVCS
		}

		if username == "" {
			var err error
			username, err = gitconfig.GithubUser()

			if err != nil {
				username, err = gitconfig.Username()
				if err != nil {
					msg := "Cannot find `~/.gitconfig` file.\n" +
						"Please use -u option"
					fmt.Println(msg)
					os.Exit(1)
				}
			}
		}

		project := os.Args[2]

		detail := Detail{vcs, username, project}

		cmdNew(detail)

	default:
		usage()
	}

}

func cmdNew(detail Detail) {
	gopath := os.Getenv("GOPATH")

	if gopath == "" {
		fmt.Println("Error: $GOPATH is not found")
		os.Exit(1)
	}

	outDir := filepath.Join(gopath, "src", detail.VCS, detail.User, detail.Project)

	if err := generateSkeleton(detail, outDir); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}

func cmdGen(modelDir, outDir string) {
	if !fileExists(outDir) {
		if err := mkdir(outDir); err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
	}

	files, err := ioutil.ReadDir(modelDir)

	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	var models []*Model

	for _, file := range files {
		if file.IsDir() {
			continue
		}

		if !strings.HasSuffix(file.Name(), ".go") {
			continue
		}

		modelPath := filepath.Join(modelDir, file.Name())
		ms, err := parseFile(modelPath)

		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}

		for _, model := range ms {
			models = append(models, model)
		}
	}

	if err := generateREADME(models, outDir); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	if err := generateRouter(models, outDir); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	for _, model := range models {
		if err := generateController(model, outDir); err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"time"

	"github.com/cloudfoundry-attic/jibber_jabber"
	"github.com/itchio/itch-setup/bindata"
	"github.com/itchio/itch-setup/cl"
	"github.com/itchio/itch-setup/localize"
	"github.com/itchio/itch-setup/native"
	"github.com/pkg/errors"

	kingpin "gopkg.in/alecthomas/kingpin.v2"
)

var (
	version       = "head" // set by command-line on CI release builds
	builtAt       = ""     // set by command-line on CI release builds
	commit        = ""     // set by command-line on CI release builds
	appName       = ""     // set by command-line on CI release builds
	versionString = ""     // formatted on boot from 'version' and 'builtAt'
	app           = kingpin.New("itch-setup", "The itch installer and self-updater")
)

var cli = cl.CLI

func init() {
	app.Flag("prefer-launch", "Launch if a valid version of itch is installed").BoolVar(&cli.PreferLaunch)

	app.Flag("upgrade", "Upgrade the itch app if necessary").BoolVar(&cli.Upgrade)

	app.Flag("uninstall", "Uninstall the itch app").BoolVar(&cli.Uninstall)

	app.Flag("relaunch", "Relaunch a new version of the itch app").BoolVar(&cli.Relaunch)
	app.Flag("relaunch-pid", "PID to wait for before relaunching").IntVar(&cli.RelaunchPID)

	app.Flag("appname", "Application name (itch or kitch)").StringVar(&cli.AppName)

	app.Flag("silent", "Run installation silently").BoolVar(&cli.Silent)

	app.Arg("args", "Arguments to pass down to itch (only supported on Linux & Windows)").StringsVar(&cli.Args)
}

func must(err error) {
	if err != nil {
		log.Fatalf("%+v", err)
	}
}

func detectAppName() {
	if cli.AppName != "" {
		log.Printf("App name specified on command-line: %s", cli.AppName)
	} else if appName != "" {
		log.Printf("App name specified at build time: %s", cli.AppName)
	} else {
		execPath, err := os.Executable()
		must(err)

		ext := ""
		if runtime.GOOS == "windows" {
			ext = ".exe"
		}
		kitchBinary := fmt.Sprintf("kitch-setup%s", ext)

		if filepath.Base(execPath) == kitchBinary {
			cli.AppName = "kitch"
		} else {
			cli.AppName = "itch"
		}
		log.Printf("App name detected: %s", cli.AppName)
	}

	app.Name = fmt.Sprintf("%s-setup", cli.AppName)
}

const DefaultLocale = "en-US"

var localizer *localize.Localizer

func main() {
	app.UsageTemplate(kingpin.CompactUsageTemplate)

	app.HelpFlag.Short('h')
	if builtAt != "" {
		epoch, err := strconv.ParseInt(builtAt, 10, 64)
		must(err)
		versionString = fmt.Sprintf("%s, built on %s", version, time.Unix(epoch, 0).Format("Jan _2 2006 @ 15:04:05"))
	} else {
		versionString = fmt.Sprintf("%s, no build date", version)
	}
	if commit != "" {
		versionString = fmt.Sprintf("%s, ref %s", versionString, commit)
	}

	app.Version(versionString)
	app.VersionFlag.Short('V')
	app.Author("Amos Wenger <amos@itch.io>")

	cli.VersionString = versionString

	_, err := app.Parse(os.Args[1:])
	must(err)

	detectAppName()

	userLocale, err := jibber_jabber.DetectIETF()
	if err != nil {
		log.Println("Couldn't detect locale, falling back to default", DefaultLocale)
		userLocale = "en-US"
	}

	log.Println("Locale: ", userLocale)

	localizer, err = localize.NewLocalizer(bindata.Asset)
	if err != nil {
		log.Fatal(err)
	}

	err = localizer.LoadLocale(userLocale)
	if err != nil {
		userLocale = userLocale[:2]
		err = localizer.LoadLocale(userLocale)
	}

	if err == nil {
		localizer.SetLang(userLocale)
	}
	cli.Localizer = localizer

	nc, err := native.NewNativeCore(cli)
	if err != nil {
		panic(err)
	}

	var verbs []string

	if cli.Upgrade {
		verbs = append(verbs, "upgrade")
	}
	if cli.Relaunch {
		verbs = append(verbs, "relaunch")
	}
	if cli.Uninstall {
		verbs = append(verbs, "uninstall")
	}

	if len(verbs) > 1 {
		nc.ErrorDialog(errors.Errorf("Cannot specify more than one verb: got %s", strings.Join(verbs, ", ")))
	}

	if len(verbs) == 0 {
		verbs = append(verbs, "install")
	}

	switch verbs[0] {
	case "install":
		err = nc.Install()
		if err != nil {
			nc.ErrorDialog(err)
		}
	case "upgrade":
		err = nc.Upgrade()
		if err != nil {
			jsonlBail(errors.WithMessage(err, "Fatal upgrade error"))
		}
	case "relaunch":
		if cli.RelaunchPID <= 0 {
			jsonlBail(errors.Errorf("--relaunch needs a valid --relaunch-pid (got %d)", cli.RelaunchPID))
		}

		err = nc.Relaunch()
		if err != nil {
			jsonlBail(errors.WithMessage(err, "Fatal relaunch error"))
		}
	case "uninstall":
		err = nc.Uninstall()
		if err != nil {
			nc.ErrorDialog(err)
		}
	}
}

func jsonlBail(err error) {
	// TODO: use json-lines
	log.Fatalf("%+v", err)
}
<commit_msg>at last, a compile error<commit_after>package main

import (
	"fmt"
	"log"
	"os"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"time"

	"github.com/cloudfoundry-attic/jibber_jabber"
	"github.com/itchio/itch-setup/bindata"
	"github.com/itchio/itch-setup/cl"
	"github.com/itchio/itch-setup/localize"
	"github.com/itchio/itch-setup/native"
	"github.com/pkg/errors"

	kingpin "gopkg.in/alecthomas/kingpin.v2"
)

var (
	version       = "head" // set by command-line on CI release builds
	builtAt       = ""     // set by command-line on CI release builds
	commit        = ""     // set by command-line on CI release builds
	appName       = ""     // set by command-line on CI release builds
	versionString = ""     // formatted on boot from 'version' and 'builtAt'
	app           = kingpin.New("itch-setup", "The itch installer and self-updater")
)

var cli cl.CLI

func init() {
	app.Flag("prefer-launch", "Launch if a valid version of itch is installed").BoolVar(&cli.PreferLaunch)

	app.Flag("upgrade", "Upgrade the itch app if necessary").BoolVar(&cli.Upgrade)

	app.Flag("uninstall", "Uninstall the itch app").BoolVar(&cli.Uninstall)

	app.Flag("relaunch", "Relaunch a new version of the itch app").BoolVar(&cli.Relaunch)
	app.Flag("relaunch-pid", "PID to wait for before relaunching").IntVar(&cli.RelaunchPID)

	app.Flag("appname", "Application name (itch or kitch)").StringVar(&cli.AppName)

	app.Flag("silent", "Run installation silently").BoolVar(&cli.Silent)

	app.Arg("args", "Arguments to pass down to itch (only supported on Linux & Windows)").StringsVar(&cli.Args)
}

func must(err error) {
	if err != nil {
		log.Fatalf("%+v", err)
	}
}

func detectAppName() {
	if cli.AppName != "" {
		log.Printf("App name specified on command-line: %s", cli.AppName)
	} else if appName != "" {
		log.Printf("App name specified at build time: %s", cli.AppName)
	} else {
		execPath, err := os.Executable()
		must(err)

		ext := ""
		if runtime.GOOS == "windows" {
			ext = ".exe"
		}
		kitchBinary := fmt.Sprintf("kitch-setup%s", ext)

		if filepath.Base(execPath) == kitchBinary {
			cli.AppName = "kitch"
		} else {
			cli.AppName = "itch"
		}
		log.Printf("App name detected: %s", cli.AppName)
	}

	app.Name = fmt.Sprintf("%s-setup", cli.AppName)
}

const DefaultLocale = "en-US"

var localizer *localize.Localizer

func main() {
	app.UsageTemplate(kingpin.CompactUsageTemplate)

	app.HelpFlag.Short('h')
	if builtAt != "" {
		epoch, err := strconv.ParseInt(builtAt, 10, 64)
		must(err)
		versionString = fmt.Sprintf("%s, built on %s", version, time.Unix(epoch, 0).Format("Jan _2 2006 @ 15:04:05"))
	} else {
		versionString = fmt.Sprintf("%s, no build date", version)
	}
	if commit != "" {
		versionString = fmt.Sprintf("%s, ref %s", versionString, commit)
	}

	app.Version(versionString)
	app.VersionFlag.Short('V')
	app.Author("Amos Wenger <amos@itch.io>")

	cli.VersionString = versionString

	_, err := app.Parse(os.Args[1:])
	must(err)

	detectAppName()

	userLocale, err := jibber_jabber.DetectIETF()
	if err != nil {
		log.Println("Couldn't detect locale, falling back to default", DefaultLocale)
		userLocale = "en-US"
	}

	log.Println("Locale: ", userLocale)

	localizer, err = localize.NewLocalizer(bindata.Asset)
	if err != nil {
		log.Fatal(err)
	}

	err = localizer.LoadLocale(userLocale)
	if err != nil {
		userLocale = userLocale[:2]
		err = localizer.LoadLocale(userLocale)
	}

	if err == nil {
		localizer.SetLang(userLocale)
	}
	cli.Localizer = localizer

	nc, err := native.NewNativeCore(cli)
	if err != nil {
		panic(err)
	}

	var verbs []string

	if cli.Upgrade {
		verbs = append(verbs, "upgrade")
	}
	if cli.Relaunch {
		verbs = append(verbs, "relaunch")
	}
	if cli.Uninstall {
		verbs = append(verbs, "uninstall")
	}

	if len(verbs) > 1 {
		nc.ErrorDialog(errors.Errorf("Cannot specify more than one verb: got %s", strings.Join(verbs, ", ")))
	}

	if len(verbs) == 0 {
		verbs = append(verbs, "install")
	}

	switch verbs[0] {
	case "install":
		err = nc.Install()
		if err != nil {
			nc.ErrorDialog(err)
		}
	case "upgrade":
		err = nc.Upgrade()
		if err != nil {
			jsonlBail(errors.WithMessage(err, "Fatal upgrade error"))
		}
	case "relaunch":
		if cli.RelaunchPID <= 0 {
			jsonlBail(errors.Errorf("--relaunch needs a valid --relaunch-pid (got %d)", cli.RelaunchPID))
		}

		err = nc.Relaunch()
		if err != nil {
			jsonlBail(errors.WithMessage(err, "Fatal relaunch error"))
		}
	case "uninstall":
		err = nc.Uninstall()
		if err != nil {
			nc.ErrorDialog(err)
		}
	}
}

func jsonlBail(err error) {
	// TODO: use json-lines
	log.Fatalf("%+v", err)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"os"
	"strings"
	"time"

	"github.com/contactless/wb-rules/wbrules"
	"github.com/contactless/wbgo"
)

const (
	DRIVER_CLIENT_ID = "rules"
	DRIVER_CONV_ID   = "wb-rules"
	ENGINE_CLIENT_ID = "wb-rules-engine"

	PERSISTENT_DB_FILE      = "/var/lib/wirenboard/wbrules-persistent.db"
	VIRTUAL_DEVICES_DB_FILE = "/var/lib/wirenboard/wbrules-vdev.db"

	WBRULES_MODULES_ENV = "WB_RULES_MODULES"
)

func main() {
	brokerAddress := flag.String("broker", "tcp://localhost:1883", "MQTT broker url")
	editDir := flag.String("editdir", "", "Editable script directory")
	debug := flag.Bool("debug", false, "Enable debugging")
	useSyslog := flag.Bool("syslog", false, "Use syslog for logging")
	mqttDebug := flag.Bool("mqttdebug", false, "Enable MQTT debugging")
	flag.Parse()
	if flag.NArg() < 1 {
		wbgo.Error.Fatal("must specify rule file/directory name(s)")
	}
	if *useSyslog {
		wbgo.UseSyslog()
	}
	if *debug {
		wbgo.SetDebuggingEnabled(true)
	}
	if *mqttDebug {
		wbgo.EnableMQTTDebugLog(*useSyslog)
	}
	wbgo.MaybeInitProfiling(nil)

	// model := wbrules.NewCellModel()
	driverMqttClient := wbgo.NewPahoMQTTClient(*brokerAddress, DRIVER_CLIENT_ID)
	// driver := wbgo.NewDriver(model, mqttClient)
	driver, err := wbgo.NewDriverBase(wbgo.NewDriverArgs().SetId(DRIVER_CONV_ID).SetMqtt(driverMqttClient).SetStoragePath(VIRTUAL_DEVICES_DB_FILE))
	if err != nil {
		wbgo.Error.Fatalf("error creating driver: %s", err)
	}
	driver.SetFilter(&wbgo.AllDevicesFilter{})

	engineOptions := wbrules.NewESEngineOptions()
	engineOptions.SetPersistentDBFile(PERSISTENT_DB_FILE)
	engineOptions.SetModulesDirs(strings.Split(os.Getenv(WBRULES_MODULES_ENV), ":"))

	engineMqttClient := wbgo.NewPahoMQTTClient(*brokerAddress, ENGINE_CLIENT_ID)
	engine, err := wbrules.NewESEngine(driver, engineMqttClient, engineOptions)
	if err != nil {
		wbgo.Error.Fatalf("error creating engine: %s", err)
	}

	gotSome := false
	watcher := wbgo.NewDirWatcher("\\.js$", engine)
	if *editDir != "" {
		engine.SetSourceRoot(*editDir)
	}
	for _, path := range flag.Args() {
		if err := watcher.Load(path); err != nil {
			wbgo.Error.Printf("error loading script file/dir %s: %s", path, err)
		} else {
			gotSome = true
		}
	}
	if !gotSome {
		wbgo.Error.Fatalf("no valid scripts found")
	}

	if err := driver.StartLoop(); err != nil {
		wbgo.Error.Fatalf("error starting the driver: %s", err)
	}

	if *editDir != "" {
		rpc := wbgo.NewMQTTRPCServer("wbrules", engineMqttClient)
		rpc.Register(wbrules.NewEditor(engine))
		rpc.Start()
	}

	engine.Start()

	for {
		time.Sleep(1 * time.Second)
	}
}
<commit_msg>wbrules: fix start sequence<commit_after>package main

import (
	"flag"
	"os"
	"strings"
	"time"

	"github.com/contactless/wb-rules/wbrules"
	"github.com/contactless/wbgo"
)

const (
	DRIVER_CLIENT_ID = "rules"
	DRIVER_CONV_ID   = "wb-rules"
	ENGINE_CLIENT_ID = "wb-rules-engine"

	PERSISTENT_DB_FILE      = "/var/lib/wirenboard/wbrules-persistent.db"
	VIRTUAL_DEVICES_DB_FILE = "/var/lib/wirenboard/wbrules-vdev.db"

	WBRULES_MODULES_ENV = "WB_RULES_MODULES"
)

func main() {
	brokerAddress := flag.String("broker", "tcp://localhost:1883", "MQTT broker url")
	editDir := flag.String("editdir", "", "Editable script directory")
	debug := flag.Bool("debug", false, "Enable debugging")
	useSyslog := flag.Bool("syslog", false, "Use syslog for logging")
	mqttDebug := flag.Bool("mqttdebug", false, "Enable MQTT debugging")
	flag.Parse()
	if flag.NArg() < 1 {
		wbgo.Error.Fatal("must specify rule file/directory name(s)")
	}
	if *useSyslog {
		wbgo.UseSyslog()
	}
	if *debug {
		wbgo.SetDebuggingEnabled(true)
	}
	if *mqttDebug {
		wbgo.EnableMQTTDebugLog(*useSyslog)
	}
	wbgo.MaybeInitProfiling(nil)

	// model := wbrules.NewCellModel()
	driverMqttClient := wbgo.NewPahoMQTTClient(*brokerAddress, DRIVER_CLIENT_ID)
	// driver := wbgo.NewDriver(model, mqttClient)
	driver, err := wbgo.NewDriverBase(wbgo.NewDriverArgs().SetId(DRIVER_CONV_ID).SetMqtt(driverMqttClient).SetStoragePath(VIRTUAL_DEVICES_DB_FILE))
	if err != nil {
		wbgo.Error.Fatalf("error creating driver: %s", err)
	}

	if err := driver.StartLoop(); err != nil {
		wbgo.Error.Fatalf("error starting the driver: %s", err)
	}

	driver.SetFilter(&wbgo.AllDevicesFilter{})

	engineOptions := wbrules.NewESEngineOptions()
	engineOptions.SetPersistentDBFile(PERSISTENT_DB_FILE)
	engineOptions.SetModulesDirs(strings.Split(os.Getenv(WBRULES_MODULES_ENV), ":"))

	engineMqttClient := wbgo.NewPahoMQTTClient(*brokerAddress, ENGINE_CLIENT_ID)
	engine, err := wbrules.NewESEngine(driver, engineMqttClient, engineOptions)
	if err != nil {
		wbgo.Error.Fatalf("error creating engine: %s", err)
	}

	gotSome := false
	watcher := wbgo.NewDirWatcher("\\.js$", engine)
	if *editDir != "" {
		engine.SetSourceRoot(*editDir)
	}
	for _, path := range flag.Args() {
		if err := watcher.Load(path); err != nil {
			wbgo.Error.Printf("error loading script file/dir %s: %s", path, err)
		} else {
			gotSome = true
		}
	}
	if !gotSome {
		wbgo.Error.Fatalf("no valid scripts found")
	}

	if *editDir != "" {
		rpc := wbgo.NewMQTTRPCServer("wbrules", engineMqttClient)
		rpc.Register(wbrules.NewEditor(engine))
		rpc.Start()
	}

	engine.Start()

	for {
		time.Sleep(1 * time.Second)
	}
}
<|endoftext|>


<commit_before>package s3log

import (
	"fmt"
	"log"
	"net"
	"regexp"
	"strconv"
	"strings"
	"time"
)

type Entry struct {
	Owner      string
	Bucket     string
	Time       time.Time
	Remote     net.IP
	Requester  string
	RequestID  string
	Operation  string
	Key        string
	RequestURI string
	Status     int
	Error      string
	Bytes      int
	Size       int
	Total      int
	Turnaround int
	Referrer   string
	UserAgent  string
	Version    string
}

var logLine = regexp.MustCompile(`[^" ]+|("[^"]*")`)
var brackets = regexp.MustCompile(`[\[\]]`)

func Parse(line string) (Entry, error) {
	e := Entry{}
	l := logLine.FindAllString(line, -1)

	e.Owner = l[0]
	e.Bucket = l[1]
	pt, err := parseTime(l[2] + " " + l[3])
	if err != nil {
		return e, err
	}
	e.Time = pt
	e.Remote = net.ParseIP(l[4])
	e.Requester = l[5]
	e.RequestID = l[6]
	e.Operation = l[7]
	e.Key = l[8]

	prequestURI := l[9]
	e.RequestURI = strings.Replace(prequestURI, `"`, "", -1)

	pstatus, err := strconv.Atoi(l[10])
	if err != nil {
		return e, err
	}
	e.Status = pstatus

	perrorRaw := l[11]
	if perrorRaw == "-" {
		e.Error = ""
	} else {
		e.Error = perrorRaw
	}
	e.Error = perrorRaw

	pbytesRaw := l[12]
	pbytes := 0
	if pbytesRaw != "-" {
		pbytes, err = strconv.Atoi(pbytesRaw)
		if err != nil {
			return e, err
		}
	}
	e.Bytes = pbytes

	psizeRaw := l[13]
	psize := 0
	if psizeRaw != "-" {
		psize, err = strconv.Atoi(l[13])
		if err != nil {
			return e, err
		}
	}
	e.Size = psize

	ptotalRaw := l[14]
	ptotal := 0
	if ptotalRaw != "-" {
		ptotal, err = strconv.Atoi(l[14])
		if err != nil {
			return e, err
		}
	}
	e.Total = ptotal

	pturnaroundRaw := l[15]
	pturnaround := 0
	if pturnaroundRaw != "-" {
		pturnaround, err = strconv.Atoi(l[15])
		if err != nil {
			log.Panic(err)

			return e, err
		}
	}
	e.Turnaround = pturnaround

	pReferrer := l[16]
	pReferrer = strings.Replace(pReferrer, `"`, "", -1)
	if pReferrer == "-" {
		pReferrer = ""
	}
	e.Referrer = pReferrer

	pUserAgent := l[17]
	e.UserAgent = strings.Replace(pUserAgent, `"`, "", -1)

	pVersion := l[18]
	if pVersion == "-" {
		pVersion = ""
	}
	e.Version = pVersion

	return e, nil
}

func parseTime(tl string) (time.Time, error) {
	tl = brackets.ReplaceAllString(tl, "")
	t, err := time.Parse("02/Jan/2006:15:04:05 -0700", tl)
	if err != nil {
		return time.Now(), fmt.Errorf("Error parsing time: %s", err)
	}

	t = t.In(time.UTC)

	return t, nil
}
<commit_msg>godoc formatting<commit_after>// Work with Amazon S3-style logs
package s3log

import (
	"fmt"
	"log"
	"net"
	"regexp"
	"strconv"
	"strings"
	"time"
)

// A Entry is a structured log entry that describes a S3 request
type Entry struct {
	Owner      string
	Bucket     string
	Time       time.Time
	Remote     net.IP
	Requester  string
	RequestID  string
	Operation  string
	Key        string
	RequestURI string
	Status     int
	Error      string
	Bytes      int
	Size       int
	Total      int
	Turnaround int
	Referrer   string
	UserAgent  string
	Version    string
}

var logLine = regexp.MustCompile(`[^" ]+|("[^"]*")`)
var brackets = regexp.MustCompile(`[\[\]]`)

// Parse parses a Amazon S3-style log line into a Entry
func Parse(line string) (Entry, error) {
	e := Entry{}
	l := logLine.FindAllString(line, -1)

	e.Owner = l[0]
	e.Bucket = l[1]
	pt, err := parseTime(l[2] + " " + l[3])
	if err != nil {
		return e, err
	}
	e.Time = pt
	e.Remote = net.ParseIP(l[4])
	e.Requester = l[5]
	e.RequestID = l[6]
	e.Operation = l[7]
	e.Key = l[8]

	prequestURI := l[9]
	e.RequestURI = strings.Replace(prequestURI, `"`, "", -1)

	pstatus, err := strconv.Atoi(l[10])
	if err != nil {
		return e, err
	}
	e.Status = pstatus

	perrorRaw := l[11]
	if perrorRaw == "-" {
		e.Error = ""
	} else {
		e.Error = perrorRaw
	}
	e.Error = perrorRaw

	pbytesRaw := l[12]
	pbytes := 0
	if pbytesRaw != "-" {
		pbytes, err = strconv.Atoi(pbytesRaw)
		if err != nil {
			return e, err
		}
	}
	e.Bytes = pbytes

	psizeRaw := l[13]
	psize := 0
	if psizeRaw != "-" {
		psize, err = strconv.Atoi(l[13])
		if err != nil {
			return e, err
		}
	}
	e.Size = psize

	ptotalRaw := l[14]
	ptotal := 0
	if ptotalRaw != "-" {
		ptotal, err = strconv.Atoi(l[14])
		if err != nil {
			return e, err
		}
	}
	e.Total = ptotal

	pturnaroundRaw := l[15]
	pturnaround := 0
	if pturnaroundRaw != "-" {
		pturnaround, err = strconv.Atoi(l[15])
		if err != nil {
			log.Panic(err)

			return e, err
		}
	}
	e.Turnaround = pturnaround

	pReferrer := l[16]
	pReferrer = strings.Replace(pReferrer, `"`, "", -1)
	if pReferrer == "-" {
		pReferrer = ""
	}
	e.Referrer = pReferrer

	pUserAgent := l[17]
	e.UserAgent = strings.Replace(pUserAgent, `"`, "", -1)

	pVersion := l[18]
	if pVersion == "-" {
		pVersion = ""
	}
	e.Version = pVersion

	return e, nil
}

func parseTime(tl string) (time.Time, error) {
	tl = brackets.ReplaceAllString(tl, "")
	t, err := time.Parse("02/Jan/2006:15:04:05 -0700", tl)
	if err != nil {
		return time.Now(), fmt.Errorf("Error parsing time: %s", err)
	}

	t = t.In(time.UTC)

	return t, nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	log "github.com/Sirupsen/logrus"
	"github.com/pemcconnell/amald/config"
	"github.com/pemcconnell/amald/defs"
	"github.com/pemcconnell/amald/loaders"
	"github.com/pemcconnell/amald/notifiers"
	"github.com/pemcconnell/amald/storage"
	"github.com/pemcconnell/amald/urltest"
)

const (
	VERSION string = "0.1.0"
)

var (
	configPath = flag.String("configPath", "./config.yaml",
		"[config] set the path for the yaml config file. This defaults to "+
			"./config.yaml")
	templatePath = flag.String("templatePath", "reports/tmpl/",
		"[templates directory] set the path for the templates directory")
	logLevel = flag.String("logLevel", "info",
		"[loglevel] set the verbosity of the log levels. Can be: debug, "+
			"info, warn, error, panic, fatal")
)

func init() {
	// parse flagenikto
	flag.Parse()

	// Set logrus level
	if level, err := log.ParseLevel(*logLevel); err == nil {
		log.SetLevel(level)
	}
}

func main() {

	// load the config
	cfg, err := config.Load(*configPath)
	if err != nil {
		log.Fatalf("Failed to load the config from %s", *configPath)
	}
	cfg.Reports["templates"]["path"] = *templatePath

	// collect list of URLs from all possible loaders
	loaders.GetLoaders(cfg.Loaders)
	urls, err := loaders.CollectUrls()
	if err != nil {
		log.Fatalf("Failed to CollectUrls(): %s", err)
	}

	// test all of the urls
	scanResults, err := urltest.Batch(urls)
	if err != nil {
		log.Fatalf("Failed to Batch urltest: %s", err)
	}
	// if there aren't any urls found, don't continue
	if len(scanResults) == 0 {
		log.Fatal("No URLs found in loaders")
	}

	records := defs.SiteDefinitionsToRecords(scanResults)

	// grab a summary (compare current scan against old data)
	if cfg.Tests["storage"] {
		olddata, err := storage.LoadSiteDefsFromStorage(cfg.Storage["json"]["path"])
		if err != nil {
			log.Fatalf("Failed to get summary: %s", err)
		}
		// store latest test
		records = storage.MergeData(scanResults, olddata)
		storage.StoreScan(cfg.Storage["json"]["path"], records)
	}

	// run an analysis on the results, that we can use in reports
	summaries := defs.AnalyseRecords(cfg, records)

	// fire off each notifier
	notifiers.FireNotifiers(cfg, summaries)
}
<commit_msg>Changing version number<commit_after>package main

import (
	"flag"
	log "github.com/Sirupsen/logrus"
	"github.com/pemcconnell/amald/config"
	"github.com/pemcconnell/amald/defs"
	"github.com/pemcconnell/amald/loaders"
	"github.com/pemcconnell/amald/notifiers"
	"github.com/pemcconnell/amald/storage"
	"github.com/pemcconnell/amald/urltest"
)

const (
	VERSION string = "0.0.4"
)

var (
	configPath = flag.String("configPath", "./config.yaml",
		"[config] set the path for the yaml config file. This defaults to "+
			"./config.yaml")
	templatePath = flag.String("templatePath", "reports/tmpl/",
		"[templates directory] set the path for the templates directory")
	logLevel = flag.String("logLevel", "info",
		"[loglevel] set the verbosity of the log levels. Can be: debug, "+
			"info, warn, error, panic, fatal")
)

func init() {
	// parse flagenikto
	flag.Parse()

	// Set logrus level
	if level, err := log.ParseLevel(*logLevel); err == nil {
		log.SetLevel(level)
	}
}

func main() {

	// load the config
	cfg, err := config.Load(*configPath)
	if err != nil {
		log.Fatalf("Failed to load the config from %s", *configPath)
	}
	cfg.Reports["templates"]["path"] = *templatePath

	// collect list of URLs from all possible loaders
	loaders.GetLoaders(cfg.Loaders)
	urls, err := loaders.CollectUrls()
	if err != nil {
		log.Fatalf("Failed to CollectUrls(): %s", err)
	}

	// test all of the urls
	scanResults, err := urltest.Batch(urls)
	if err != nil {
		log.Fatalf("Failed to Batch urltest: %s", err)
	}
	// if there aren't any urls found, don't continue
	if len(scanResults) == 0 {
		log.Fatal("No URLs found in loaders")
	}

	records := defs.SiteDefinitionsToRecords(scanResults)

	// grab a summary (compare current scan against old data)
	if cfg.Tests["storage"] {
		olddata, err := storage.LoadSiteDefsFromStorage(cfg.Storage["json"]["path"])
		if err != nil {
			log.Fatalf("Failed to get summary: %s", err)
		}
		// store latest test
		records = storage.MergeData(scanResults, olddata)
		storage.StoreScan(cfg.Storage["json"]["path"], records)
	}

	// run an analysis on the results, that we can use in reports
	summaries := defs.AnalyseRecords(cfg, records)

	// fire off each notifier
	notifiers.FireNotifiers(cfg, summaries)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/ChimeraCoder/anaconda"
	"github.com/darkhelmet/twitterstream"
	"io/ioutil"
	"log"
	"net/url"
	"strings"
)

func main() {
	fmt.Println("hi")
	b, e := ioutil.ReadFile("./twittercfg")
	if e != nil {
		log.Fatal("Could not read the ./twittercfg file.")
	}
	twittertemp := string(b)
	twitterbits := strings.Split(twittertemp, "\n")
	if len(twitterbits) != 5 {
		log.Fatal("Not enought things in twitter cfg, Needs to be (seperated by \\n) username, consumerKey, consumerSecret, accessToken, accessSecret")
	}
	Client := twitterstream.NewClient(twitterbits[1], twitterbits[2], twitterbits[3], twitterbits[4])
	Conn, e := Client.Track(fmt.Sprintf("@%s", twitterbits[0]))
	// Streamign API is setup now, now just setup the general purpose one now
	anaconda.SetConsumerKey(twitterbits[1])
	anaconda.SetConsumerSecret(twitterbits[2])
	api := anaconda.NewTwitterApi(twitterbits[3], twitterbits[4])

	if e != nil {
		log.Fatal("could not open a streaming connection to get mentions :(")
	}
	for {
		t, e := Conn.Next()
		if e == nil {
			fmt.Printf("TWEET: %s\n", t.Text)
			fmt.Printf("OWNER @%s\n", strings.ToLower(twitterbits[0]))
			if strings.HasPrefix(strings.ToLower(t.Text), fmt.Sprintf("@%s", strings.ToLower(twitterbits[0]))) {
				v := url.Values{} // I dont even know
				t, e := api.PostTweet(fmt.Sprintf("@%s pong", t.User.ScreenName), v)
				if e == nil {
					fmt.Println(t)
				} else {
					fmt.Println(e)
				}
			} else {
				fmt.Println("Does not start with @<user> ignoring")
			}
		}
	}

}
<commit_msg>Add version number that I know everything works at.<commit_after>package main

import (
	"fmt"
	"github.com/ChimeraCoder/anaconda"    // Working at 2002271f2160a4d243f0308af0827893e2868157
	"github.com/darkhelmet/twitterstream" // Working at 4051c41877496d38d54647c35897e768fd34385f
	"io/ioutil"
	"log"
	"net/url"
	"strings"
)

func main() {
	fmt.Println("hi")
	b, e := ioutil.ReadFile("./twittercfg")
	if e != nil {
		log.Fatal("Could not read the ./twittercfg file.")
	}
	twittertemp := string(b)
	twitterbits := strings.Split(twittertemp, "\n")
	if len(twitterbits) != 5 {
		log.Fatal("Not enought things in twitter cfg, Needs to be (seperated by \\n) username, consumerKey, consumerSecret, accessToken, accessSecret")
	}
	Client := twitterstream.NewClient(twitterbits[1], twitterbits[2], twitterbits[3], twitterbits[4])
	Conn, e := Client.Track(fmt.Sprintf("@%s", twitterbits[0]))
	// Streamign API is setup now, now just setup the general purpose one now
	anaconda.SetConsumerKey(twitterbits[1])
	anaconda.SetConsumerSecret(twitterbits[2])
	api := anaconda.NewTwitterApi(twitterbits[3], twitterbits[4])

	if e != nil {
		log.Fatal("could not open a streaming connection to get mentions :(")
	}
	for {
		t, e := Conn.Next()
		if e == nil {
			fmt.Printf("TWEET: %s\n", t.Text)
			fmt.Printf("OWNER @%s\n", strings.ToLower(twitterbits[0]))
			if strings.HasPrefix(strings.ToLower(t.Text), fmt.Sprintf("@%s", strings.ToLower(twitterbits[0]))) {
				v := url.Values{} // I dont even know
				t, e := api.PostTweet(fmt.Sprintf("@%s pong", t.User.ScreenName), v)
				if e == nil {
					fmt.Println(t)
				} else {
					fmt.Println(e)
				}
			} else {
				fmt.Println("Does not start with @<user> ignoring")
			}
		}
	}

}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"github.com/gruntwork-io/terragrunt/util"
	"github.com/gruntwork-io/terragrunt/cli"
	"github.com/gruntwork-io/terragrunt/errors"
)

// This variable is set at build time using -ldflags parameters. For more info, see:
// http://stackoverflow.com/a/11355611/483528
var VERSION string

// The main entrypoint for Terragrunt
func main() {
	defer errors.Recover(checkForErrorsAndExit)

	app := cli.CreateTerragruntCli(VERSION)
	err := app.Run(os.Args)

	checkForErrorsAndExit(err)
}

// If there is an error, display it in the console and exit with a non-zero exit code. Otherwise, exit 0.
func checkForErrorsAndExit(err error) {
	if err == nil {
		os.Exit(0)
	} else {
		logger := util.CreateLogger("")
		if os.Getenv("TERRAGRUNT_DEBUG") != "" {
			logger.Println(errors.PrintErrorWithStackTrace(err))
		} else {
			logger.Println(err)
		}
		os.Exit(1)
	}

}<commit_msg>fixes gruntwork-io/terragrunt#37 exit with underlying exit code<commit_after>package main

import (
	"os"
	"os/exec"
	"syscall"
	"github.com/gruntwork-io/terragrunt/util"
	"github.com/gruntwork-io/terragrunt/cli"
	"github.com/gruntwork-io/terragrunt/errors"
)

// This variable is set at build time using -ldflags parameters. For more info, see:
// http://stackoverflow.com/a/11355611/483528
var VERSION string

// The main entrypoint for Terragrunt
func main() {
	defer errors.Recover(checkForErrorsAndExit)

	app := cli.CreateTerragruntCli(VERSION)
	err := app.Run(os.Args)

	checkForErrorsAndExit(err)
}

// If there is an error, display it in the console and exit with a non-zero exit code. Otherwise, exit 0.
func checkForErrorsAndExit(err error) {
	if err == nil {
		os.Exit(0)
	} else {
		logger := util.CreateLogger("")
		if os.Getenv("TERRAGRUNT_DEBUG") != "" {
			logger.Println(errors.PrintErrorWithStackTrace(err))
		} else {
			logger.Println(err)
		}
		// exit with the underlying error code
		var retCode int
		if exiterr, ok := errors.Unwrap(err).(*exec.ExitError); ok {
			status := exiterr.Sys().(syscall.WaitStatus)
			retCode = status.ExitStatus()
		}
		os.Exit(retCode)
	}

}<|endoftext|>


<commit_before>package main

import (
	"flag"
	"log"
	"time"

	"github.com/digitalocean/godo"
	"golang.org/x/oauth2"
)

var token = flag.String("token", "", "DigitalOcean access token")
var regions = flag.Bool("regions", false, "If true, will display all available regions")
var destroy = flag.Bool("destroy", false, "If true, will remove all droplets with given name")

func initClient() *godo.Client {
	tokenSource := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: *token})
	oauthClient := oauth2.NewClient(oauth2.NoContext, tokenSource)
	return godo.NewClient(oauthClient)
}

func waitForNetwork(client *godo.Client, dropletID int) string {
	for {
		drop, _, err := client.Droplets.Get(dropletID)
		if err != nil {
			log.Fatal(err)
		}
		if drop.Status == "active" {
			if len(drop.Networks.V4) > 0 {
				return drop.Networks.V4[0].IPAddress
			}
		}
		<-time.After(time.Second * 5)
	}
}

func main() {
	flag.Parse()

	if len(*token) == 0 {
		log.Fatal("Token can not be empty!")
		return
	}

	client := initClient()

	if *regions {
		showRegions(client)
		return
	}

	if *destroy {
		deleteDroplets(client)
		return
	}

	log.Println("Loading SSH key...")
	sshKey := loadPrivateKey()
	sshKeyFingerprint := publicFingerprint(sshKey)
	log.Printf("Loaded key: %s\n", sshKeyFingerprint)

	log.Println("Looking for key...")
	key, err := loadPublicKey(client)
	switch {
	case err == errKeyNotFound:
		log.Println("Key not found. Uploading...")
		key, err = uploadPublicKey(client, sshKey.PublicKey())
		if err != nil {
			log.Fatal(err)
		}
		log.Println("Uploaded key.")
	case err != nil:
		log.Fatal(err)
	}
	if key.Fingerprint != sshKeyFingerprint {
		log.Fatal("Key fingerprints do not match!")
	}
	log.Printf("Using key with fingerprint %s", key.Fingerprint)

	log.Println("Creating droplet...")
	drop := createDroplet(client, key)
	log.Printf("Created droplet %s (%d)", drop.Name, drop.ID)

	log.Println("Waiting for droplet to be ready...")
	dropletIP := waitForNetwork(client, drop.ID)
	log.Printf("Droplet is ready: %s", dropletIP)

	log.Println("Waiting for setup script to complete...")
	sshClient := waitForSetup(sshKey, dropletIP)
	defer sshClient.Close()
	log.Println("Setup complete.")

	log.Println("Reading secret...")
	secret := readSecret(sshClient)
	log.Printf("Successfully read secret.")

	log.Println("Writing configuration...")
	file := writeConfigFile(dropletIP, secret)
	log.Printf("Written configuration to %s", file)
}
<commit_msg>Some more output.<commit_after>package main

import (
	"flag"
	"log"
	"time"

	"github.com/digitalocean/godo"
	"golang.org/x/oauth2"
)

var token = flag.String("token", "", "DigitalOcean access token")
var regions = flag.Bool("regions", false, "If true, will display all available regions")
var destroy = flag.Bool("destroy", false, "If true, will remove all droplets with given name")

func initClient() *godo.Client {
	tokenSource := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: *token})
	oauthClient := oauth2.NewClient(oauth2.NoContext, tokenSource)
	return godo.NewClient(oauthClient)
}

func waitForNetwork(client *godo.Client, dropletID int) string {
	for {
		drop, _, err := client.Droplets.Get(dropletID)
		if err != nil {
			log.Fatal(err)
		}
		if drop.Status == "active" {
			if len(drop.Networks.V4) > 0 {
				return drop.Networks.V4[0].IPAddress
			}
		}
		<-time.After(time.Second * 5)
	}
}

func main() {
	flag.Parse()

	if len(*token) == 0 {
		log.Fatal("Token can not be empty!")
		return
	}

	client := initClient()

	if *regions {
		log.Println("Showing regions...")
		showRegions(client)
		return
	}

	if *destroy {
		log.Println("Removing droplets...")
		deleteDroplets(client)
		return
	}

	log.Println("Loading SSH key...")
	sshKey := loadPrivateKey()
	sshKeyFingerprint := publicFingerprint(sshKey)
	log.Printf("Loaded key: %s\n", sshKeyFingerprint)

	log.Println("Looking for key...")
	key, err := loadPublicKey(client)
	switch {
	case err == errKeyNotFound:
		log.Println("Key not found. Uploading...")
		key, err = uploadPublicKey(client, sshKey.PublicKey())
		if err != nil {
			log.Fatal(err)
		}
		log.Println("Uploaded key.")
	case err != nil:
		log.Fatal(err)
	}
	if key.Fingerprint != sshKeyFingerprint {
		log.Fatal("Key fingerprints do not match!")
	}
	log.Printf("Using key with fingerprint %s", key.Fingerprint)

	log.Println("Creating droplet...")
	drop := createDroplet(client, key)
	log.Printf("Created droplet %s (%d)", drop.Name, drop.ID)

	log.Println("Waiting for droplet to be ready...")
	dropletIP := waitForNetwork(client, drop.ID)
	log.Printf("Droplet is ready: %s", dropletIP)

	log.Println("Waiting for setup script to complete...")
	sshClient := waitForSetup(sshKey, dropletIP)
	defer sshClient.Close()
	log.Println("Setup complete.")

	log.Println("Reading secret...")
	secret := readSecret(sshClient)
	log.Printf("Successfully read secret.")

	log.Println("Writing configuration...")
	file := writeConfigFile(dropletIP, secret)
	log.Printf("Written configuration to %s", file)
}
<|endoftext|>


<commit_before>/* Copyright 2015 LinkedIn Corp. Licensed under the Apache License, Version
 * 2.0 (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 */

package main

import (
	"flag"
	"fmt"
	log "github.com/cihub/seelog"
	"github.com/samuel/go-zookeeper/zk"
	"os"
	"os/signal"
	"syscall"
	"time"
)

type KafkaCluster struct {
	Client    *KafkaClient
	Zookeeper *ZookeeperClient
}

type ApplicationContext struct {
	Config       *BurrowConfig
	Storage      *OffsetStorage
	Clusters     map[string]*KafkaCluster
	Server       *HttpServer
	Emailer      *Emailer
	HttpNotifier *HttpNotifier
	NotifierLock *zk.Lock
}

func loadNotifiers(app *ApplicationContext) error {
	// Set up the Emailer, if configured
	if len(app.Config.Email) > 0 {
		log.Info("Configuring Email notifier")
		emailer, err := NewEmailer(app)
		if err != nil {
			log.Criticalf("Cannot configure email notifier: %v", err)
			return err
		}
		app.Emailer = emailer
	}

	// Set up the HTTP Notifier, if configured
	if app.Config.Httpnotifier.Url != "" {
		log.Info("Configuring HTTP notifier")
		httpnotifier, err := NewHttpNotifier(app)
		if err != nil {
			log.Criticalf("Cannot configure HTTP notifier: %v", err)
			return err
		}
		app.HttpNotifier = httpnotifier
	}

	return nil
}

func startNotifiers(app *ApplicationContext) {
	// Do not proceed until we get the Zookeeper lock
	err := app.NotifierLock.Lock()
	if err != nil {
		log.Criticalf("Cannot get ZK notifier lock: %v", err)
		os.Exit(1)
	}
	log.Info("Acquired Zookeeper notifier lock")

	if app.Emailer != nil {
		log.Info("Starting Email notifier")
		app.Emailer.Start()
	}
	if app.HttpNotifier != nil {
		log.Info("Starting HTTP notifier")
		app.HttpNotifier.Start()
	}
}

func stopNotifiers(app *ApplicationContext) {
	// Ignore errors on unlock - we're quitting anyways, and it might not be locked
	app.NotifierLock.Unlock()

	if app.Emailer != nil {
		log.Info("Stopping Email notifier")
		app.Emailer.Stop()
	}
	if app.HttpNotifier != nil {
		log.Info("Stopping HTTP notifier")
		app.HttpNotifier.Stop()
	}
}

// Why two mains? Golang doesn't let main() return, which means defers will not run.
// So we do everything in a separate main, that way we can easily exit out with an error code and still run defers
func burrowMain() int {
	// The only command line arg is the config file
	var cfgfile = flag.String("config", "burrow.cfg", "Full path to the configuration file")
	flag.Parse()

	// Load and validate the configuration
	fmt.Fprintln(os.Stderr, "Reading configuration from", *cfgfile)
	appContext := &ApplicationContext{Config: ReadConfig(*cfgfile)}
	if err := ValidateConfig(appContext); err != nil {
		log.Criticalf("Cannot validate configuration: %v", err)
		return 1
	}

	// Create the PID file to lock out other processes. Defer removal so it's the last thing to go
	createPidFile(appContext.Config.General.LogDir + "/" + appContext.Config.General.PIDFile)
	defer removePidFile(appContext.Config.General.LogDir + "/" + appContext.Config.General.PIDFile)

	// Set up stderr/stdout to go to a separate log file
	openOutLog(appContext.Config.General.LogDir + "/burrow.out")
	fmt.Println("Started Burrow at", time.Now().Format("January 2, 2006 at 3:04pm (MST)"))

	// If a logging config is specified, replace the existing loggers
	if appContext.Config.General.LogConfig != "" {
		NewLogger(appContext.Config.General.LogConfig)
	}

	// Start a local Zookeeper client (used for application locks)
	log.Info("Starting Zookeeper client")
	zkhosts := make([]string, len(appContext.Config.Zookeeper.Hosts))
	for i, host := range appContext.Config.Zookeeper.Hosts {
		zkhosts[i] = fmt.Sprintf("%s:%v", host, appContext.Config.Zookeeper.Port)
	}
	zkconn, _, err := zk.Connect(zkhosts, time.Duration(appContext.Config.Zookeeper.Timeout)*time.Second)
	if err != nil {
		log.Criticalf("Cannot start Zookeeper client: %v", err)
		return 1
	}
	defer zkconn.Close()

	// Start an offsets storage module
	log.Info("Starting Offsets Storage module")
	appContext.Storage, err = NewOffsetStorage(appContext)
	if err != nil {
		log.Criticalf("Cannot configure offsets storage module: %v", err)
		return 1
	}
	defer appContext.Storage.Stop()

	// Start an HTTP server
	log.Info("Starting HTTP server")
	appContext.Server, err = NewHttpServer(appContext)
	if err != nil {
		log.Criticalf("Cannot start HTTP server: %v", err)
		return 1
	}
	defer appContext.Server.Stop()

	// Start Kafka clients and Zookeepers for each cluster
	appContext.Clusters = make(map[string]*KafkaCluster, len(appContext.Config.Kafka))
	for cluster, _ := range appContext.Config.Kafka {
		log.Infof("Starting Zookeeper client for cluster %s", cluster)
		zkconn, err := NewZookeeperClient(appContext, cluster)
		if err != nil {
			log.Criticalf("Cannot start Zookeeper client for cluster %s: %v", cluster, err)
			return 1
		}
		defer zkconn.Stop()

		log.Infof("Starting Kafka client for cluster %s", cluster)
		client, err := NewKafkaClient(appContext, cluster)
		if err != nil {
			log.Criticalf("Cannot start Kafka client for cluster %s: %v", cluster, err)
			return 1
		}
		defer client.Stop()

		appContext.Clusters[cluster] = &KafkaCluster{Client: client, Zookeeper: zkconn}
	}

	// Set up the Zookeeper lock for notification
	appContext.NotifierLock = zk.NewLock(zkconn, appContext.Config.Zookeeper.LockPath, make([]zk.ACL, 0))

	// Load the notifiers, but do not start them
	err = loadNotifiers(appContext)
	if err != nil {
		// Error was already logged
		return 1
	}

	// Notifiers are started in a goroutine if we get the ZK lock
	go startNotifiers(appContext)
	defer stopNotifiers(appContext)

	// Register signal handlers for exiting
	exitChannel := make(chan os.Signal, 1)
	signal.Notify(exitChannel, syscall.SIGINT, syscall.SIGQUIT, syscall.SIGSTOP, syscall.SIGTERM)

	// Wait until we're told to exit
	<-exitChannel
	log.Info("Shutdown triggered")
	return 0
}

func main() {
	rv := burrowMain()
	if rv != 0 {
		fmt.Println("Burrow failed at", time.Now().Format("January 2, 2006 at 3:04pm (MST)"))
	} else {
		fmt.Println("Stopped Burrow at", time.Now().Format("January 2, 2006 at 3:04pm (MST)"))
	}
	os.Exit(rv)
}
<commit_msg>Fix to blank ACL definition for issue #3<commit_after>/* Copyright 2015 LinkedIn Corp. Licensed under the Apache License, Version
 * 2.0 (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 */

package main

import (
	"flag"
	"fmt"
	log "github.com/cihub/seelog"
	"github.com/samuel/go-zookeeper/zk"
	"os"
	"os/signal"
	"syscall"
	"time"
)

type KafkaCluster struct {
	Client    *KafkaClient
	Zookeeper *ZookeeperClient
}

type ApplicationContext struct {
	Config       *BurrowConfig
	Storage      *OffsetStorage
	Clusters     map[string]*KafkaCluster
	Server       *HttpServer
	Emailer      *Emailer
	HttpNotifier *HttpNotifier
	NotifierLock *zk.Lock
}

func loadNotifiers(app *ApplicationContext) error {
	// Set up the Emailer, if configured
	if len(app.Config.Email) > 0 {
		log.Info("Configuring Email notifier")
		emailer, err := NewEmailer(app)
		if err != nil {
			log.Criticalf("Cannot configure email notifier: %v", err)
			return err
		}
		app.Emailer = emailer
	}

	// Set up the HTTP Notifier, if configured
	if app.Config.Httpnotifier.Url != "" {
		log.Info("Configuring HTTP notifier")
		httpnotifier, err := NewHttpNotifier(app)
		if err != nil {
			log.Criticalf("Cannot configure HTTP notifier: %v", err)
			return err
		}
		app.HttpNotifier = httpnotifier
	}

	return nil
}

func startNotifiers(app *ApplicationContext) {
	// Do not proceed until we get the Zookeeper lock
	err := app.NotifierLock.Lock()
	if err != nil {
		log.Criticalf("Cannot get ZK notifier lock: %v", err)
		os.Exit(1)
	}
	log.Info("Acquired Zookeeper notifier lock")

	if app.Emailer != nil {
		log.Info("Starting Email notifier")
		app.Emailer.Start()
	}
	if app.HttpNotifier != nil {
		log.Info("Starting HTTP notifier")
		app.HttpNotifier.Start()
	}
}

func stopNotifiers(app *ApplicationContext) {
	// Ignore errors on unlock - we're quitting anyways, and it might not be locked
	app.NotifierLock.Unlock()

	if app.Emailer != nil {
		log.Info("Stopping Email notifier")
		app.Emailer.Stop()
	}
	if app.HttpNotifier != nil {
		log.Info("Stopping HTTP notifier")
		app.HttpNotifier.Stop()
	}
}

// Why two mains? Golang doesn't let main() return, which means defers will not run.
// So we do everything in a separate main, that way we can easily exit out with an error code and still run defers
func burrowMain() int {
	// The only command line arg is the config file
	var cfgfile = flag.String("config", "burrow.cfg", "Full path to the configuration file")
	flag.Parse()

	// Load and validate the configuration
	fmt.Fprintln(os.Stderr, "Reading configuration from", *cfgfile)
	appContext := &ApplicationContext{Config: ReadConfig(*cfgfile)}
	if err := ValidateConfig(appContext); err != nil {
		log.Criticalf("Cannot validate configuration: %v", err)
		return 1
	}

	// Create the PID file to lock out other processes. Defer removal so it's the last thing to go
	createPidFile(appContext.Config.General.LogDir + "/" + appContext.Config.General.PIDFile)
	defer removePidFile(appContext.Config.General.LogDir + "/" + appContext.Config.General.PIDFile)

	// Set up stderr/stdout to go to a separate log file
	openOutLog(appContext.Config.General.LogDir + "/burrow.out")
	fmt.Println("Started Burrow at", time.Now().Format("January 2, 2006 at 3:04pm (MST)"))

	// If a logging config is specified, replace the existing loggers
	if appContext.Config.General.LogConfig != "" {
		NewLogger(appContext.Config.General.LogConfig)
	}

	// Start a local Zookeeper client (used for application locks)
	log.Info("Starting Zookeeper client")
	zkhosts := make([]string, len(appContext.Config.Zookeeper.Hosts))
	for i, host := range appContext.Config.Zookeeper.Hosts {
		zkhosts[i] = fmt.Sprintf("%s:%v", host, appContext.Config.Zookeeper.Port)
	}
	zkconn, _, err := zk.Connect(zkhosts, time.Duration(appContext.Config.Zookeeper.Timeout)*time.Second)
	if err != nil {
		log.Criticalf("Cannot start Zookeeper client: %v", err)
		return 1
	}
	defer zkconn.Close()

	// Start an offsets storage module
	log.Info("Starting Offsets Storage module")
	appContext.Storage, err = NewOffsetStorage(appContext)
	if err != nil {
		log.Criticalf("Cannot configure offsets storage module: %v", err)
		return 1
	}
	defer appContext.Storage.Stop()

	// Start an HTTP server
	log.Info("Starting HTTP server")
	appContext.Server, err = NewHttpServer(appContext)
	if err != nil {
		log.Criticalf("Cannot start HTTP server: %v", err)
		return 1
	}
	defer appContext.Server.Stop()

	// Start Kafka clients and Zookeepers for each cluster
	appContext.Clusters = make(map[string]*KafkaCluster, len(appContext.Config.Kafka))
	for cluster, _ := range appContext.Config.Kafka {
		log.Infof("Starting Zookeeper client for cluster %s", cluster)
		zkconn, err := NewZookeeperClient(appContext, cluster)
		if err != nil {
			log.Criticalf("Cannot start Zookeeper client for cluster %s: %v", cluster, err)
			return 1
		}
		defer zkconn.Stop()

		log.Infof("Starting Kafka client for cluster %s", cluster)
		client, err := NewKafkaClient(appContext, cluster)
		if err != nil {
			log.Criticalf("Cannot start Kafka client for cluster %s: %v", cluster, err)
			return 1
		}
		defer client.Stop()

		appContext.Clusters[cluster] = &KafkaCluster{Client: client, Zookeeper: zkconn}
	}

	// Set up the Zookeeper lock for notification
	appContext.NotifierLock = zk.NewLock(zkconn, appContext.Config.Zookeeper.LockPath, zk.WorldACL(zk.PermAll))

	// Load the notifiers, but do not start them
	err = loadNotifiers(appContext)
	if err != nil {
		// Error was already logged
		return 1
	}

	// Notifiers are started in a goroutine if we get the ZK lock
	go startNotifiers(appContext)
	defer stopNotifiers(appContext)

	// Register signal handlers for exiting
	exitChannel := make(chan os.Signal, 1)
	signal.Notify(exitChannel, syscall.SIGINT, syscall.SIGQUIT, syscall.SIGSTOP, syscall.SIGTERM)

	// Wait until we're told to exit
	<-exitChannel
	log.Info("Shutdown triggered")
	return 0
}

func main() {
	rv := burrowMain()
	if rv != 0 {
		fmt.Println("Burrow failed at", time.Now().Format("January 2, 2006 at 3:04pm (MST)"))
	} else {
		fmt.Println("Stopped Burrow at", time.Now().Format("January 2, 2006 at 3:04pm (MST)"))
	}
	os.Exit(rv)
}
<|endoftext|>


<commit_before>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"flag"
	"log"
	"os"
)

// The set of commands supported by the tool.
var commands = []*Command{
	cmdList,
	cmdRestore,
	cmdSave,
	cmdVerify,
}

func main() {
	flag.Parse()

	// Set up bare logging output.
	log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds)

	// We get the command name.
	args := flag.Args()
	if len(args) < 1 {
		log.Println("Missing command name. Choices are:")
		for _, cmd := range commands {
			log.Printf("  %s\n", cmd.Name)
		}

		os.Exit(1)
	}

	cmdName := args[0]

	// Find and run the appropriate command.
	for _, cmd := range commands {
		if cmd.Name == cmdName {
			cmd.Flags.Parse(args[1:])
			args = cmd.Flags.Args()
			cmd.Run(args)
			return
		}
	}

	log.Fatalln("Unknown command:", cmdName)
}
<commit_msg>Register command gc.<commit_after>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"flag"
	"log"
	"os"
)

// The set of commands supported by the tool.
var commands = []*Command{
	cmdGC,
	cmdList,
	cmdRestore,
	cmdSave,
	cmdVerify,
}

func main() {
	flag.Parse()

	// Set up bare logging output.
	log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds)

	// We get the command name.
	args := flag.Args()
	if len(args) < 1 {
		log.Println("Missing command name. Choices are:")
		for _, cmd := range commands {
			log.Printf("  %s\n", cmd.Name)
		}

		os.Exit(1)
	}

	cmdName := args[0]

	// Find and run the appropriate command.
	for _, cmd := range commands {
		if cmd.Name == cmdName {
			cmd.Flags.Parse(args[1:])
			args = cmd.Flags.Args()
			cmd.Run(args)
			return
		}
	}

	log.Fatalln("Unknown command:", cmdName)
}
<|endoftext|>