<commit_before>package main

import (
	"expvar"
	"flag"
	"fmt"
	"log"
	"os"
	"os/signal"
	"regexp"
	"strings"
	"syscall"
	"time"

	"github.com/garyburd/redigo/redis"
	"github.com/marpaia/graphite-golang"
)

// https://godoc.org/github.com/garyburd/redigo/redis#Pool
func newPool(port string) *redis.Pool {
	return &redis.Pool{
		MaxIdle:     4,
		IdleTimeout: 60 * time.Second,
		Dial: func() (redis.Conn, error) {
			c, err := redis.Dial("tcp", ":"+port)
			if err != nil {
				return nil, err
			}
			return c, err
		},
		TestOnBorrow: func(c redis.Conn, t time.Time) error {
			_, err := c.Do("PING")
			return err
		},
	}
}

// fqdn with underscores
func HostnameGraphite() string {
	hostname, _ := os.Hostname()
	return strings.Replace(hostname, ".", "_", -1)
}

func keyspaceEnable(pool *redis.Pool) {
	c := pool.Get()
	defer c.Close()

	// check if notify-keyspace-events are enabled
	notify, err := redis.StringMap(c.Do("CONFIG", "GET", "notify-keyspace-events"))
	if err != nil {
		log.Println(err)
		return
	}
	for _, v := range notify {
		config := keyspaceConfigRegex.FindString(v)
		if config != "" {
			// already enabled, we can listen for the LIST events
			log.Println("LIST events notifications already enabled")
		} else {
			// enable LIST events without replacing the existing config (if any)
			log.Println("Enabling LIST events notifications")
			if v == "" {
				_, err := redis.String(c.Do("CONFIG", "SET", "notify-keyspace-events", "lK"))
				if err != nil {
					log.Println(err)
					return
				}
			} else {
				// do not override the existing config
				_, err := redis.String(c.Do("CONFIG", "SET", "notify-keyspace-events", v+"lK"))
				if err != nil {
					log.Println(err)
					return
				}
			}
		}
	}
}

func instanceAlive(pool *redis.Pool) bool {
	c := pool.Get()
	defer c.Close()
	_, err := c.Do("PING")
	if err != nil {
		return false
	}
	return true
}

func instanceIsMaster(pool *redis.Pool, port string) bool {
	c := pool.Get()
	defer c.Close()

	master, err := redis.StringMap(c.Do("CONFIG", "GET", "slaveof"))
	if err != nil {
		log.Println(err)
		return false
	}
	for _, value := range master {
		if value == "" {
			log.Printf("instance on port %s is a master\n", port)
			return true
		} else {
			log.Printf("instance on port %s is a slave of %s\n", port, value)
		}
	}
	return false
}

func queueStats(port string) {
	// connect to redis
	pool := newPool(port)
	c := pool.Get()
	if !instanceAlive(pool) {
		log.Printf("error: no redis instance listening on port %s, aborting\n", port)
		return
	}

	go keyspaceEnable(pool)

	// subscribe to the keyspace notifications
	c.Send("PSUBSCRIBE", "__keyspace*")
	c.Flush()
	// ignore first message received when subscribing
	c.Receive()

	// wait for published notifications
	for {
		reply, err := redis.StringMap(c.Receive())
		if err != nil {
			log.Println(err)

			// Retry connection to Redis until it is back
			defer c.Close()
			log.Printf("connection to redis lost. retry in 1s\n")
			time.Sleep(time.Second * 1)
			c = pool.Get()
			go keyspaceEnable(pool)
			c.Send("PSUBSCRIBE", "*")
			c.Flush()
			c.Receive()
			continue
		}
		// match for a LIST keyspace event
		for k, v := range reply {
			operation := listOperationsRegex.FindString(v)
			queue := keyspaceRegex.FindStringSubmatch(k)
			if len(queue) == 2 && operation != "" {
				//log.Printf("%s on %s queue\n", operation, queue[1])
				Stats.Add(fmt.Sprintf("%s.%s.%s", port, queue[1], operation), 1)
			}
		}
	}
}

var Stats = expvar.NewMap("stats").Init()
var listOperationsRegex = regexp.MustCompile("^(lpush|lpushx|rpush|rpushx|lpop|blpop|rpop|brpop)$")
var keyspaceRegex = regexp.MustCompile("^__keyspace.*__:(?P<queue_name>.*)$")
var keyspaceConfigRegex = regexp.MustCompile("^(AK.*|.*l.*K.*)$")
var ports redisPorts
var graph *graphite.Graphite

func main() {
	flag.Var(&ports, "ports", "comma-separated list of redis ports")
	graphiteHost := flag.String("graphite-host", "localhost", "graphite hostname")
	graphitePort := flag.Int("graphite-port", 2003, "graphite port")
	interval := flag.Int("interval", 60, "interval for sending graphite metrics")
	simulate := flag.Bool("simulate", false, "simulate sending to graphite via stdout")
	flag.Parse()

	// flag checks
	if len(ports) == 0 {
		log.Println("no redis instances defined, aborting")
		return
	}

	if *simulate {
		graph = graphite.NewGraphiteNop(*graphiteHost, *graphitePort)
	} else {
		var err error
		graph, err = graphite.NewGraphite(*graphiteHost, *graphitePort)
		if err != nil {
			log.Println(err)
			return
		}
	}
	hostname := HostnameGraphite()
	ticker := time.NewTicker(time.Second * time.Duration(*interval)).C

	for _, port := range ports {
		log.Println("spawning collector for port ", port)
		go queueStats(port)
	}

	sig := make(chan os.Signal, 1)
	done := make(chan bool, 1)
	signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM)

	go func() {
		<-sig
		done <- true
	}()

	for {
		select {
		case <-ticker:
			Stats.Do(func(kv expvar.KeyValue) {
				graph.SimpleSend(fmt.Sprintf("scouter.%s.%s", hostname, kv.Key), kv.Value.String())
			})
		case <-done:
			log.Println("user aborted")
			return
		}
	}
}
<commit_msg>new goroutine for reporting if an instance is a master/slave<commit_after>package main

import (
	"expvar"
	"flag"
	"fmt"
	"log"
	"os"
	"os/signal"
	"regexp"
	"strings"
	"syscall"
	"time"

	"github.com/garyburd/redigo/redis"
	"github.com/marpaia/graphite-golang"
)

// https://godoc.org/github.com/garyburd/redigo/redis#Pool
func newPool(port string) *redis.Pool {
	return &redis.Pool{
		MaxIdle:     4,
		IdleTimeout: 60 * time.Second,
		Dial: func() (redis.Conn, error) {
			c, err := redis.Dial("tcp", ":"+port)
			if err != nil {
				return nil, err
			}
			return c, err
		},
		TestOnBorrow: func(c redis.Conn, t time.Time) error {
			_, err := c.Do("PING")
			return err
		},
	}
}

// fqdn with underscores
func HostnameGraphite() string {
	hostname, _ := os.Hostname()
	return strings.Replace(hostname, ".", "_", -1)
}

func keyspaceEnable(pool *redis.Pool) {
	c := pool.Get()
	defer c.Close()

	// check if notify-keyspace-events are enabled
	notify, err := redis.StringMap(c.Do("CONFIG", "GET", "notify-keyspace-events"))
	if err != nil {
		log.Println(err)
		return
	}
	for _, v := range notify {
		config := keyspaceConfigRegex.FindString(v)
		if config != "" {
			// already enabled, we can listen for the LIST events
			log.Println("LIST events notifications already enabled")
		} else {
			// enable LIST events without replacing the existing config (if any)
			log.Println("Enabling LIST events notifications")
			if v == "" {
				_, err := redis.String(c.Do("CONFIG", "SET", "notify-keyspace-events", "lK"))
				if err != nil {
					log.Println(err)
					return
				}
			} else {
				// do not override the existing config
				_, err := redis.String(c.Do("CONFIG", "SET", "notify-keyspace-events", v+"lK"))
				if err != nil {
					log.Println(err)
					return
				}
			}
		}
	}
}

func instanceAlive(pool *redis.Pool) bool {
	c := pool.Get()
	defer c.Close()
	_, err := c.Do("PING")
	if err != nil {
		return false
	}
	return true
}

func instanceIsMaster(pool *redis.Pool, port string) {
	c := pool.Get()
	//defer c.Close()

	for {
		master, err := redis.StringMap(c.Do("CONFIG", "GET", "slaveof"))
		if err != nil {
			log.Println(err)
			// Retry connection to Redis until it is back
			defer c.Close()
			time.Sleep(time.Second * 1)
			c = pool.Get()
			continue
		}
		for _, value := range master {
			if value == "" {
				log.Printf("instance on port %s is a master\n", port)
			} else {
				log.Printf("instance on port %s is a slave of %s\n", port, value)
			}
		}
		time.Sleep(time.Second * 5)
	}
}

func queueStats(port string) {
	// connect to redis
	pool := newPool(port)
	c := pool.Get()
	if !instanceAlive(pool) {
		log.Printf("error: no redis instance listening on port %s, aborting\n", port)
		return
	}

	go keyspaceEnable(pool)

	// subscribe to the keyspace notifications
	c.Send("PSUBSCRIBE", "__keyspace*")
	c.Flush()
	// ignore first message received when subscribing
	c.Receive()

	// wait for published notifications
	for {
		reply, err := redis.StringMap(c.Receive())
		if err != nil {
			log.Println(err)

			// Retry connection to Redis until it is back
			defer c.Close()
			log.Printf("connection to redis lost. retry in 1s\n")
			time.Sleep(time.Second * 1)
			c = pool.Get()
			go keyspaceEnable(pool)
			c.Send("PSUBSCRIBE", "*")
			c.Flush()
			c.Receive()
			continue
		}
		// match for a LIST keyspace event
		for k, v := range reply {
			operation := listOperationsRegex.FindString(v)
			queue := keyspaceRegex.FindStringSubmatch(k)
			if len(queue) == 2 && operation != "" {
				//log.Printf("%s on %s queue\n", operation, queue[1])
				Stats.Add(fmt.Sprintf("%s.%s.%s", port, queue[1], operation), 1)
			}
		}
	}
}

var Stats = expvar.NewMap("stats").Init()
var listOperationsRegex = regexp.MustCompile("^(lpush|lpushx|rpush|rpushx|lpop|blpop|rpop|brpop)$")
var keyspaceRegex = regexp.MustCompile("^__keyspace.*__:(?P<queue_name>.*)$")
var keyspaceConfigRegex = regexp.MustCompile("^(AK.*|.*l.*K.*)$")
var ports redisPorts
var graph *graphite.Graphite

func main() {
	flag.Var(&ports, "ports", "comma-separated list of redis ports")
	graphiteHost := flag.String("graphite-host", "localhost", "graphite hostname")
	graphitePort := flag.Int("graphite-port", 2003, "graphite port")
	interval := flag.Int("interval", 60, "interval for sending graphite metrics")
	simulate := flag.Bool("simulate", false, "simulate sending to graphite via stdout")
	flag.Parse()

	// flag checks
	if len(ports) == 0 {
		log.Println("no redis instances defined, aborting")
		return
	}

	if *simulate {
		graph = graphite.NewGraphiteNop(*graphiteHost, *graphitePort)
	} else {
		var err error
		graph, err = graphite.NewGraphite(*graphiteHost, *graphitePort)
		if err != nil {
			log.Println(err)
			return
		}
	}
	hostname := HostnameGraphite()
	ticker := time.NewTicker(time.Second * time.Duration(*interval)).C

	for _, port := range ports {
		log.Println("spawning collector for port ", port)
		go queueStats(port)
	}

	sig := make(chan os.Signal, 1)
	done := make(chan bool, 1)
	signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM)

	go func() {
		<-sig
		done <- true
	}()

	for {
		select {
		case <-ticker:
			Stats.Do(func(kv expvar.KeyValue) {
				graph.SimpleSend(fmt.Sprintf("scouter.%s.%s", hostname, kv.Key), kv.Value.String())
			})
		case <-done:
			log.Println("user aborted")
			return
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"crypto/tls"
	"flag"
	"html/template"
	"log"
	"net/http"
	"os"
	"os/signal"
	"path/filepath"
	"reflect"
	"time"

	"golang.org/x/crypto/acme/autocert"

	"github.com/gorilla/mux"
)

var (
	// netParams is the Bitcoin network that the faucet is operating on.
	netParams = flag.String("net", "testnet", "bitcoin network to operate on")

	// lndNodes is a list of lnd nodes that the faucet should connect out
	// to.
	//
	// TODO(roasbeef): channels should be balanced in a round-robin manner
	// between all available lnd nodes.
	lndNodes = flag.String("nodes", "localhost:10009", "comma separated "+
		"list of host:port")

	// lndIP is the IP address that should be advertised on the home page.
	// Users can connect out to this address in order to sync up their
	// graph state.
	lndIP = flag.String("lnd_ip", "10.0.0.9", "the public IP address of "+
		"the faucet's node")

	// port is the port that the http server should listen on.
	port = flag.String("port", "8080", "port to list for http")
)

// equal reports whether the first argument is equal to any of the remaining
// arguments. This function is used as a custom function within templates to do
// richer equality tests.
func equal(x, y interface{}) bool {
	if reflect.DeepEqual(x, y) {
		return true
	}

	return false
}

var (
	// templateGlobPattern is the pattern than matches all the HTML
	// templates in the static directory
	templateGlobPattern = filepath.Join(staticDirName, "*.html")

	// customFuncs is a registry of custom functions we use from within the
	// templates.
	customFuncs = template.FuncMap{
		"equal": equal,
	}

	// ctxb is a global context with no timeouts that's used within the
	// gRPC requests to lnd.
	ctxb = context.Background()
)

const (
	staticDirName = "static"
)

func main() {
	flag.Parse()

	// Pre-compile the list of templates so we'll catch any error sin the
	// templates as soon as the binary is run.
	faucetTemplates := template.Must(template.New("faucet").
		Funcs(customFuncs).
		ParseGlob(templateGlobPattern))

	// With the templates
	faucet, err := newLightningFaucet(*lndNodes, faucetTemplates)
	if err != nil {
		log.Fatalf("unable to create faucet: %v", err)
		return
	}

	// Create a new mux in order to route a request based on its path to a
	// dedicated http.Handler.
	r := mux.NewRouter()
	r.HandleFunc("/", faucet.faucetHome).Methods("POST", "GET")
	r.HandleFunc("/channels/active", faucet.activeChannels)
	r.HandleFunc("/channels/pending", faucet.activeChannels)

	// Next create a static file server which will dispatch our static
	// files. We rap the file sever http.Handler is a handler that strips
	// out the absolute file path since it'll dispatch based on solely the
	// file name.
	staticFileServer := http.FileServer(http.Dir(staticDirName))
	staticHandler := http.StripPrefix("/static/", staticFileServer)
	r.PathPrefix("/static/").Handler(staticHandler)

	// With all of our paths registered we'll register our mux as part of
	// the global http handler.
	http.Handle("/", r)

	// Create a directory cache so the certs we get from Let's Encrypt are
	// cached locally. This avoids running into their rate-limiting by
	// requesting too many certs.
	certCache := autocert.DirCache("certs")

	// Create the auto-cert manager which will automatically obtain a
	// certificate provided by Let's Encrypt.
	m := autocert.Manager{
		Prompt:     autocert.AcceptTOS,
		Cache:      certCache,
		HostPolicy: autocert.HostWhitelist("faucet.lightning.community"),
	}

	// Finally, create the http server, passing in our TLS configuration.
	httpServer := &http.Server{
		Handler:      r,
		WriteTimeout: 30 * time.Second,
		ReadTimeout:  30 * time.Second,
		Addr:         ":https",
		TLSConfig:    &tls.Config{GetCertificate: m.GetCertificate},
	}
	if err := httpServer.ListenAndServeTLS("", ""); err != nil {
		log.Fatal(err)
	}

	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	<-c
}
<commit_msg>redirect all http requests to https<commit_after>package main

import (
	"context"
	"crypto/tls"
	"flag"
	"html/template"
	"log"
	"net/http"
	"os"
	"os/signal"
	"path/filepath"
	"reflect"
	"time"

	"golang.org/x/crypto/acme/autocert"

	"github.com/gorilla/mux"
)

var (
	// netParams is the Bitcoin network that the faucet is operating on.
	netParams = flag.String("net", "testnet", "bitcoin network to operate on")

	// lndNodes is a list of lnd nodes that the faucet should connect out
	// to.
	//
	// TODO(roasbeef): channels should be balanced in a round-robin manner
	// between all available lnd nodes.
	lndNodes = flag.String("nodes", "localhost:10009", "comma separated "+
		"list of host:port")

	// lndIP is the IP address that should be advertised on the home page.
	// Users can connect out to this address in order to sync up their
	// graph state.
	lndIP = flag.String("lnd_ip", "10.0.0.9", "the public IP address of "+
		"the faucet's node")

	// port is the port that the http server should listen on.
	port = flag.String("port", "8080", "port to list for http")
)

// equal reports whether the first argument is equal to any of the remaining
// arguments. This function is used as a custom function within templates to do
// richer equality tests.
func equal(x, y interface{}) bool {
	if reflect.DeepEqual(x, y) {
		return true
	}

	return false
}

var (
	// templateGlobPattern is the pattern than matches all the HTML
	// templates in the static directory
	templateGlobPattern = filepath.Join(staticDirName, "*.html")

	// customFuncs is a registry of custom functions we use from within the
	// templates.
	customFuncs = template.FuncMap{
		"equal": equal,
	}

	// ctxb is a global context with no timeouts that's used within the
	// gRPC requests to lnd.
	ctxb = context.Background()
)

const (
	staticDirName = "static"
)

func main() {
	flag.Parse()

	// Pre-compile the list of templates so we'll catch any error sin the
	// templates as soon as the binary is run.
	faucetTemplates := template.Must(template.New("faucet").
		Funcs(customFuncs).
		ParseGlob(templateGlobPattern))

	// With the templates
	faucet, err := newLightningFaucet(*lndNodes, faucetTemplates)
	if err != nil {
		log.Fatalf("unable to create faucet: %v", err)
		return
	}

	// Create a new mux in order to route a request based on its path to a
	// dedicated http.Handler.
	r := mux.NewRouter()
	r.HandleFunc("/", faucet.faucetHome).Methods("POST", "GET")
	r.HandleFunc("/channels/active", faucet.activeChannels)
	r.HandleFunc("/channels/pending", faucet.activeChannels)

	// Next create a static file server which will dispatch our static
	// files. We rap the file sever http.Handler is a handler that strips
	// out the absolute file path since it'll dispatch based on solely the
	// file name.
	staticFileServer := http.FileServer(http.Dir(staticDirName))
	staticHandler := http.StripPrefix("/static/", staticFileServer)
	r.PathPrefix("/static/").Handler(staticHandler)

	// With all of our paths registered we'll register our mux as part of
	// the global http handler.
	http.Handle("/", r)

	// Create a directory cache so the certs we get from Let's Encrypt are
	// cached locally. This avoids running into their rate-limiting by
	// requesting too many certs.
	certCache := autocert.DirCache("certs")

	// Create the auto-cert manager which will automatically obtain a
	// certificate provided by Let's Encrypt.
	m := autocert.Manager{
		Prompt:     autocert.AcceptTOS,
		Cache:      certCache,
		HostPolicy: autocert.HostWhitelist("faucet.lightning.community"),
	}

	// As we'd like all requests to default to https, redirect all regular
	// http requests to the https version of the faucet.
	go http.ListenAndServe(":80", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		targetURL := "https://" + r.Host + r.URL.String()
		if len(r.URL.RawQuery) > 0 {
			targetURL += "?" + r.URL.RawQuery
		}

		http.Redirect(w, r, targetURL, http.StatusPermanentRedirect)
	}))

	// Finally, create the http server, passing in our TLS configuration.
	httpServer := &http.Server{
		Handler:      r,
		WriteTimeout: 30 * time.Second,
		ReadTimeout:  30 * time.Second,
		Addr:         ":https",
		TLSConfig:    &tls.Config{GetCertificate: m.GetCertificate},
	}
	if err := httpServer.ListenAndServeTLS("", ""); err != nil {
		log.Fatal(err)
	}

	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	<-c
}
<|endoftext|>


<commit_before>package main // import "github.com/tutumcloud/weave-daemon"

import (
	"log"
	"os"
	"os/exec"
	"strings"
	"sync"
	"time"

	"github.com/fsouza/go-dockerclient"
	"github.com/tutumcloud/go-tutum/tutum"
	"github.com/tutumcloud/weave-daemon/nodes"
)

const version = "0.15.2"

func stringInSlice(a string, list []string) bool {
	for _, b := range list {
		if b == a {
			return true
		}
	}
	return false
}

func AttachContainer(c *docker.Client, container_id string) error {
	log.Println("[CONTAINER ATTACH]: Inspecting Containers " + container_id)
	inspect, err := c.InspectContainer(container_id)

	if err != nil {
		log.Println("[CONTAINER ATTACH]: Inspecting Containers failed")
		return err
	}

	log.Println("[CONTAINER ATTACH]: Attaching container " + container_id)

	cidr := ""
	env_vars := inspect.Config.Env

	for i := range env_vars {
		if strings.HasPrefix(env_vars[i], "TUTUM_IP_ADDRESS=") {
			cidr = env_vars[i][len("TUTUM_IP_ADDRESS="):]
			break
		}
	}

	if cidr != "" {
		tries := 0
		for {

			cmd := exec.Command("/weave", "--local", "attach", cidr, container_id)

			_, err := cmd.StdoutPipe()
			if err != nil {
				return err
			}

			if err := cmd.Start(); err != nil {
				tries++
				time.Sleep(2 * time.Second)
				log.Println("[CONTAINER ATTACH ERROR]: Start weave cmd failed")
				if tries > 3 {
					return err
				}
			}

			if err := cmd.Wait(); err != nil {
				tries++
				time.Sleep(2 * time.Second)
				log.Println("[CONTAINER ATTACH ERROR]: Wait weave cmd failed")
				if tries > 3 {
					return err
				}
			} else {
				log.Printf("%s: adding to weave with IP %s", container_id, cidr)
				break
			}
		}
	} else {
		log.Printf("%s: cannot find the IP address to add to weave", container_id)
	}
	return nil
}

func ContainerAttachThread(c *docker.Client) error {
	var weaveID = ""
	listener := make(chan *docker.APIEvents)
	containerAttached := make(map[string]string)
	containerList := []string{}

	containers, err := c.ListContainers(docker.ListContainersOptions{All: false, Size: true, Limit: 0, Since: "", Before: ""})
	if err != nil {
		log.Println("[CONTAINER ATTACH THREAD ERROR]: Listing Containers failed")
		return err
	}

	for _, container := range containers {

		runningContainer, err := c.InspectContainer(container.ID)

		if err != nil {
			return err
		}

		log.Println("[CONTAINER ATTACH THREAD]: Found running container with ID: " + container.ID)

		if strings.HasPrefix(container.Image, "weaveworks/weave:") {
			weaveID = container.ID
		}

		err = AttachContainer(c, container.ID)
		if err != nil {
			log.Println("[CONTAINER ATTACH THREAD ERROR]: Attaching Containers failed")
			return err
		}
		containerAttached[container.ID] = runningContainer.State.StartedAt.Format(time.RFC3339)
	}

	err = c.AddEventListener(listener)
	if err != nil {
		log.Println("[CONTAINER ATTACH THREAD ERROR]: Listening Containers Events failed")
		return err
	}

	defer func() error {

		err = c.RemoveEventListener(listener)
		if err != nil {
			return err
		}
		return nil
	}()

	if weaveID == "" {
		os.Exit(1)
	}

	log.Println("WEAVE ID is : " + weaveID)

	for {
		timeout := time.Tick(2 * time.Minute)
		select {
		case msg := <-listener:
			if msg.Status == "die" && strings.HasPrefix(msg.From, "weaveworks/weave:") {
				os.Exit(1)
			}
			if msg.Status == "start" {
				startingContainer, err := c.InspectContainer(msg.ID)

				if err != nil {
					return err
				}

				if val, ok := containerAttached[msg.ID]; ok && val == startingContainer.State.StartedAt.Format(time.RFC3339) {
					break
				} else {
					err := AttachContainer(c, msg.ID)
					if err != nil {
						log.Println("[CONTAINER ATTACH THREAD ERROR]: " + err.Error())
						break
					}
					containerAttached[msg.ID] = startingContainer.State.StartedAt.Format(time.RFC3339)
				}
			}
		case <-timeout:

			weave, err := c.InspectContainer(weaveID)
			if err != nil {
				return err
			}

			if weave.State.Running != true {
				os.Exit(1)
			}

			containers, err := c.ListContainers(docker.ListContainersOptions{All: false, Size: true, Limit: 0, Since: "", Before: ""})
			if err != nil {
				log.Println("[CONTAINER ATTACH THREAD ERROR]: Listing Containers failed")
				return err
			}

			for _, container := range containers {

				containerList = append(containerList, container.ID)

				for k, _ := range containerAttached {
					if !stringInSlice(k, containerList) {
						delete(containerAttached, k)
					}
				}

				containerList = []string{}

				runningContainer, err := c.InspectContainer(container.ID)

				if err != nil {
					return err
				}

				if val, ok := containerAttached[container.ID]; ok && val == runningContainer.State.StartedAt.Format(time.RFC3339) {
					break
				} else {
					log.Println("[CONTAINER ATTACH THREAD]: Found running container with ID: " + container.ID)
					err := AttachContainer(c, container.ID)
					if err != nil {
						log.Println("[CONTAINER ATTACH THREAD ERROR]: Attaching Containers failed")
						return err
					}

					containerAttached[container.ID] = runningContainer.State.StartedAt.Format(time.RFC3339)

					err = c.AddEventListener(listener)
					if err != nil {
						log.Println("[CONTAINER ATTACH THREAD ERROR]: Listening Containers Events failed")
						return err
					}

					defer func() error {

						err = c.RemoveEventListener(listener)
						if err != nil {
							return err
						}
						return nil
					}()
				}
				break
			}
			break
		}
	}
}

func discovering(wg *sync.WaitGroup) {
	defer wg.Done()
	c := make(chan tutum.Event)
	e := make(chan error)

	nodes.DiscoverPeers()

	go tutum.TutumEvents(c, e)
Loop:
	for {
		select {
		case event := <-c:
			if event.Type == "node" && (event.State == "Deployed" || event.State == "Terminated") {
				err := nodes.DiscoverPeers()
				if err != nil {
					log.Println(err)
				}
			}
			break
		case err := <-e:
			log.Println("[NODE DISCOVERY ERROR]: " + err.Error())
			time.Sleep(5 * time.Second)
			wg.Add(1)
			go discovering(wg)
			break Loop
		}
	}
}

func containerThread(client *docker.Client, wg *sync.WaitGroup) {
	defer wg.Done()
	for {
		err := ContainerAttachThread(client)
		if err != nil {
			log.Println(err)
			time.Sleep(15 * time.Second)
		}
	}
}

func connectToDocker() (*docker.Client, error) {
	endpoint := "unix:///var/run/docker.sock"

	client, err := docker.NewClient(endpoint)

	if err != nil {

		log.Println(err)
	}
	return client, nil
}

func main() {

	log.Println("Start running daemon")
	wg := &sync.WaitGroup{}
	wg.Add(2)
	//Init Docker client
	client, err := connectToDocker()
	if err != nil {
		log.Println(err)
		os.Exit(1)
	}
	log.Println("Starting container discovery goroutine")
	go containerThread(client, wg)

	tries := 0
	if nodes.Tutum_Node_Api_Uri != "" {
	Loop:
		for {
			tutum.SetUserAgent("weave-daemon/" + version)
			node, err := tutum.GetNode(nodes.Tutum_Node_Api_Uri)
			if err != nil {
				tries++
				log.Println(err)
				time.Sleep(5 * time.Second)
				if tries > 3 {
					time.Sleep(30 * time.Second)
					tries = 0
				}
				continue Loop
			} else {
				nodes.Tutum_Node_Public_Ip = node.Public_ip
				log.Printf("This node IP is %s", nodes.Tutum_Node_Public_Ip)
				if os.Getenv("TUTUM_AUTH") != "" {
					log.Println("Detected Tutum API access - starting peer discovery goroutine")
					go discovering(wg)
					break Loop
				}
			}
		}
	}
	wg.Wait()
}
<commit_msg>Change websocket library + ping handler<commit_after>package main // import "github.com/tutumcloud/weave-daemon"

import (
	"log"
	"os"
	"os/exec"
	"strings"
	"sync"
	"time"

	"github.com/fsouza/go-dockerclient"
	"github.com/tutumcloud/go-tutum/tutum"
	"github.com/tutumcloud/weave-daemon/nodes"
)

const version = "0.15.2"

func stringInSlice(a string, list []string) bool {
	for _, b := range list {
		if b == a {
			return true
		}
	}
	return false
}

func AttachContainer(c *docker.Client, container_id string) error {
	log.Println("[CONTAINER ATTACH]: Inspecting Containers " + container_id)
	inspect, err := c.InspectContainer(container_id)

	if err != nil {
		log.Println("[CONTAINER ATTACH]: Inspecting Containers failed")
		return err
	}

	log.Println("[CONTAINER ATTACH]: Attaching container " + container_id)

	cidr := ""
	env_vars := inspect.Config.Env

	for i := range env_vars {
		if strings.HasPrefix(env_vars[i], "TUTUM_IP_ADDRESS=") {
			cidr = env_vars[i][len("TUTUM_IP_ADDRESS="):]
			break
		}
	}

	if cidr != "" {
		tries := 0
		for {

			cmd := exec.Command("/weave", "--local", "attach", cidr, container_id)

			_, err := cmd.StdoutPipe()
			if err != nil {
				return err
			}

			if err := cmd.Start(); err != nil {
				tries++
				time.Sleep(2 * time.Second)
				log.Println("[CONTAINER ATTACH ERROR]: Start weave cmd failed")
				if tries > 3 {
					return err
				}
			}

			if err := cmd.Wait(); err != nil {
				tries++
				time.Sleep(2 * time.Second)
				log.Println("[CONTAINER ATTACH ERROR]: Wait weave cmd failed")
				if tries > 3 {
					return err
				}
			} else {
				log.Printf("%s: adding to weave with IP %s", container_id, cidr)
				break
			}
		}
	} else {
		log.Printf("%s: cannot find the IP address to add to weave", container_id)
	}
	return nil
}

func ContainerAttachThread(c *docker.Client) error {
	var weaveID = ""
	listener := make(chan *docker.APIEvents)
	containerAttached := make(map[string]string)
	containerList := []string{}

	containers, err := c.ListContainers(docker.ListContainersOptions{All: false, Size: true, Limit: 0, Since: "", Before: ""})
	if err != nil {
		log.Println("[CONTAINER ATTACH THREAD ERROR]: Listing Containers failed")
		return err
	}

	for _, container := range containers {

		runningContainer, err := c.InspectContainer(container.ID)

		if err != nil {
			return err
		}

		log.Println("[CONTAINER ATTACH THREAD]: Found running container with ID: " + container.ID)

		if strings.HasPrefix(container.Image, "weaveworks/weave:") {
			weaveID = container.ID
		}

		err = AttachContainer(c, container.ID)
		if err != nil {
			log.Println("[CONTAINER ATTACH THREAD ERROR]: Attaching Containers failed")
			return err
		}
		containerAttached[container.ID] = runningContainer.State.StartedAt.Format(time.RFC3339)
	}

	err = c.AddEventListener(listener)
	if err != nil {
		log.Println("[CONTAINER ATTACH THREAD ERROR]: Listening Containers Events failed")
		return err
	}

	defer func() error {

		err = c.RemoveEventListener(listener)
		if err != nil {
			return err
		}
		return nil
	}()

	if weaveID == "" {
		os.Exit(1)
	}

	log.Println("WEAVE ID is : " + weaveID)

	for {
		timeout := time.Tick(2 * time.Minute)
		select {
		case msg := <-listener:
			if msg.Status == "die" && strings.HasPrefix(msg.From, "weaveworks/weave:") {
				os.Exit(1)
			}
			if msg.Status == "start" {
				startingContainer, err := c.InspectContainer(msg.ID)

				if err != nil {
					return err
				}

				if val, ok := containerAttached[msg.ID]; ok && val == startingContainer.State.StartedAt.Format(time.RFC3339) {
					break
				} else {
					err := AttachContainer(c, msg.ID)
					if err != nil {
						log.Println("[CONTAINER ATTACH THREAD ERROR]: " + err.Error())
						break
					}
					containerAttached[msg.ID] = startingContainer.State.StartedAt.Format(time.RFC3339)
				}
			}
		case <-timeout:

			weave, err := c.InspectContainer(weaveID)
			if err != nil {
				return err
			}

			if weave.State.Running != true {
				os.Exit(1)
			}

			containers, err := c.ListContainers(docker.ListContainersOptions{All: false, Size: true, Limit: 0, Since: "", Before: ""})
			if err != nil {
				log.Println("[CONTAINER ATTACH THREAD ERROR]: Listing Containers failed")
				return err
			}

			for _, container := range containers {

				containerList = append(containerList, container.ID)

				for k, _ := range containerAttached {
					if !stringInSlice(k, containerList) {
						delete(containerAttached, k)
					}
				}

				containerList = []string{}

				runningContainer, err := c.InspectContainer(container.ID)

				if err != nil {
					return err
				}

				if val, ok := containerAttached[container.ID]; ok && val == runningContainer.State.StartedAt.Format(time.RFC3339) {
					break
				} else {
					log.Println("[CONTAINER ATTACH THREAD]: Found running container with ID: " + container.ID)
					err := AttachContainer(c, container.ID)
					if err != nil {
						log.Println("[CONTAINER ATTACH THREAD ERROR]: Attaching Containers failed")
						return err
					}

					containerAttached[container.ID] = runningContainer.State.StartedAt.Format(time.RFC3339)

					err = c.AddEventListener(listener)
					if err != nil {
						log.Println("[CONTAINER ATTACH THREAD ERROR]: Listening Containers Events failed")
						return err
					}

					defer func() error {

						err = c.RemoveEventListener(listener)
						if err != nil {
							return err
						}
						return nil
					}()
				}
				break
			}
			break
		}
	}
}

func discovering(wg *sync.WaitGroup) {
	defer wg.Done()
	c := make(chan tutum.Event)
	e := make(chan error)

	nodes.DiscoverPeers()

	go tutum.TutumEvents(c, e)
Loop:
	//ticker := time.NewTicker(5 * time.Second)
	for {
		select {
		case event := <-c:
			if event.Type == "node" && (event.State == "Deployed" || event.State == "Terminated") {
				err := nodes.DiscoverPeers()
				if err != nil {
					log.Println(err)
				}
			}
			break
		case err := <-e:
			log.Println("[NODE DISCOVERY ERROR]: " + err.Error())
			time.Sleep(5 * time.Second)
			wg.Add(1)
			go discovering(wg)
			break Loop
		}
	}
}

func containerThread(client *docker.Client, wg *sync.WaitGroup) {
	defer wg.Done()
	for {
		err := ContainerAttachThread(client)
		if err != nil {
			log.Println(err)
			time.Sleep(15 * time.Second)
		}
	}
}

func connectToDocker() (*docker.Client, error) {
	endpoint := "unix:///var/run/docker.sock"

	client, err := docker.NewClient(endpoint)

	if err != nil {

		log.Println(err)
	}
	return client, nil
}

func main() {

	log.Println("Start running daemon")
	wg := &sync.WaitGroup{}
	wg.Add(2)
	//Init Docker client
	client, err := connectToDocker()
	if err != nil {
		log.Println(err)
		os.Exit(1)
	}
	log.Println("Starting container discovery goroutine")
	go containerThread(client, wg)

	tries := 0
	if nodes.Tutum_Node_Api_Uri != "" {
	Loop:
		for {
			tutum.SetUserAgent("weave-daemon/" + version)
			node, err := tutum.GetNode(nodes.Tutum_Node_Api_Uri)
			if err != nil {
				tries++
				log.Println(err)
				time.Sleep(5 * time.Second)
				if tries > 3 {
					time.Sleep(30 * time.Second)
					tries = 0
				}
				continue Loop
			} else {
				nodes.Tutum_Node_Public_Ip = node.Public_ip
				log.Printf("This node IP is %s", nodes.Tutum_Node_Public_Ip)
				if os.Getenv("TUTUM_AUTH") != "" {
					log.Println("Detected Tutum API access - starting peer discovery goroutine")
					go discovering(wg)
					break Loop
				}
			}
		}
	}
	wg.Wait()
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	"log"
	"os"
	"os/exec"
	"sync"
)

func run(in, out chan []byte, wg *sync.WaitGroup) {

	wg.Add(1)
	defer wg.Done()

	cmd := exec.Command(flag.Args()[0], flag.Args()[1:]...)
	stdin, _ := cmd.StdinPipe()
	stdout, _ := cmd.StdoutPipe()
	cmd.Start()

	scanner := bufio.NewScanner(stdout)
	go func() {
		wg.Add(1)
		defer wg.Done()
		for scanner.Scan() {
			b := scanner.Bytes()
			c := make([]byte, len(b), len(b)+1)
			copy(c, b)
			out <- append(c, '\n')
		}
	}()

	for b := range in {
		i, err := stdin.Write(b)
		if err != nil {
			log.Println(i, err)
		}
	}

	stdin.Close() // signal for child process to exit
	cmd.Wait()
}

func init() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "usage: np [-n numprocs] command\n")
		flag.PrintDefaults()
	}
}

func main() {

	var n int
	flag.IntVar(&n, "n", 1, "number of processes to run")
	flag.Parse()
	if flag.NArg() == 0 {
		flag.Usage()
		os.Exit(1)
	}

	in := make(chan []byte)
	out := make(chan []byte)

	var wg sync.WaitGroup
	for i := 0; i < n; i++ {
		go run(in, out, &wg)
	}

	go func() {
		scanner := bufio.NewScanner(os.Stdin)
		for scanner.Scan() {
			b := scanner.Bytes()
			c := make([]byte, len(b), len(b)+1)
			copy(c, b)
			in <- append(c, '\n')
		}
		close(in)
		wg.Wait() // wait for all children to finish
		close(out)
	}()

	for b := range out {
		os.Stdout.Write(b)
	}
}
<commit_msg>Cleaned up vars names and commented.<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	"log"
	"os"
	"os/exec"
	"sync"
)

var (
	inChan  = make(chan []byte)
	outChan = make(chan []byte)
	wg      sync.WaitGroup
)

func init() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "usage: np [-n numprocs] command\n")
		flag.PrintDefaults()
	}
}

func worker() {
	defer wg.Done()

	cmd := exec.Command(flag.Args()[0], flag.Args()[1:]...)
	cmdStdin, err := cmd.StdinPipe()
	if err != nil {
		log.Fatal(err)
	}
	cmdStdout, err := cmd.StdoutPipe()
	if err != nil {
		log.Fatal(err)
	}
	cmd.Start()

	output := bufio.NewScanner(cmdStdout)
	go func() {
		for output.Scan() {
			b := output.Bytes()
			c := make([]byte, len(b), len(b)+1)
			copy(c, b)
			outChan <- append(c, '\n')
		}
	}()

	for line := range inChan {
		i, err := cmdStdin.Write(line)
		if err != nil {
			log.Println(i, err)
		}
	}

	cmdStdin.Close() // signal for child process to exit
	cmd.Wait()
}

func main() {

	var n int
	flag.IntVar(&n, "n", 1, "number of processes to run")
	flag.Parse()
	if flag.NArg() == 0 {
		flag.Usage()
		os.Exit(1)
	}

	for i := 0; i < n; i++ {
		wg.Add(1)
		go worker()
	}

	go func() {
		input := bufio.NewScanner(os.Stdin)
		for input.Scan() {
			b := input.Bytes()
			c := make([]byte, len(b), len(b)+1)
			copy(c, b)
			inChan <- append(c, '\n')
		}
		close(inChan)  // signal to workers to exit
		wg.Wait()      // wait for all workers
		close(outChan) // signal completion to exit main loop
	}()

	// this when exit when outChan closed by the goroutine above
	for line := range outChan {
		os.Stdout.Write(line)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"crypto/rand"
	"crypto/tls"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/mpl/basicauth"
	"golang.org/x/crypto/acme/autocert"
)

const (
	idstring = "http://golang.org/pkg/http/#ListenAndServe"
)

var (
	flagHost     = flag.String("host", "0.0.0.0:8080", "listening port and hostname")
	flagHelp     = flag.Bool("h", false, "show this help")
	flagUserpass = flag.String("userpass", "", "optional username:password protection")
	flagCommand  = flag.String("command", "", "The command to run")
	flagRate     = flag.Duration("rate", time.Second, "To limit the number of processes created to no more than one per given duration. Set to 0 for no limit.")
	flagAutocert = flag.Bool("autocert", true, `Get https certificate from Let's Encrypt. Obviously -host must contain a full qualified domain name. The cached certificate(s) will be in $HOME/keys/letsencrypt.cache.`)
)

var (
	rootdir, _ = os.Getwd()
	up         *basicauth.UserPass
	tlsKey     = filepath.Join(os.Getenv("HOME"), "keys", "key.pem")
	tlsCert    = filepath.Join(os.Getenv("HOME"), "keys", "cert.pem")
	certCache  = filepath.Join(os.Getenv("HOME"), "keys", "letsencrypt.cache")

	childrenMu sync.RWMutex
	children   map[time.Time]*os.Process

	// TODO(mpl): rate limit per source ip instead of for all requests?
	lastRunMu sync.RWMutex
	lastRun   time.Time
)

func usage() {
	fmt.Fprintf(os.Stderr, "\t httprunner \n")
	flag.PrintDefaults()
	fmt.Fprint(os.Stderr, "The endpoints are /run, /ls, /kill, and /die.\n")
	os.Exit(2)
}

func makeHandler(fn func(http.ResponseWriter, *http.Request)) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		defer func() {
			if e, ok := recover().(error); ok {
				http.Error(w, e.Error(), http.StatusInternalServerError)
				return
			}
		}()
		w.Header().Set("Server", idstring)
		if isAllowed(r) {
			fn(w, r)
		} else {
			basicauth.SendUnauthorized(w, r, "httprunner")
		}
	}
}

func isAllowed(r *http.Request) bool {
	if *flagUserpass == "" {
		return true
	}
	return up.IsAllowed(r)
}

func initUserPass() {
	if *flagUserpass == "" {
		return
	}
	var err error
	up, err = basicauth.New(*flagUserpass)
	if err != nil {
		log.Fatal(err)
	}
}

// TODO(mpl): have a look at https://github.com/cespare/window
// Does not work for me as it is, since it's not a reader as well.

type limitWriter struct {
	deadline time.Time
	limit    int
	sum      int

	bufMu sync.Mutex
	buf   *bytes.Buffer

	discardingMu sync.RWMutex
	discarding   bool
}

func (lw limitWriter) Write(p []byte) (n int, err error) {
	lw.discardingMu.RLock()
	if lw.discarding {
		lw.discardingMu.RUnlock()
		return ioutil.Discard.Write(p)
	}
	lw.discardingMu.RUnlock()
	lw.bufMu.Lock()
	n, err = lw.buf.Write(p)
	lw.bufMu.Unlock()
	lw.sum += n
	if lw.sum > lw.limit {
		lw.discardingMu.Lock()
		lw.discarding = true
		lw.discardingMu.Unlock()
	}
	return
}

func (lw limitWriter) Read(p []byte) (n int, err error) {
	lw.discardingMu.RLock()
	if lw.discarding {
		lw.discardingMu.RUnlock()
		return 0, io.EOF
	}
	lw.discardingMu.RUnlock()
	lw.bufMu.Lock()
	defer lw.bufMu.Unlock()
	return lw.buf.Read(p)
}

func killChildren() {
	childrenMu.Lock()
	defer childrenMu.Unlock()
	for _, v := range children {
		if err := v.Kill(); err != nil {
			log.Printf("couldn't kill child: %v", err)
		}
	}
	children = make(map[time.Time]*os.Process)
}

func handleKillAll(w http.ResponseWriter, r *http.Request) {
	killChildren()
	if _, err := io.Copy(w, strings.NewReader("They have left for a better world.")); err != nil {
		log.Print(err)
	}
}

func handleDie(w http.ResponseWriter, r *http.Request) {
	killChildren()
	sayonara := "The sweet embrace of death, finally."
	if _, err := io.Copy(w, strings.NewReader(sayonara)); err != nil {
		log.Print(err)
	}
	log.Print(sayonara)
	time.Sleep(time.Second)
	os.Exit(0)
}

type times []time.Time

func (t times) Len() int           { return len(t) }
func (t times) Swap(i, j int)      { t[i], t[j] = t[j], t[i] }
func (t times) Less(i, j int) bool { return t[i].Before(t[j]) }

func handleList(w http.ResponseWriter, r *http.Request) {
	childrenMu.RLock()
	defer childrenMu.RUnlock()
	var t times
	for k, _ := range children {
		t = append(t, k)
	}
	sort.Sort(t)
	var out bytes.Buffer
	for _, pt := range t {
		if _, err := out.WriteString(fmt.Sprintf("%s : %d\n", pt.Format(time.RFC3339), children[pt].Pid)); err != nil {
			http.Error(w, "can't print children list", http.StatusInternalServerError)
			return
		}
	}
	if _, err := io.Copy(w, &out); err != nil {
		log.Printf("error listing children: %v", err)
	}
}

func handleCommand(w http.ResponseWriter, r *http.Request) {
	if *flagRate != 0 {
		lastRunMu.RLock()
		if time.Now().Before(lastRun.Add(*flagRate)) {
			http.Error(w, "Command process creation is rate limited", http.StatusTooManyRequests)
			lastRunMu.RUnlock()
			return
		}
		lastRunMu.RUnlock()
	}
	// TODO(mpl): be less lazy about the doubled spaces, and probably other things.
	args := strings.Fields(*flagCommand)
	cmd := exec.Command(args[0], args[1:]...)
	var buf, berr bytes.Buffer
	lw := limitWriter{
		limit: 1 << 20,
		buf:   &buf,
	}
	stdout := io.MultiWriter(os.Stdout, lw)
	cmd.Stdout = stdout
	cmd.Stderr = &berr
	if err := cmd.Start(); err != nil {
		log.Printf("%v failed to start: %v, %v", args[0], err, berr.String())
		return
	}
	log.Printf("Started %v with pid %v", args[0], cmd.Process.Pid)
	startTime := time.Now()
	childrenMu.Lock()
	children[startTime] = cmd.Process
	childrenMu.Unlock()
	lastRunMu.Lock()
	lastRun = time.Now()
	lastRunMu.Unlock()
	go func() {
		if err := cmd.Wait(); err != nil {
			log.Printf("%v failed: %v, %v", args[0], err, berr.String())
		}
		childrenMu.Lock()
		delete(children, startTime)
		childrenMu.Unlock()
	}()
	var bufout bytes.Buffer
	sendResponse := func(b *bytes.Buffer) {
		var response io.Reader
		if b.Len() > 0 {
			response = b
		} else {
			response = strings.NewReader("Command started but no output yet.")
		}
		if _, err := io.Copy(w, response); err != nil {
			log.Printf("response copy error: %v", err)
		}
	}
	var seenData bool
	// TODO(mpl): test if we could relax both these times now that we're sending the header asap.
	maxIdle := 200 * time.Millisecond
	t := time.After(1 * time.Second)
	lastDataTime := time.Now()
	for {
		select {
		case <-t:
			sendResponse(&bufout)
			return
		default:
		}
		n, err := io.Copy(&bufout, lw)
		if err != nil {
			log.Printf("output copy error: %v", err)
			break
		}
		if n > 0 {
			if !seenData {
				w.Header().Set("Content-Type", "text/plain; charset=utf-8")
				w.WriteHeader(http.StatusOK)
				seenData = true
			}
			lastDataTime = time.Now()
		} else {
			if lastDataTime.Add(maxIdle).Before(time.Now()) {
				log.Printf("no output for more than %v, wrapping up.", maxIdle)
				break
			}
		}
	}
	sendResponse(&bufout)
}

func setupTLS() (*tls.Config, error) {
	hostname := *flagHost
	if strings.Contains(hostname, ":") {
		h, _, err := net.SplitHostPort(hostname)
		if err != nil {
			return nil, err
		}
		hostname = h
	}
	if *flagAutocert {
		m := autocert.Manager{
			Prompt:     autocert.AcceptTOS,
			HostPolicy: autocert.HostWhitelist(hostname),
			Cache:      autocert.DirCache(certCache),
		}
		return &tls.Config{
			Rand:           rand.Reader,
			Time:           time.Now,
			NextProtos:     []string{"http/1.1"},
			GetCertificate: m.GetCertificate,
		}, nil
	}
	cert, err := tls.LoadX509KeyPair(tlsCert, tlsKey)
	if err != nil {
		return nil, fmt.Errorf("Failed to load TLS cert: %v", err)
	}
	return &tls.Config{
		Rand:         rand.Reader,
		Time:         time.Now,
		NextProtos:   []string{"http/1.1"},
		Certificates: []tls.Certificate{cert},
	}, nil
}

func main() {
	flag.Usage = usage
	flag.Parse()
	if *flagHelp {
		usage()
	}
	nargs := flag.NArg()
	if nargs > 0 {
		usage()
	}
	if *flagCommand == "" {
		fmt.Printf("No command to run")
		usage()
	}

	initUserPass()
	children = make(map[time.Time]*os.Process)

	listener, err := net.Listen("tcp", *flagHost)
	if err != nil {
		log.Fatalf("Failed to listen on %s: %v", *flagHost, err)
	}
	config, err := setupTLS()
	if err != nil {
		log.Fatalf("could not configure TLS connection: %v", err)
	}
	listener = tls.NewListener(listener, config)

	http.Handle("/run", makeHandler(handleCommand))
	http.Handle("/kill", makeHandler(handleKillAll))
	http.Handle("/die", makeHandler(handleDie))
	http.Handle("/ls", makeHandler(handleList))
	log.Fatal(http.Serve(listener, nil))
}
<commit_msg>use simpletls for autocert<commit_after>package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/mpl/basicauth"
	"github.com/mpl/simpletls"
)

const (
	idstring = "http://golang.org/pkg/http/#ListenAndServe"
)

var (
	flagHost     = flag.String("host", "0.0.0.0:8080", "listening port and hostname")
	flagHelp     = flag.Bool("h", false, "show this help")
	flagUserpass = flag.String("userpass", "", "optional username:password protection")
	flagCommand  = flag.String("command", "", "The command to run")
	flagRate     = flag.Duration("rate", time.Second, "To limit the number of processes created to no more than one per given duration. Set to 0 for no limit.")
)

var (
	rootdir, _ = os.Getwd()
	up         *basicauth.UserPass

	childrenMu sync.RWMutex
	children   map[time.Time]*os.Process

	// TODO(mpl): rate limit per source ip instead of for all requests?
	lastRunMu sync.RWMutex
	lastRun   time.Time
)

func usage() {
	fmt.Fprintf(os.Stderr, "\t httprunner \n")
	flag.PrintDefaults()
	fmt.Fprint(os.Stderr, "The endpoints are /run, /ls, /kill, and /die.\n")
	os.Exit(2)
}

func makeHandler(fn func(http.ResponseWriter, *http.Request)) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		defer func() {
			if e, ok := recover().(error); ok {
				http.Error(w, e.Error(), http.StatusInternalServerError)
				return
			}
		}()
		w.Header().Set("Server", idstring)
		if isAllowed(r) {
			fn(w, r)
		} else {
			basicauth.SendUnauthorized(w, r, "httprunner")
		}
	}
}

func isAllowed(r *http.Request) bool {
	if *flagUserpass == "" {
		return true
	}
	return up.IsAllowed(r)
}

func initUserPass() {
	if *flagUserpass == "" {
		return
	}
	var err error
	up, err = basicauth.New(*flagUserpass)
	if err != nil {
		log.Fatal(err)
	}
}

// TODO(mpl): have a look at https://github.com/cespare/window
// Does not work for me as it is, since it's not a reader as well.

type limitWriter struct {
	deadline time.Time
	limit    int
	sum      int

	bufMu sync.Mutex
	buf   *bytes.Buffer

	discardingMu sync.RWMutex
	discarding   bool
}

func (lw limitWriter) Write(p []byte) (n int, err error) {
	lw.discardingMu.RLock()
	if lw.discarding {
		lw.discardingMu.RUnlock()
		return ioutil.Discard.Write(p)
	}
	lw.discardingMu.RUnlock()
	lw.bufMu.Lock()
	n, err = lw.buf.Write(p)
	lw.bufMu.Unlock()
	lw.sum += n
	if lw.sum > lw.limit {
		lw.discardingMu.Lock()
		lw.discarding = true
		lw.discardingMu.Unlock()
	}
	return
}

func (lw limitWriter) Read(p []byte) (n int, err error) {
	lw.discardingMu.RLock()
	if lw.discarding {
		lw.discardingMu.RUnlock()
		return 0, io.EOF
	}
	lw.discardingMu.RUnlock()
	lw.bufMu.Lock()
	defer lw.bufMu.Unlock()
	return lw.buf.Read(p)
}

func killChildren() {
	childrenMu.Lock()
	defer childrenMu.Unlock()
	for _, v := range children {
		if err := v.Kill(); err != nil {
			log.Printf("couldn't kill child: %v", err)
		}
	}
	children = make(map[time.Time]*os.Process)
}

func handleKillAll(w http.ResponseWriter, r *http.Request) {
	killChildren()
	if _, err := io.Copy(w, strings.NewReader("They have left for a better world.")); err != nil {
		log.Print(err)
	}
}

func handleDie(w http.ResponseWriter, r *http.Request) {
	killChildren()
	sayonara := "The sweet embrace of death, finally."
	if _, err := io.Copy(w, strings.NewReader(sayonara)); err != nil {
		log.Print(err)
	}
	log.Print(sayonara)
	time.Sleep(time.Second)
	os.Exit(0)
}

type times []time.Time

func (t times) Len() int           { return len(t) }
func (t times) Swap(i, j int)      { t[i], t[j] = t[j], t[i] }
func (t times) Less(i, j int) bool { return t[i].Before(t[j]) }

func handleList(w http.ResponseWriter, r *http.Request) {
	childrenMu.RLock()
	defer childrenMu.RUnlock()
	var t times
	for k, _ := range children {
		t = append(t, k)
	}
	sort.Sort(t)
	var out bytes.Buffer
	for _, pt := range t {
		if _, err := out.WriteString(fmt.Sprintf("%s : %d\n", pt.Format(time.RFC3339), children[pt].Pid)); err != nil {
			http.Error(w, "can't print children list", http.StatusInternalServerError)
			return
		}
	}
	if _, err := io.Copy(w, &out); err != nil {
		log.Printf("error listing children: %v", err)
	}
}

func handleCommand(w http.ResponseWriter, r *http.Request) {
	if *flagRate != 0 {
		lastRunMu.RLock()
		if time.Now().Before(lastRun.Add(*flagRate)) {
			http.Error(w, "Command process creation is rate limited", http.StatusTooManyRequests)
			lastRunMu.RUnlock()
			return
		}
		lastRunMu.RUnlock()
	}
	// TODO(mpl): be less lazy about the doubled spaces, and probably other things.
	args := strings.Fields(*flagCommand)
	cmd := exec.Command(args[0], args[1:]...)
	var buf, berr bytes.Buffer
	lw := limitWriter{
		limit: 1 << 20,
		buf:   &buf,
	}
	stdout := io.MultiWriter(os.Stdout, lw)
	cmd.Stdout = stdout
	cmd.Stderr = &berr
	if err := cmd.Start(); err != nil {
		log.Printf("%v failed to start: %v, %v", args[0], err, berr.String())
		return
	}
	log.Printf("Started %v with pid %v", args[0], cmd.Process.Pid)
	startTime := time.Now()
	childrenMu.Lock()
	children[startTime] = cmd.Process
	childrenMu.Unlock()
	lastRunMu.Lock()
	lastRun = time.Now()
	lastRunMu.Unlock()
	go func() {
		if err := cmd.Wait(); err != nil {
			log.Printf("%v failed: %v, %v", args[0], err, berr.String())
		}
		childrenMu.Lock()
		delete(children, startTime)
		childrenMu.Unlock()
	}()
	var bufout bytes.Buffer
	sendResponse := func(b *bytes.Buffer) {
		var response io.Reader
		if b.Len() > 0 {
			response = b
		} else {
			response = strings.NewReader("Command started but no output yet.")
		}
		if _, err := io.Copy(w, response); err != nil {
			log.Printf("response copy error: %v", err)
		}
	}
	var seenData bool
	// TODO(mpl): test if we could relax both these times now that we're sending the header asap.
	maxIdle := 200 * time.Millisecond
	t := time.After(1 * time.Second)
	lastDataTime := time.Now()
	for {
		select {
		case <-t:
			sendResponse(&bufout)
			return
		default:
		}
		n, err := io.Copy(&bufout, lw)
		if err != nil {
			log.Printf("output copy error: %v", err)
			break
		}
		if n > 0 {
			if !seenData {
				w.Header().Set("Content-Type", "text/plain; charset=utf-8")
				w.WriteHeader(http.StatusOK)
				seenData = true
			}
			lastDataTime = time.Now()
		} else {
			if lastDataTime.Add(maxIdle).Before(time.Now()) {
				log.Printf("no output for more than %v, wrapping up.", maxIdle)
				break
			}
		}
	}
	sendResponse(&bufout)
}

func main() {
	flag.Usage = usage
	flag.Parse()
	if *flagHelp {
		usage()
	}
	nargs := flag.NArg()
	if nargs > 0 {
		usage()
	}
	if *flagCommand == "" {
		fmt.Printf("No command to run")
		usage()
	}

	initUserPass()
	children = make(map[time.Time]*os.Process)

	listener, err := simpletls.Listen(*flagHost)
	if err != nil {
		log.Fatalf("Failed to listen on %s: %v", *flagHost, err)
	}

	http.Handle("/run", makeHandler(handleCommand))
	http.Handle("/kill", makeHandler(handleKillAll))
	http.Handle("/die", makeHandler(handleDie))
	http.Handle("/ls", makeHandler(handleList))
	log.Fatal(http.Serve(listener, nil))
}
<|endoftext|>


<commit_before>package main

import (
	"net/url"
	"strings"

	"github.com/fsouza/go-dockerclient"
)

var proxiedPorts map[int]*SinglePortProxy
var b2dhost string

func main() {
	//todo get this information from boot2docker or shell environment
	endpoint := "tcp://192.168.59.103:2376"
	parsed, err := url.ParseRequestURI(endpoint)
	if err != nil {
		panic(err)
	}

	colonIndex := strings.LastIndex(parsed.Host, ":")
	if colonIndex != -1 {
		b2dhost = parsed.Host[:colonIndex]
	} else {
		b2dhost = parsed.Host
	}

	client, err := docker.NewTLSClient(endpoint,
		"/Users/joerg/.boot2docker/certs/boot2docker-vm/cert.pem",
		"/Users/joerg/.boot2docker/certs/boot2docker-vm/key.pem",
		"/Users/joerg/.boot2docker/certs/boot2docker-vm/ca.pem")
	if err != nil {
		panic(err)
	}

	//initial read of ports
	proxiedPorts = make(map[int]*SinglePortProxy)
	updateports(client)

	//now listen for events to update ports
	events := make(chan *docker.APIEvents)
	err = client.AddEventListener(events)
	if err != nil {
		panic(err)
	}

	for {
		<-events //it does't matter what kind of update, just refresh the ports
		updateports(client)
	}

}

func updateports(client *docker.Client) {
	containers, err := client.ListContainers(docker.ListContainersOptions{All: true})
	if err != nil {
		panic(err)
	}

	currentports := findCurrentPorts(containers)
	removeOldPorts(currentports)
	addNewPorts(currentports)
}

func findCurrentPorts(containers []docker.APIContainers) map[int]bool {

	currentports := make(map[int]bool)

	for _, container := range containers {
		for _, port := range container.Ports {
			if port.PublicPort != 0 {
				currentports[int(port.PublicPort)] = true
			}
		}
	}

	return currentports
}

func removeOldPorts(currentports map[int]bool) {
	for port, proxy := range proxiedPorts {
		if !currentports[port] {
			proxy.stopListen()
			delete(proxiedPorts, port)
		}
	}
}

func addNewPorts(currentports map[int]bool) {
	for port, _ := range currentports {
		if proxiedPorts[port] == nil {
			proxiedPorts[port] = NewSinglePortProxy(b2dhost, port)
		}
	}
}
<commit_msg>Using net.SplitHostPort<commit_after>package main

import (
	"net"
	"net/url"

	"github.com/fsouza/go-dockerclient"
)

var proxiedPorts map[int]*SinglePortProxy
var b2dhost string

func main() {
	//todo get this information from boot2docker or shell environment
	endpoint := "tcp://192.168.59.103:2376"
	parsed, err := url.ParseRequestURI(endpoint)
	if err != nil {
		panic(err)
	}

	b2dhost, _, err = net.SplitHostPort(parsed.Host)
	if err != nil {
		panic(err)
	}

	client, err := docker.NewTLSClient(endpoint,
		"/Users/joerg/.boot2docker/certs/boot2docker-vm/cert.pem",
		"/Users/joerg/.boot2docker/certs/boot2docker-vm/key.pem",
		"/Users/joerg/.boot2docker/certs/boot2docker-vm/ca.pem")
	if err != nil {
		panic(err)
	}

	//initial read of ports
	proxiedPorts = make(map[int]*SinglePortProxy)
	updateports(client)

	//now listen for events to update ports
	events := make(chan *docker.APIEvents)
	err = client.AddEventListener(events)
	if err != nil {
		panic(err)
	}

	for {
		<-events //it does't matter what kind of update, just refresh the ports
		updateports(client)
	}

}

func updateports(client *docker.Client) {
	containers, err := client.ListContainers(docker.ListContainersOptions{All: true})
	if err != nil {
		panic(err)
	}

	currentports := findCurrentPorts(containers)
	removeOldPorts(currentports)
	addNewPorts(currentports)
}

func findCurrentPorts(containers []docker.APIContainers) map[int]bool {

	currentports := make(map[int]bool)

	for _, container := range containers {
		for _, port := range container.Ports {
			if port.PublicPort != 0 {
				currentports[int(port.PublicPort)] = true
			}
		}
	}

	return currentports
}

func removeOldPorts(currentports map[int]bool) {
	for port, proxy := range proxiedPorts {
		if !currentports[port] {
			proxy.stopListen()
			delete(proxiedPorts, port)
		}
	}
}

func addNewPorts(currentports map[int]bool) {
	for port, _ := range currentports {
		if proxiedPorts[port] == nil {
			proxiedPorts[port] = NewSinglePortProxy(b2dhost, port)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"path/filepath"
)

var isVerbose bool
var isQuiet bool

func main() {
	logger.msg("Start.")

	// Parse arguments

	args, err := parseArguments()
	if err != nil {
		outError("Error parsing aruments: %s", err)
		exit(1)
	}

	logger.msg("Arguments passed: %+v", args)

	isVerbose = args["--verbose"].(bool)
	isQuiet = args["--quiet"].(bool)

	var configPath string
	if args["--config"] == nil {
		configPath = ""
	} else {
		configPath = args["--config"].(string)
	}

	// Process arguments
	var rc RC
	if configPath == "" {
		rc, err = readRC()
		if err != nil {
			outError("Error reading rc file: %s", err)
			exit(1)
		}
		if rc.Config.Path == "" {
			outError("Config file not specified.")
			exit(1)
		}
	} else {
		// Save to RC file
		configPath, err = filepath.Abs(configPath)
		if err != nil {
			outError("Bad config path: %s", err)
			exit(1)
		}

		rc, err = saveRC(configPath)
		if err != nil {
			outError("Cannot save rc file: %s", err)
			exit(1)
		}
	}

	// Process config

	config, err := configurationFromFile(rc.Config.Path)
	if err != nil {
		outError("Error reading configuration from file %s: %s", rc.Config.Path, err)
		exit(1)
	}

	// Preparations

	err = os.MkdirAll(config.Directories.Backup, 0755)
	if err != nil && !os.IsExist(err) {
		outError("Error creating backup directory: %s", err)
		exit(1)
	}

	if isVerbose {
		outVerbose("Dotfiles root: %s", config.Directories.Dotfiles)
		outVerbose("Dotfiles src: %s", config.Directories.Sources)
		outVerbose("Destination dir: %s", config.Directories.Destination)
	}

	err = cleanDeadSymlinks(config.Directories.Destination)
	if err != nil {
		outError("Error cleaning dead symlinks: %s", err)
		exit(1)
	}

	srcDirAbs := config.Directories.Dotfiles
	if config.Directories.Sources != "" {
		if _, err = os.Stat(config.Directories.Sources); os.IsNotExist(err) {
			outError("Sources directory `%s' does not exist.", config.Directories.Sources)
			exit(1)
		}
		if err != nil {
			outError("Error reading sources directory `%s': %s", config.Directories.Sources, err)
			exit(1)
		}
		srcDirAbs += "/" + config.Directories.Sources
	}

	mapping := make(map[string]string)

	if len(config.Mapping) == 0 {
		// install all the things
		outVerbose("Mapping is not specified - install all the things")
		dir, err := os.Open(srcDirAbs)
		if err != nil {
			outError("Error reading dotfiles source dir: %s", err)
			exit(1)
		}

		defer dir.Close()

		files, err := dir.Readdir(0)
		if err != nil {
			outError("Error reading dotfiles source dir: %s", err)
			exit(1)
		}

		for _, fileInfo := range files {
			mapping[fileInfo.Name()] = fileInfo.Name()
		}

		// filter excludes
		for _, exclude := range config.Files.Excludes {
			if _, ok := mapping[exclude]; ok {
				delete(mapping, exclude)
			}
		}
	} else {
		// install by mapping
		if len(config.Files.Excludes) > 0 {
			outWarn("Excludes in config make no sense when mapping is specified, omitting them.")
		}

		mapping = config.Mapping
	}

	outInfo("Installing dotfiles...")
	for src, dest := range mapping {
		srcAbs := srcDirAbs + "/" + src
		destAbs := config.Directories.Destination + "/" + dest

		exists, err := isExists(srcAbs)
		if !exists {
			outWarn("Source file %s does not exist", srcAbs)
			continue
		}

		if err != nil {
			outError("Error processing source file %s: %s", src, err)
			exit(1)
		}

		needSymlink, needBackup, err := processDest(srcAbs, destAbs)
		if err != nil {
			outError("Error processing destination file %s: %s", destAbs, err)
			exit(1)
		}

		if !needSymlink {
			continue
		}

		if needBackup {
			err = backup(dest, destAbs, config.Directories.Backup)
			if err != nil {
				outError("Error backuping file %s: %s", destAbs, err)
				exit(1)
			}
		}

		err = setSymlink(srcAbs, destAbs)
		if err != nil {
			outError("Error creating symlink from %s to %s: %s", srcAbs, destAbs, err)
			exit(1)
		}
	}

	outInfo("All done (─‿‿─)")
	exit(0)
}
<commit_msg>Reduce cyclomatic complexity in main.go<commit_after>package main

import (
	"os"
	"path/filepath"
)

var isVerbose bool
var isQuiet bool

func main() {
	logger.msg("Start.")

	// Parse arguments

	args, err := parseArguments()
	if err != nil {
		outError("Error parsing aruments: %s", err)
		exit(1)
	}

	logger.msg("Arguments passed: %+v", args)

	isVerbose = args["--verbose"].(bool)
	isQuiet = args["--quiet"].(bool)
	configPath := getConfigPath(args)

	// Process config

	config, err := configurationFromFile(configPath)
	if err != nil {
		outError("Error reading configuration from file %s: %s", configPath, err)
		exit(1)
	}

	// Preparations

	err = os.MkdirAll(config.Directories.Backup, 0755)
	if err != nil && !os.IsExist(err) {
		outError("Error creating backup directory: %s", err)
		exit(1)
	}

	if isVerbose {
		outVerbose("Dotfiles root: %s", config.Directories.Dotfiles)
		outVerbose("Dotfiles src: %s", config.Directories.Sources)
		outVerbose("Destination dir: %s", config.Directories.Destination)
	}

	err = cleanDeadSymlinks(config.Directories.Destination)
	if err != nil {
		outError("Error cleaning dead symlinks: %s", err)
		exit(1)
	}

	srcDirAbs := config.Directories.Dotfiles
	if config.Directories.Sources != "" {
		if _, err = os.Stat(config.Directories.Sources); os.IsNotExist(err) {
			outError("Sources directory `%s' does not exist.", config.Directories.Sources)
			exit(1)
		}
		if err != nil {
			outError("Error reading sources directory `%s': %s", config.Directories.Sources, err)
			exit(1)
		}
		srcDirAbs += "/" + config.Directories.Sources
	}

	mapping := getMapping(config, srcDirAbs)

	outInfo("Installing dotfiles...")
	for src, dest := range mapping {
		installDotfile(src, dest, config, srcDirAbs)
	}

	outInfo("All done (─‿‿─)")
	exit(0)
}

func getConfigPath(args map[string]interface{}) string {
	var configPath string
	if args["--config"] == nil {
		configPath = ""
	} else {
		configPath = args["--config"].(string)
	}

	var rc RC
	var err error

	if configPath == "" {
		rc, err = readRC()
		if err != nil {
			outError("Error reading rc file: %s", err)
			exit(1)
		}
		if rc.Config.Path == "" {
			outError("Config file not specified.")
			exit(1)
		}
	} else {
		// Save to RC file
		configPath, err = filepath.Abs(configPath)
		if err != nil {
			outError("Bad config path: %s", err)
			exit(1)
		}

		rc, err = saveRC(configPath)
		if err != nil {
			outError("Cannot save rc file: %s", err)
			exit(1)
		}
	}

	return rc.Config.Path
}

func getMapping(config Configuration, srcDirAbs string) map[string]string {
	mapping := make(map[string]string)

	if len(config.Mapping) == 0 {
		// install all the things
		outVerbose("Mapping is not specified - install all the things")
		dir, err := os.Open(srcDirAbs)
		if err != nil {
			outError("Error reading dotfiles source dir: %s", err)
			exit(1)
		}

		defer dir.Close()

		files, err := dir.Readdir(0)
		if err != nil {
			outError("Error reading dotfiles source dir: %s", err)
			exit(1)
		}

		for _, fileInfo := range files {
			mapping[fileInfo.Name()] = fileInfo.Name()
		}

		// filter excludes
		for _, exclude := range config.Files.Excludes {
			if _, ok := mapping[exclude]; ok {
				delete(mapping, exclude)
			}
		}
	} else {
		// install by mapping
		if len(config.Files.Excludes) > 0 {
			outWarn("Excludes in config make no sense when mapping is specified, omitting them.")
		}

		mapping = config.Mapping
	}

	return mapping
}

func installDotfile(src string, dest string, config Configuration, srcDirAbs string) {
	srcAbs := srcDirAbs + "/" + src
	destAbs := config.Directories.Destination + "/" + dest

	exists, err := isExists(srcAbs)
	if !exists {
		outWarn("Source file %s does not exist", srcAbs)
		return
	}

	if err != nil {
		outError("Error processing source file %s: %s", src, err)
		exit(1)
	}

	needSymlink, needBackup, err := processDest(srcAbs, destAbs)
	if err != nil {
		outError("Error processing destination file %s: %s", destAbs, err)
		exit(1)
	}

	if !needSymlink {
		return
	}

	if needBackup {
		err = backup(dest, destAbs, config.Directories.Backup)
		if err != nil {
			outError("Error backuping file %s: %s", destAbs, err)
			exit(1)
		}
	}

	err = setSymlink(srcAbs, destAbs)
	if err != nil {
		outError("Error creating symlink from %s to %s: %s", srcAbs, destAbs, err)
		exit(1)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	log "github.com/Sirupsen/logrus"
	flags "github.com/jessevdk/go-flags"
	_ "github.com/mattn/go-sqlite3"
	"math/rand"
	"net/http"
	"os"
	"path"
	"time"
)

const coreOSAppID = "{e96281a6-d1af-4bde-9a0a-97b76e56dc57}"

var db *userDB
var fileBE fileBackend

var opts struct {
	Hostname          string `short:"H" long:"hostname" description:"hostname advertised when using local file backend"`
	ListenAddr        string `short:"l" long:"listenaddr" default:"0.0.0.0" description:"address to listen on"`
	Port              int    `short:"P" long:"port" default:"8080" description:"port to listen on"`
	DisableTimestamps bool   `short:"t" long:"disabletimestamps" description:"disable timestamps in logs (useful when using journald)"`
}

func main() {
	var err error
	flags.Parse(&opts)
	if opts.Hostname == "" {
		log.Error("You must set the 'hostname' parameter when using local file backend.")
		os.Exit(1)
	}

	log.SetFormatter(&log.TextFormatter{DisableTimestamp: opts.DisableTimestamps})
	log.SetLevel(log.DebugLevel)
	log.Info("COmaha update server starting")

	// seed the RNG
	rand.Seed(time.Now().UnixNano())

	// open db
	db, err = newUserDB("users.sqlite")
	if err != nil {
		log.Errorf("Could not open database: %v", err.Error())
		os.Exit(1)
	}
	defer db.Close()

	cwd, err := os.Getwd()
	if err != nil {
		log.Errorf("Could not cwd: %v", err.Error())
	}
	lfbe := newLocalFileBackend(path.Join(cwd, "storage"))
	fileBE = &lfbe

	http.HandleFunc("/file", fileHandler)
	http.HandleFunc("/update", updateHandler)
	http.HandleFunc("/shutdown", shutdownHandler)
	//http.HandleFunc("/admin/add_group", addGroupHandler)
	http.HandleFunc("/admin/add_payload", addPayloadHandler)
	http.HandleFunc("/panel", panelHandler)
	//http.HandleFunc("/admin/add_user", addUserHandler)
	http.HandleFunc("/", homeHandler)

	listenString := fmt.Sprintf("%v:%v", opts.ListenAddr, opts.Port)
	http.ListenAndServe(listenString, nil)
}
<commit_msg>debug enableable by a flag (#5)<commit_after>package main

import (
	"fmt"
	log "github.com/Sirupsen/logrus"
	flags "github.com/jessevdk/go-flags"
	_ "github.com/mattn/go-sqlite3"
	"math/rand"
	"net/http"
	"os"
	"path"
	"time"
)

const coreOSAppID = "{e96281a6-d1af-4bde-9a0a-97b76e56dc57}"

var db *userDB
var fileBE fileBackend

var opts struct {
	Hostname          string `short:"H" long:"hostname" description:"hostname advertised when using local file backend"`
	ListenAddr        string `short:"l" long:"listenaddr" default:"0.0.0.0" description:"address to listen on"`
	Port              int    `short:"P" long:"port" default:"8080" description:"port to listen on"`
	DisableTimestamps bool   `short:"t" long:"disabletimestamps" description:"disable timestamps in logs (useful when using journald)"`
	Debug             bool   `short:"d" long:"debug" description:"run in debug mode"`
}

func main() {
	var err error
	flags.Parse(&opts)
	if opts.Hostname == "" {
		log.Error("You must set the 'hostname' parameter when using local file backend.")
		os.Exit(1)
	}

	log.SetFormatter(&log.TextFormatter{DisableTimestamp: opts.DisableTimestamps})
	if opts.Debug {
		log.SetLevel(log.DebugLevel)
	} else {
		log.SetLevel(log.InfoLevel)
	}
	log.Info("COmaha update server starting")

	// seed the RNG
	rand.Seed(time.Now().UnixNano())

	// open db
	db, err = newUserDB("users.sqlite")
	if err != nil {
		log.Errorf("Could not open database: %v", err.Error())
		os.Exit(1)
	}
	defer db.Close()

	cwd, err := os.Getwd()
	if err != nil {
		log.Errorf("Could not cwd: %v", err.Error())
	}
	lfbe := newLocalFileBackend(path.Join(cwd, "storage"))
	fileBE = &lfbe

	http.HandleFunc("/file", fileHandler)
	http.HandleFunc("/update", updateHandler)
	http.HandleFunc("/shutdown", shutdownHandler)
	//http.HandleFunc("/admin/add_group", addGroupHandler)
	http.HandleFunc("/admin/add_payload", addPayloadHandler)
	http.HandleFunc("/panel", panelHandler)
	//http.HandleFunc("/admin/add_user", addUserHandler)
	http.HandleFunc("/", homeHandler)

	listenString := fmt.Sprintf("%v:%v", opts.ListenAddr, opts.Port)
	http.ListenAndServe(listenString, nil)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"log"
	"net/http"
	"time"
)

var Seed int64

func main() {
	flag.Int64Var(&Seed, "seed", time.Now().UnixNano(), "the world seed (default: the number of nanoseconds since midnight UTC on 1970-01-01)")

	flag.Parse()

	go func() {
		for _ = range time.Tick(time.Minute) {
			EachLoadedZone(func(z *Zone) {
				z.Save()
			})
		}
	}()

	go func() {
		for _ = range time.Tick(200 * time.Millisecond) {
			EachLoadedZone(func(z *Zone) {
				z.Think()
			})
		}
	}()

	for {
		log.Print(http.ListenAndServe(":2064", nil))
		time.Sleep(time.Second)
	}
}

type InteractHUD struct {
	Player       *Player
	TileX, TileY uint8
	Objects      []Object
	Offset       int
}

func (h *InteractHUD) Paint(setcell func(int, int, rune, Color)) {
	if h.Player.TileX != h.TileX || h.Player.TileY != h.TileY || h.Objects == nil {
		h.TileX, h.TileY = h.Player.TileX, h.Player.TileY
		minX := h.TileX - 1
		if minX == 255 {
			minX = 0
		}
		maxX := h.TileX + 1
		if maxX == 0 {
			maxX = 255
		}
		minY := h.TileY - 1
		if minY == 255 {
			minY = 0
		}
		maxY := h.TileY + 1
		if maxY == 0 {
			maxY = 255
		}
		z := GrabZone(h.Player.ZoneX, h.Player.ZoneY)
		z.Lock()
		var objects []Object
		for x := minX; x >= minX && x <= maxX; x++ {
			for y := minY; y >= minY && y <= maxY; y++ {
				objects = append(objects, z.Tile(x, y).Objects...)
			}
		}
		z.Unlock()
		ReleaseZone(z)
		h.Objects = objects
		h.Offset = 0
	}
	const keys = "12345678"
	for i, o := range h.Objects[h.Offset:] {
		if i >= len(keys) {
			break
		}
		setcell(0, i, rune(keys[i]), "#fff")
		setcell(1, i, ' ', "#fff")
		j := 1
		for _, r := range o.Name() {
			j++
			setcell(j, i, r, "#fff")
		}
	}
	if h.Offset > 0 {
		setcell(0, 8, '9', "#fff")
		setcell(1, 8, ' ', "#fff")
		j := 1
		for _, r := range "previous" {
			j++
			setcell(j, 8, r, "#fff")
		}
	}
	if len(h.Objects) > h.Offset+len(keys) {
		setcell(0, 9, '0', "#fff")
		setcell(1, 9, ' ', "#fff")
		j := 1
		for _, r := range "next" {
			j++
			setcell(j, 9, r, "#fff")
		}
	}
}

func (h *InteractHUD) Key(code int) bool {
	switch code {
	case '1', '2', '3', '4', '5', '6', '7', '8':
		// TODO
		return true
	case '9':
		if h.Offset > 0 {
			h.Offset--
			h.Player.Repaint()
		}
		return true
	case '0':
		if h.Offset+8 < len(h.Objects) {
			h.Offset++
			h.Player.Repaint()
		}
		return true

	case 27: // esc
		h.Player.hud = nil
		h.Player.Repaint()
		return true
	}
	return false
}
<commit_msg>examine<commit_after>package main

import (
	"flag"
	"log"
	"net/http"
	"time"
	"unicode"
)

var Seed int64

func main() {
	flag.Int64Var(&Seed, "seed", time.Now().UnixNano(), "the world seed (default: the number of nanoseconds since midnight UTC on 1970-01-01)")

	flag.Parse()

	go func() {
		for _ = range time.Tick(time.Minute) {
			EachLoadedZone(func(z *Zone) {
				z.Save()
			})
		}
	}()

	go func() {
		for _ = range time.Tick(200 * time.Millisecond) {
			EachLoadedZone(func(z *Zone) {
				z.Think()
			})
		}
	}()

	for {
		log.Print(http.ListenAndServe(":2064", nil))
		time.Sleep(time.Second)
	}
}

type ExamineHUD struct {
	Player *Player
	Object Object
}

func (h *ExamineHUD) Paint(setcell func(int, int, rune, Color)) {
	i := 0
	for _, r := range h.Object.Name() {
		setcell(i, 0, unicode.ToUpper(r), "#fff")
		i++
	}
	i = 0
	for _, r := range h.Object.Examine() {
		setcell(i, 1, r, "#fff")
		i++
	}
}

func (h *ExamineHUD) Key(code int) bool {
	switch code {
	case 27: // esc
		h.Player.hud = nil
		h.Player.Repaint()
		return true
	}
	return false
}

type InteractHUD struct {
	Player       *Player
	TileX, TileY uint8
	Objects      []Object
	Offset       int
}

func (h *InteractHUD) Paint(setcell func(int, int, rune, Color)) {
	if h.Player.TileX != h.TileX || h.Player.TileY != h.TileY || h.Objects == nil {
		h.TileX, h.TileY = h.Player.TileX, h.Player.TileY
		minX := h.TileX - 1
		if minX == 255 {
			minX = 0
		}
		maxX := h.TileX + 1
		if maxX == 0 {
			maxX = 255
		}
		minY := h.TileY - 1
		if minY == 255 {
			minY = 0
		}
		maxY := h.TileY + 1
		if maxY == 0 {
			maxY = 255
		}
		z := GrabZone(h.Player.ZoneX, h.Player.ZoneY)
		z.Lock()
		var objects []Object
		for x := minX; x >= minX && x <= maxX; x++ {
			for y := minY; y >= minY && y <= maxY; y++ {
				objects = append(objects, z.Tile(x, y).Objects...)
			}
		}
		z.Unlock()
		ReleaseZone(z)
		h.Objects = objects
		h.Offset = 0
	}
	for i, r := range "EXAMINE" {
		setcell(i, 0, r, "#fff")
	}
	const keys = "12345678"
	for i, o := range h.Objects[h.Offset:] {
		if i >= len(keys) {
			break
		}
		setcell(0, i+1, rune(keys[i]), "#fff")
		setcell(1, i+1, ' ', "#fff")
		j := 2
		for _, r := range o.Name() {
			setcell(j, i+1, r, "#fff")
			j++
		}
	}
	if h.Offset > 0 {
		setcell(0, 9, '9', "#fff")
		setcell(1, 9, ' ', "#fff")
		j := 1
		for _, r := range "previous" {
			j++
			setcell(j, 9, r, "#fff")
		}
	}
	if len(h.Objects) > h.Offset+len(keys) {
		setcell(0, 10, '0', "#fff")
		setcell(1, 10, ' ', "#fff")
		j := 2
		for _, r := range "next" {
			setcell(j, 10, r, "#fff")
			j++
		}
	}
}

func (h *InteractHUD) Key(code int) bool {
	switch code {
	case '1', '2', '3', '4', '5', '6', '7', '8':
		i := code - '1' + h.Offset
		if i < len(h.Objects) {
			h.Player.hud = &ExamineHUD{
				Player: h.Player,
				Object: h.Objects[i],
			}
			h.Player.Repaint()
		}
		return true
	case '9':
		if h.Offset > 0 {
			h.Offset--
			h.Player.Repaint()
		}
		return true
	case '0':
		if h.Offset+8 < len(h.Objects) {
			h.Offset++
			h.Player.Repaint()
		}
		return true

	case 27: // esc
		h.Player.hud = nil
		h.Player.Repaint()
		return true
	}
	return false
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"path"
	"time"

	"github.com/gorilla/mux"
	"github.com/voxelbrain/goptions"

	"labix.org/v2/mgo"
)

var (
	options = struct {
		Port              int           `goptions:"-p, --port, description='Port to bind webserver to'"`
		MongoDB           string        `goptions:"-m, --mongodb, description='URL of MongoDB', obligatory"`
		StaticContent     string        `goptions:"--static, description='Path to static content folder'"`
		SummonerWhitelist string        `goptions:"--whitelist, description='List of whitelisted summoner IDs separated by colon'"`
		Help              goptions.Help `goptions:"-h, --help, description='Show this help'"`
	}{
		Port:          5000,
		StaticContent: "static",
	}
)

type Match struct {
	GameType string     `json:"game_type" lolkaiser:"game_type"`
	Date     time.Time  `json:"timestamp" lolkaiser:"timestamp"`
	Win      bool       `json:"win" lolkaiser:"win"`
	Length   int        `json:"length" lolkaiser:"length"`
	Teams    [][]Player `json:"teams" lolkaiser:"teams"`

	Champion         string `json:"champion" lolkaiser:"champion"`
	KDA              []int  `json:"kda" lolkaiser:"kda"`
	Gold             int    `json:"gold" lolkaiser:"gold"`
	Minions          int    `json:"minions" lolkaiser:"minions"`
	LargestMultikill int    `json:"largest_multikill" lolkaiser:"largest_multikill"`
	TimeDead         int    `json:"time_dead" lolkaiser:"time_dead"`
}

type Player struct {
	Champion     string `json:"champion"`
	SummonerName string `json:"summoner_name"`
}

func main() {
	goptions.ParseAndFail(&options)

	session, err := mgo.Dial(options.MongoDB)
	if err != nil {
		log.Fatalf("Could not connect to MongoDB: %s", err)
	}
	db := session.DB("")
	_ = db

	r := mux.NewRouter()
	r.HandleFunc("/{server:euw|na}/{id}", func(w http.ResponseWriter, r *http.Request) {
		vars := mux.Vars(r)
		mh, err := LolKingMatchHistory(path.Join(vars["server"], vars["id"]))
		if err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}
		json.NewEncoder(w).Encode(mh)
	})
	r.PathPrefix("/").Handler(http.FileServer(http.Dir(options.StaticContent)))

	addr := fmt.Sprintf("0.0.0.0:%d", options.Port)
	log.Printf("Starting webserver on %s...", addr)
	if err := http.ListenAndServe(addr, r); err != nil {
		log.Fatalf("Could not start webserver: %s", err)
	}
}
<commit_msg>Add summoner whitelist<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"path"
	"strings"
	"time"

	"github.com/voxelbrain/goptions"
	"gopkg.in/surma/v1.2.1/httptools"

	"labix.org/v2/mgo"
)

var (
	options = struct {
		Port              int           `goptions:"-p, --port, description='Port to bind webserver to'"`
		MongoDB           string        `goptions:"-m, --mongodb, description='URL of MongoDB', obligatory"`
		StaticContent     string        `goptions:"--static, description='Path to static content folder'"`
		SummonerWhitelist string        `goptions:"--whitelist, description='List of whitelisted summoner IDs separated by colon'"`
		Help              goptions.Help `goptions:"-h, --help, description='Show this help'"`
	}{
		Port:          5000,
		StaticContent: "static",
	}
)

type Match struct {
	GameType string     `json:"game_type" lolkaiser:"game_type"`
	Date     time.Time  `json:"timestamp" lolkaiser:"timestamp"`
	Win      bool       `json:"win" lolkaiser:"win"`
	Length   int        `json:"length" lolkaiser:"length"`
	Teams    [][]Player `json:"teams" lolkaiser:"teams"`

	Champion         string `json:"champion" lolkaiser:"champion"`
	KDA              []int  `json:"kda" lolkaiser:"kda"`
	Gold             int    `json:"gold" lolkaiser:"gold"`
	Minions          int    `json:"minions" lolkaiser:"minions"`
	LargestMultikill int    `json:"largest_multikill" lolkaiser:"largest_multikill"`
	TimeDead         int    `json:"time_dead" lolkaiser:"time_dead"`
}

type Player struct {
	Champion     string `json:"champion"`
	SummonerName string `json:"summoner_name"`
}

func main() {
	goptions.ParseAndFail(&options)

	session, err := mgo.Dial(options.MongoDB)
	if err != nil {
		log.Fatalf("Could not connect to MongoDB: %s", err)
	}
	db := session.DB("")
	_ = db

	r := httptools.NewRegexpSwitch(map[string]http.Handler{
		"/update/(euw|na)/([0-9]+)": http.HandlerFunc(updateCollectionHandler),
		"/.+": http.FileServer(http.Dir(options.StaticContent)),
	})

	addr := fmt.Sprintf("0.0.0.0:%d", options.Port)
	log.Printf("Starting webserver on %s...", addr)
	if err := http.ListenAndServe(addr, r); err != nil {
		log.Fatalf("Could not start webserver: %s", err)
	}
}

func updateCollectionHandler(w http.ResponseWriter, r *http.Request) {
	vars := w.(httptools.VarsResponseWriter).Vars()
	server, summonderId := vars["1"].(string), vars["2"].(string)

	if !StringArray(strings.Split(options.SummonerWhitelist, ":")).Contains(server + "/" + summonderId) {
		http.Error(w, "Forbidden", http.StatusForbidden)
		return
	}

	mh, err := LolKingMatchHistory(path.Join(server, summonderId))
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	json.NewEncoder(w).Encode(mh)
}

type StringArray []string

func (sa StringArray) Contains(s string) bool {
	for _, v := range sa {
		if v == s {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"sync"
	"time"

	auth "github.com/abbot/go-http-auth"
	"github.com/buchgr/bazel-remote/cache"
	"github.com/buchgr/bazel-remote/cache/disk"
	"github.com/buchgr/bazel-remote/cache/gcs"
	"github.com/buchgr/bazel-remote/cache/s3"

	cachehttp "github.com/buchgr/bazel-remote/cache/http"

	"github.com/buchgr/bazel-remote/config"
	"github.com/buchgr/bazel-remote/server"
	"github.com/urfave/cli"

	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
)

const (
	logFlags = log.Ldate | log.Ltime | log.LUTC
)

func main() {

	log.SetFlags(logFlags)

	app := cli.NewApp()
	app.Description = "A remote build cache for Bazel."
	app.Usage = "A remote build cache for Bazel"
	app.HideVersion = true

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "config_file",
			Value: "",
			Usage: "Path to a YAML configuration file. If this flag is specified then all other flags " +
				"are ignored.",
			EnvVar: "BAZEL_REMOTE_CONFIG_FILE",
		},
		cli.StringFlag{
			Name:   "dir",
			Value:  "",
			Usage:  "Directory path where to store the cache contents. This flag is required.",
			EnvVar: "BAZEL_REMOTE_DIR",
		},
		cli.Int64Flag{
			Name:   "max_size",
			Value:  -1,
			Usage:  "The maximum size of the remote cache in GiB. This flag is required.",
			EnvVar: "BAZEL_REMOTE_MAX_SIZE",
		},
		cli.StringFlag{
			Name:   "host",
			Value:  "",
			Usage:  "Address to listen on. Listens on all network interfaces by default.",
			EnvVar: "BAZEL_REMOTE_HOST",
		},
		cli.IntFlag{
			Name:   "port",
			Value:  8080,
			Usage:  "The port the HTTP server listens on.",
			EnvVar: "BAZEL_REMOTE_PORT",
		},
		cli.IntFlag{
			Name:   "grpc_port",
			Value:  9092,
			Usage:  "The port the EXPERIMENTAL gRPC server listens on. Set to 0 to disable.",
			EnvVar: "BAZEL_REMOTE_GRPC_PORT",
		},
		cli.StringFlag{
			Name:   "htpasswd_file",
			Value:  "",
			Usage:  "Path to a .htpasswd file. This flag is optional. Please read https://httpd.apache.org/docs/2.4/programs/htpasswd.html.",
			EnvVar: "BAZEL_REMOTE_HTPASSWD_FILE",
		},
		cli.BoolFlag{
			Name:   "tls_enabled",
			Usage:  "This flag has been deprecated. Specify tls_cert_file and tls_key_file instead.",
			EnvVar: "BAZEL_REMOTE_TLS_ENABLED",
		},
		cli.StringFlag{
			Name:   "tls_cert_file",
			Value:  "",
			Usage:  "Path to a pem encoded certificate file.",
			EnvVar: "BAZEL_REMOTE_TLS_CERT_FILE",
		},
		cli.StringFlag{
			Name:   "tls_key_file",
			Value:  "",
			Usage:  "Path to a pem encoded key file.",
			EnvVar: "BAZEL_REMOTE_TLS_KEY_FILE",
		},
		cli.DurationFlag{
			Name:   "idle_timeout",
			Value:  0,
			Usage:  "The maximum period of having received no request after which the server will shut itself down. Disabled by default.",
			EnvVar: "BAZEL_REMOTE_IDLE_TIMEOUT",
		},
		cli.StringFlag{
			Name:   "s3.endpoint",
			Value:  "",
			Usage:  "The S3/minio endpoint to use when using S3 cache backend.",
			EnvVar: "BAZEL_REMOTE_S3_ENDPOINT",
		},
		cli.StringFlag{
			Name:   "s3.bucket",
			Value:  "",
			Usage:  "The S3/minio bucket to use when using S3 cache backend.",
			EnvVar: "BAZEL_REMOTE_S3_BUCKET",
		},
		cli.StringFlag{
			Name:   "s3.prefix",
			Value:  "",
			Usage:  "The S3/minio object prefix to use when using S3 cache backend.",
			EnvVar: "BAZEL_REMOTE_S3_PREFIX",
		},
		cli.StringFlag{
			Name:   "s3.access_key_id",
			Value:  "",
			Usage:  "The S3/minio access key to use when using S3 cache backend.",
			EnvVar: "BAZEL_REMOTE_S3_ACCESS_KEY_ID",
		},
		cli.StringFlag{
			Name:   "s3.secret_access_key",
			Value:  "",
			Usage:  "The S3/minio secret access key to use when using S3 cache backend.",
			EnvVar: "BAZEL_REMOTE_S3_SECRET_ACCESS_KEY",
		},
		cli.BoolFlag{
			Name:   "s3.disable_ssl",
			Usage:  "Whether to disable TLS/SSL when using the S3 cache backend.  Default is false (enable TLS/SSL).",
			EnvVar: "BAZEL_REMOTE_S3_DISABLE_SSL",
		},
		cli.BoolFlag{
			Name:   "disable_http_ac_validation",
			Usage:  "Whether to disable ActionResult validation for HTTP requests.  Default is false (enable validation).",
			EnvVar: "BAZEL_REMOTE_DISABLE_HTTP_AC_VALIDATION",
		},
	}

	app.Action = func(ctx *cli.Context) error {
		configFile := ctx.String("config_file")
		var c *config.Config
		var err error
		if configFile != "" {
			c, err = config.NewFromYamlFile(configFile)
		} else {
			var s3 *config.S3CloudStorageConfig
			if ctx.String("s3.bucket") != "" {
				s3 = &config.S3CloudStorageConfig{
					Endpoint:        ctx.String("s3.endpoint"),
					Bucket:          ctx.String("s3.bucket"),
					Prefix:          ctx.String("s3.prefix"),
					AccessKeyID:     ctx.String("s3.access_key_id"),
					SecretAccessKey: ctx.String("s3.secret_access_key"),
					DisableSSL:      ctx.Bool("s3.disable_ssl"),
				}
			}
			c, err = config.New(
				ctx.String("dir"),
				ctx.Int("max_size"),
				ctx.String("host"),
				ctx.Int("port"),
				ctx.Int("grpc_port"),
				ctx.String("htpasswd_file"),
				ctx.String("tls_cert_file"),
				ctx.String("tls_key_file"),
				ctx.Duration("idle_timeout"),
				s3,
				ctx.Bool("disable_http_ac_validation"),
			)
		}

		if err != nil {
			fmt.Fprintf(ctx.App.Writer, "%v\n\n", err)
			cli.ShowAppHelp(ctx)
			return nil
		}

		accessLogger := log.New(os.Stdout, "", logFlags)
		errorLogger := log.New(os.Stderr, "", logFlags)

		diskCache := disk.New(c.Dir, int64(c.MaxSize)*1024*1024*1024)

		var proxyCache cache.Cache
		if c.GoogleCloudStorage != nil {
			proxyCache, err = gcs.New(c.GoogleCloudStorage.Bucket,
				c.GoogleCloudStorage.UseDefaultCredentials, c.GoogleCloudStorage.JSONCredentialsFile,
				diskCache, accessLogger, errorLogger)
			if err != nil {
				log.Fatal(err)
			}
		} else if c.HTTPBackend != nil {
			httpClient := &http.Client{}
			baseURL, err := url.Parse(c.HTTPBackend.BaseURL)
			if err != nil {
				log.Fatal(err)
			}
			proxyCache = cachehttp.New(baseURL, diskCache,
				httpClient, accessLogger, errorLogger)
		} else if c.S3CloudStorage != nil {
			proxyCache = s3.New(c.S3CloudStorage, diskCache, accessLogger, errorLogger)
		} else {
			proxyCache = diskCache
		}

		mux := http.NewServeMux()
		httpServer := &http.Server{
			Addr:    c.Host + ":" + strconv.Itoa(c.Port),
			Handler: mux,
		}
		validateAC := !c.DisableHTTPACValidation
		h := server.NewHTTPCache(proxyCache, accessLogger, errorLogger, validateAC)
		mux.HandleFunc("/status", h.StatusPageHandler)

		cacheHandler := h.CacheHandler
		if c.HtpasswdFile != "" {
			cacheHandler = wrapAuthHandler(cacheHandler, c.HtpasswdFile, c.Host)
		}
		if c.IdleTimeout > 0 {
			cacheHandler = wrapIdleHandler(cacheHandler, c.IdleTimeout, accessLogger, httpServer)
		}
		mux.HandleFunc("/", cacheHandler)

		if c.GRPCPort > 0 {
			go func() {
				addr := c.Host + ":" + strconv.Itoa(c.GRPCPort)

				opts := []grpc.ServerOption{}

				if len(c.TLSCertFile) > 0 && len(c.TLSKeyFile) > 0 {
					creds, err := credentials.NewServerTLSFromFile(
						c.TLSCertFile, c.TLSKeyFile)
					if err != nil {
						log.Fatal(err)
					}
					opts = append(opts, grpc.Creds(creds))
				}

				err = server.ListenAndServeGRPC(addr, opts,
					proxyCache, accessLogger, errorLogger)
				if err != nil {
					log.Fatal(err)
				}
			}()
		}

		if len(c.TLSCertFile) > 0 && len(c.TLSKeyFile) > 0 {
			return httpServer.ListenAndServeTLS(c.TLSCertFile, c.TLSKeyFile)
		}
		return httpServer.ListenAndServe()
	}

	serverErr := app.Run(os.Args)
	if serverErr != nil {
		log.Fatal("bazel-remote terminated: ", serverErr)
	}
}

func wrapIdleHandler(handler http.HandlerFunc, idleTimeout time.Duration, accessLogger cache.Logger, httpServer *http.Server) http.HandlerFunc {
	lastRequest := time.Now()
	ticker := time.NewTicker(time.Second)
	var m sync.Mutex
	go func() {
		for {
			select {
			case now := <-ticker.C:
				m.Lock()
				elapsed := now.Sub(lastRequest)
				m.Unlock()
				if elapsed > idleTimeout {
					ticker.Stop()
					accessLogger.Printf("Shutting down server after having been idle for %v", idleTimeout)
					httpServer.Shutdown(context.Background())
				}
			}
		}
	}()
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		now := time.Now()
		m.Lock()
		lastRequest = now
		m.Unlock()
		handler(w, r)
	})
}

func wrapAuthHandler(handler http.HandlerFunc, htpasswdFile string, host string) http.HandlerFunc {
	secrets := auth.HtpasswdFileProvider(htpasswdFile)
	authenticator := auth.NewBasicAuthenticator(host, secrets)
	return auth.JustCheck(authenticator, handler)
}
<commit_msg>log HTTP and gRPC ports on startup, and check for conflicts (#115)<commit_after>package main

import (
	"context"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"sync"
	"time"

	auth "github.com/abbot/go-http-auth"
	"github.com/buchgr/bazel-remote/cache"
	"github.com/buchgr/bazel-remote/cache/disk"
	"github.com/buchgr/bazel-remote/cache/gcs"
	"github.com/buchgr/bazel-remote/cache/s3"

	cachehttp "github.com/buchgr/bazel-remote/cache/http"

	"github.com/buchgr/bazel-remote/config"
	"github.com/buchgr/bazel-remote/server"
	"github.com/urfave/cli"

	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
)

const (
	logFlags = log.Ldate | log.Ltime | log.LUTC
)

func main() {

	log.SetFlags(logFlags)

	app := cli.NewApp()
	app.Description = "A remote build cache for Bazel."
	app.Usage = "A remote build cache for Bazel"
	app.HideVersion = true

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "config_file",
			Value: "",
			Usage: "Path to a YAML configuration file. If this flag is specified then all other flags " +
				"are ignored.",
			EnvVar: "BAZEL_REMOTE_CONFIG_FILE",
		},
		cli.StringFlag{
			Name:   "dir",
			Value:  "",
			Usage:  "Directory path where to store the cache contents. This flag is required.",
			EnvVar: "BAZEL_REMOTE_DIR",
		},
		cli.Int64Flag{
			Name:   "max_size",
			Value:  -1,
			Usage:  "The maximum size of the remote cache in GiB. This flag is required.",
			EnvVar: "BAZEL_REMOTE_MAX_SIZE",
		},
		cli.StringFlag{
			Name:   "host",
			Value:  "",
			Usage:  "Address to listen on. Listens on all network interfaces by default.",
			EnvVar: "BAZEL_REMOTE_HOST",
		},
		cli.IntFlag{
			Name:   "port",
			Value:  8080,
			Usage:  "The port the HTTP server listens on.",
			EnvVar: "BAZEL_REMOTE_PORT",
		},
		cli.IntFlag{
			Name:   "grpc_port",
			Value:  9092,
			Usage:  "The port the EXPERIMENTAL gRPC server listens on. Set to 0 to disable.",
			EnvVar: "BAZEL_REMOTE_GRPC_PORT",
		},
		cli.StringFlag{
			Name:   "htpasswd_file",
			Value:  "",
			Usage:  "Path to a .htpasswd file. This flag is optional. Please read https://httpd.apache.org/docs/2.4/programs/htpasswd.html.",
			EnvVar: "BAZEL_REMOTE_HTPASSWD_FILE",
		},
		cli.BoolFlag{
			Name:   "tls_enabled",
			Usage:  "This flag has been deprecated. Specify tls_cert_file and tls_key_file instead.",
			EnvVar: "BAZEL_REMOTE_TLS_ENABLED",
		},
		cli.StringFlag{
			Name:   "tls_cert_file",
			Value:  "",
			Usage:  "Path to a pem encoded certificate file.",
			EnvVar: "BAZEL_REMOTE_TLS_CERT_FILE",
		},
		cli.StringFlag{
			Name:   "tls_key_file",
			Value:  "",
			Usage:  "Path to a pem encoded key file.",
			EnvVar: "BAZEL_REMOTE_TLS_KEY_FILE",
		},
		cli.DurationFlag{
			Name:   "idle_timeout",
			Value:  0,
			Usage:  "The maximum period of having received no request after which the server will shut itself down. Disabled by default.",
			EnvVar: "BAZEL_REMOTE_IDLE_TIMEOUT",
		},
		cli.StringFlag{
			Name:   "s3.endpoint",
			Value:  "",
			Usage:  "The S3/minio endpoint to use when using S3 cache backend.",
			EnvVar: "BAZEL_REMOTE_S3_ENDPOINT",
		},
		cli.StringFlag{
			Name:   "s3.bucket",
			Value:  "",
			Usage:  "The S3/minio bucket to use when using S3 cache backend.",
			EnvVar: "BAZEL_REMOTE_S3_BUCKET",
		},
		cli.StringFlag{
			Name:   "s3.prefix",
			Value:  "",
			Usage:  "The S3/minio object prefix to use when using S3 cache backend.",
			EnvVar: "BAZEL_REMOTE_S3_PREFIX",
		},
		cli.StringFlag{
			Name:   "s3.access_key_id",
			Value:  "",
			Usage:  "The S3/minio access key to use when using S3 cache backend.",
			EnvVar: "BAZEL_REMOTE_S3_ACCESS_KEY_ID",
		},
		cli.StringFlag{
			Name:   "s3.secret_access_key",
			Value:  "",
			Usage:  "The S3/minio secret access key to use when using S3 cache backend.",
			EnvVar: "BAZEL_REMOTE_S3_SECRET_ACCESS_KEY",
		},
		cli.BoolFlag{
			Name:   "s3.disable_ssl",
			Usage:  "Whether to disable TLS/SSL when using the S3 cache backend.  Default is false (enable TLS/SSL).",
			EnvVar: "BAZEL_REMOTE_S3_DISABLE_SSL",
		},
		cli.BoolFlag{
			Name:   "disable_http_ac_validation",
			Usage:  "Whether to disable ActionResult validation for HTTP requests.  Default is false (enable validation).",
			EnvVar: "BAZEL_REMOTE_DISABLE_HTTP_AC_VALIDATION",
		},
	}

	app.Action = func(ctx *cli.Context) error {
		configFile := ctx.String("config_file")
		var c *config.Config
		var err error
		if configFile != "" {
			c, err = config.NewFromYamlFile(configFile)
		} else {
			var s3 *config.S3CloudStorageConfig
			if ctx.String("s3.bucket") != "" {
				s3 = &config.S3CloudStorageConfig{
					Endpoint:        ctx.String("s3.endpoint"),
					Bucket:          ctx.String("s3.bucket"),
					Prefix:          ctx.String("s3.prefix"),
					AccessKeyID:     ctx.String("s3.access_key_id"),
					SecretAccessKey: ctx.String("s3.secret_access_key"),
					DisableSSL:      ctx.Bool("s3.disable_ssl"),
				}
			}
			c, err = config.New(
				ctx.String("dir"),
				ctx.Int("max_size"),
				ctx.String("host"),
				ctx.Int("port"),
				ctx.Int("grpc_port"),
				ctx.String("htpasswd_file"),
				ctx.String("tls_cert_file"),
				ctx.String("tls_key_file"),
				ctx.Duration("idle_timeout"),
				s3,
				ctx.Bool("disable_http_ac_validation"),
			)
		}

		if err != nil {
			fmt.Fprintf(ctx.App.Writer, "%v\n\n", err)
			cli.ShowAppHelp(ctx)
			return nil
		}

		accessLogger := log.New(os.Stdout, "", logFlags)
		errorLogger := log.New(os.Stderr, "", logFlags)

		diskCache := disk.New(c.Dir, int64(c.MaxSize)*1024*1024*1024)

		var proxyCache cache.Cache
		if c.GoogleCloudStorage != nil {
			proxyCache, err = gcs.New(c.GoogleCloudStorage.Bucket,
				c.GoogleCloudStorage.UseDefaultCredentials, c.GoogleCloudStorage.JSONCredentialsFile,
				diskCache, accessLogger, errorLogger)
			if err != nil {
				log.Fatal(err)
			}
		} else if c.HTTPBackend != nil {
			httpClient := &http.Client{}
			baseURL, err := url.Parse(c.HTTPBackend.BaseURL)
			if err != nil {
				log.Fatal(err)
			}
			proxyCache = cachehttp.New(baseURL, diskCache,
				httpClient, accessLogger, errorLogger)
		} else if c.S3CloudStorage != nil {
			proxyCache = s3.New(c.S3CloudStorage, diskCache, accessLogger, errorLogger)
		} else {
			proxyCache = diskCache
		}

		mux := http.NewServeMux()
		httpServer := &http.Server{
			Addr:    c.Host + ":" + strconv.Itoa(c.Port),
			Handler: mux,
		}
		validateAC := !c.DisableHTTPACValidation
		h := server.NewHTTPCache(proxyCache, accessLogger, errorLogger, validateAC)
		mux.HandleFunc("/status", h.StatusPageHandler)

		cacheHandler := h.CacheHandler
		if c.HtpasswdFile != "" {
			cacheHandler = wrapAuthHandler(cacheHandler, c.HtpasswdFile, c.Host)
		}
		if c.IdleTimeout > 0 {
			cacheHandler = wrapIdleHandler(cacheHandler, c.IdleTimeout, accessLogger, httpServer)
		}
		mux.HandleFunc("/", cacheHandler)

		if c.GRPCPort > 0 {

			if c.GRPCPort == c.Port {
				log.Fatalf("Error: gRPC and HTTP ports (%d) conflict", c.Port)
			}

			go func() {
				addr := c.Host + ":" + strconv.Itoa(c.GRPCPort)

				opts := []grpc.ServerOption{}

				if len(c.TLSCertFile) > 0 && len(c.TLSKeyFile) > 0 {
					creds, err := credentials.NewServerTLSFromFile(
						c.TLSCertFile, c.TLSKeyFile)
					if err != nil {
						log.Fatal(err)
					}
					opts = append(opts, grpc.Creds(creds))
				}

				log.Printf("Starting gRPC server on address %s", addr)

				err = server.ListenAndServeGRPC(addr, opts,
					proxyCache, accessLogger, errorLogger)
				if err != nil {
					log.Fatal(err)
				}
			}()
		}

		if len(c.TLSCertFile) > 0 && len(c.TLSKeyFile) > 0 {
			log.Printf("Starting HTTPS server on address %s", httpServer.Addr)
			return httpServer.ListenAndServeTLS(c.TLSCertFile, c.TLSKeyFile)
		}

		log.Printf("Starting HTTP server on address %s", httpServer.Addr)
		return httpServer.ListenAndServe()
	}

	serverErr := app.Run(os.Args)
	if serverErr != nil {
		log.Fatal("bazel-remote terminated: ", serverErr)
	}
}

func wrapIdleHandler(handler http.HandlerFunc, idleTimeout time.Duration, accessLogger cache.Logger, httpServer *http.Server) http.HandlerFunc {
	lastRequest := time.Now()
	ticker := time.NewTicker(time.Second)
	var m sync.Mutex
	go func() {
		for {
			select {
			case now := <-ticker.C:
				m.Lock()
				elapsed := now.Sub(lastRequest)
				m.Unlock()
				if elapsed > idleTimeout {
					ticker.Stop()
					accessLogger.Printf("Shutting down server after having been idle for %v", idleTimeout)
					httpServer.Shutdown(context.Background())
				}
			}
		}
	}()
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		now := time.Now()
		m.Lock()
		lastRequest = now
		m.Unlock()
		handler(w, r)
	})
}

func wrapAuthHandler(handler http.HandlerFunc, htpasswdFile string, host string) http.HandlerFunc {
	secrets := auth.HtpasswdFileProvider(htpasswdFile)
	authenticator := auth.NewBasicAuthenticator(host, secrets)
	return auth.JustCheck(authenticator, handler)
}
<|endoftext|>