<commit_before>// Filen for å eksperimentere med webserver i ICA03 (IS-105)
// Starte server med
// 			go run server.go
// Du kan ha tilgang til server fra nettleser med
// 			http://localhost:3000
//
package main

import (
	"net/http"
	"time"
	//"fmt"
)

func main() {
	http.HandleFunc("/", foo)
	http.ListenAndServe(":3000", nil)
}

func foo(w http.ResponseWriter, r *http.Request) {
	// Her kan man bl. a. skrive data som skal sendes til nettleser
	// Neste linje kan brukes for å endre koding av sekvensen som sendes til nettleser
	// Standardinstilling er UTF-8
	//w.Header().Set("Content-Type", "text/html;charset=ISO-8859-1")

	// Her skriver man data som er respons til brukeren som har skrevet
	// http://localhost:3000 i sin nettleser
	w.Write([]byte("Hvordan g\u00E5r det, \u16a6?
"))
	w.Write([]byte("\u23f0 - "))
	//fmt.Println(time.Now().Format(time.RFC850))
	w.Write([]byte(time.Now().Format(time.RFC850)))
}

//func TimeDate() {
//	fmt.Println(time.Now().Format(time.RFC850))
<commit_msg>liten endring i server.go<commit_after>// Filen for å eksperimentere med webserver i ICA03 (IS-105)
// Starte server med
// 			go run server.go
// Du kan ha tilgang til server fra nettleser med
// 			http://localhost:3000
//
package main

import (
	"net/http"
	"time"
	//"fmt"
)

func main() {
	http.HandleFunc("/", foo)
	http.ListenAndServe(":3000", nil)
}

func foo(w http.ResponseWriter, r *http.Request) {
	// Her kan man bl. a. skrive data som skal sendes til nettleser
	// Neste linje kan brukes for å endre koding av sekvensen som sendes til nettleser
	// Standardinstilling er UTF-8
	//w.Header().Set("Content-Type", "text/html;charset=ISO-8859-1")

	// Her skriver man data som er respons til brukeren som har skrevet
	// http://localhost:3000 i sin nettleser
	w.Write([]byte("Hvordan g\u00E5r det, ᚦ?
"))
	w.Write([]byte("\u23f0 - "))
	//fmt.Println(time.Now().Format(time.RFC850))
	w.Write([]byte(time.Now().Format(time.RFC850)))
}

//func TimeDate() {
//	fmt.Println(time.Now().Format(time.RFC850))
<|endoftext|>


<commit_before>package main

import (
	"crypto/md5"
	"encoding/hex"
	"fmt"
	"github.com/codegangsta/negroni"
	"github.com/fzzy/radix/extra/pool"
	"github.com/fzzy/radix/redis"
	"github.com/gorilla/mux"
	"github.com/unrolled/render"
	"net/http"
	"os"
	// "io"
	// "time"
	// "log"
	"strconv"
	"strings"
	// "regexp"
)

func CleanWords(query string) []string {
	// r := regexp.MustCompile(regexp.QuoteMeta("[]()"))
	// query = r.ReplaceAllString(query, " ")
	query = strings.Replace(query, "\"", " ", -1)
	query = strings.Replace(query, "[", " ", -1)
	query = strings.Replace(query, "]", " ", -1)
	query = strings.Replace(query, "(", " ", -1)
	query = strings.Replace(query, ")", " ", -1)
	query = strings.Replace(query, "?", " ", -1)
	query = strings.Replace(query, "!", " ", -1)
	query = strings.Replace(query, ",", " ", -1)
	query = strings.Replace(query, "-", " ", -1)
	query = strings.Replace(query, ":", " ", -1)
	query = strings.Replace(query, ";", " ", -1)
	query = strings.Replace(query, ",", " ", -1)
	query = strings.Replace(query, "'", " ", -1)
	split := strings.Fields(strings.Trim(query, " "))
	terms := make([]string, len(split))
	for i, term := range split {
		terms[i] = strings.ToLower(strings.Trim(strings.Trim(term, " "), "."))
	}
	return terms
}

func encodeString(str string) string {
	h := md5.New()
	h.Write([]byte(str))
	return hex.EncodeToString(h.Sum(nil))[0:8]
}

func getPrefixes(title string) []string {
	var prefixes []string
	for _, word := range CleanWords(title) {
		for i := 1; i <= len(word); i++ {
			prefixes = append(prefixes, word[0:i])
		}
	}
	return prefixes
}

// func QueryScore(terms []string, title) float32 {
// 	return 1.0
// }

func errHndlr(err error) {
	if err != nil {
		fmt.Println("error:", err)
		panic(err)
		os.Exit(1)
	}
}

func main() {
	is_debug := os.Getenv("DEBUG")
	// fmt.Println("debug", is_debug)
	var debug = true
	if is_debug == "" {
		debug = false
	}
	renderer := render.New(render.Options{
		IndentJSON:    debug,
		IsDevelopment: debug,
	})

	REDIS_URL := os.Getenv("REDIS_URL")
	if REDIS_URL == "" {
		REDIS_URL = "127.0.0.1:6379"
	}
	REDIS_DATABASE := os.Getenv("REDIS_DATABASE")
	if REDIS_DATABASE == "" {
		REDIS_DATABASE = "0"
	}

	df := func(network, addr string) (*redis.Client, error) {
		client, err := redis.Dial(network, addr)
		if err != nil {
			return nil, err
		}
		err = client.Cmd("SELECT", REDIS_DATABASE).Err
		if err != nil {
			return nil, err
		}
		// if err = client.Cmd("AUTH", "SUPERSECRET").Err; err != nil {
		// 	client.Close()
		// 	return nil, err
		// }
		return client, nil
	}

	pool, err := pool.NewCustomPool("tcp", REDIS_URL, 10, df)
	errHndlr(err)

	HomeHandler := func(w http.ResponseWriter, req *http.Request) {

		n_str := req.FormValue("n")

		var n int64
		if n_str == "" {
			n = 10
		} else {
			n, err = strconv.ParseInt(n_str, 10, 0)
			if err != nil {
				// error := make(map[string]string{"error": "Not a number"})
				error := map[string]string{"error": "Not a number"}
				renderer.JSON(w, http.StatusBadRequest, error)
				return
			}
			if n <= 0 {
				error := map[string]string{"error": "Number too small"}
				renderer.JSON(w, http.StatusBadRequest, error)
				return
			}
			if n > 100 {
				error := map[string]string{"error": "Number too big"}
				renderer.JSON(w, http.StatusBadRequest, error)
				return
			}
		}

		domain := strings.Trim(req.FormValue("domain"), " ")
		if domain == "" {
			error := map[string]string{"error": "'domain' missing"}
			renderer.JSON(w, http.StatusBadRequest, error)
			return
		}

		encoded := encodeString(domain)
		// fmt.Println(domain, encoded)

		q := strings.Trim(req.FormValue("q"), " ")
		terms := CleanWords(q)

		c, err := pool.Get()
		errHndlr(err)
		defer pool.Put(c)
		encoded_terms := make([]string, len(terms))
		for i, term := range terms {
			encoded_terms[i] = encoded + term
		}
		// NOTE! Maybe we don't need the ZINTERSTORE if there's only 1 command
		c.Append("ZINTERSTORE", "$tmp", len(terms), encoded_terms, "AGGREGATE", "max")
		c.Append("ZREVRANGE", "$tmp", 0, n-1, "WITHSCORES")

		c.GetReply() // the ZINTERSTORE
		replies, err := c.GetReply().List()
		// fmt.Println("replies", replies, len(replies))
		errHndlr(err)

		encoded_urls := make([]string, n+1)
		scores := make([]string, n+1)
		evens := 0
		for i, element := range replies {
			if i%2 == 0 {
				encoded_urls[evens] = element
				evens = evens + 1
			} else {
				scores[evens-1] = element
			}
		}
		encoded_urls = encoded_urls[:evens]
		scores = scores[:evens]

		var titles []string
		var urls []string
		if len(encoded_urls) == 0 {
		} else {
			titles, err = c.Cmd("HMGET", encoded+"$titles", encoded_urls).List()
			errHndlr(err)
			urls, err = c.Cmd("HMGET", encoded+"$urls", encoded_urls).List()
			errHndlr(err)
		}
		rows := make([]interface{}, len(titles))
		for i, title := range titles {
			row := make([]string, 2)
			row[0] = urls[i]
			// fmt.Println("scores", scores[i]+ 1000)
			// row[1] = scores[i] * QueryScore(terms, title)
			row[1] = title
			rows[i] = row
		}
		rows = rows[:len(titles)]

		output := make(map[string]interface{})
		output["terms"] = terms
		output["results"] = rows
		// fmt.Println(output)
		w.Header().Set("Access-Control-Allow-Origin", "*")
		renderer.JSON(w, http.StatusOK, output)
	}

	/* POSTING new stuff in */
	UpdateHandler := func(w http.ResponseWriter, req *http.Request) {
		values := make(map[string]string)

		required := []string{"domain", "url", "title"}
		errors := make(map[string]string)
		var value string
		for _, key := range required {
			value = strings.Trim(req.FormValue(key), " ")
			values[key] = value
			if value == "" {
				errors[key] = "Missing"
			}
		}

		optional := []string{"groups", "popularity"}
		for _, key := range optional {
			value = strings.Trim(req.FormValue(key), " ")
			if key == "groups" {
				// values[key] = strings.Split(value, ",")
				values[key] = value
			} else if key == "popularity" {
				if value == "" {
					values[key] = "0"
				} else {
					_, err := strconv.ParseFloat(value, 0)
					if err != nil {
						errors[key] = "Not a number"
					} else {
						values[key] = value
					}
				}
			} else {
				values[key] = value
			}
		}
		// fmt.Println("errors=", errors, len(errors))
		if len(errors) > 0 {
			error := make(map[string]interface{})
			error["error"] = errors
			renderer.JSON(w, http.StatusBadRequest, error)
			return
		}
		// popularity, _ := strconv.ParseInt(values["popularity"], 10, 0)
		popularity, _ := strconv.ParseFloat(values["popularity"], 0)

		encoded := encodeString(values["domain"])
		values["url_encoded"] = encodeString(values["url"])

		c, err := pool.Get()
		errHndlr(err)
		defer pool.Put(c)
		// c.Cmd("FLUSHALL")
		// fmt.Println("CAREFUL! Always flushing the database")
		piped_commands := 0
		for _, prefix := range getPrefixes(values["title"]) {
			c.Append("ZADD", encoded+prefix, popularity, values["url_encoded"])
			piped_commands += 1
		}
		c.Append("HSET", encoded+"$titles", values["url_encoded"], values["title"])
		piped_commands += 1
		c.Append("HSET", encoded+"$urls", values["url_encoded"], values["url"])
		piped_commands += 1
		for i := 1; i <= piped_commands; i++ {
			if err := c.GetReply().Err; err != nil {
				errHndlr(err)
			}
		}

		output := map[string]string{"message": "OK"}
		renderer.JSON(w, http.StatusCreated, output)
	}

	mux := mux.NewRouter()
	mux.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
		// this assumes there's a `templates/index.tmpl` file
		renderer.HTML(w, http.StatusOK, "index", nil)
	})
	mux.HandleFunc("/v1", HomeHandler).Methods("GET", "HEAD")
	mux.HandleFunc("/v1", UpdateHandler).Methods("POST", "PUT")

	n := negroni.Classic()
	n.UseHandler(mux)
	port := os.Getenv("PORT")
	if port == "" {
		port = "3001"
	}
	n.Run(":" + port)
}
<commit_msg>handlers use global variables<commit_after>package main

import (
	"crypto/md5"
	"encoding/hex"
	"fmt"
	"github.com/codegangsta/negroni"
	"github.com/fzzy/radix/extra/pool"
	"github.com/fzzy/radix/redis"
	"github.com/gorilla/mux"
	"github.com/unrolled/render"
	"net/http"
	"os"
	"runtime"
	// "io"
	// "time"
	// "log"
	"strconv"
	"strings"
	// "regexp"
)

func cleanWords(query string) []string {
	// r := regexp.MustCompile(regexp.QuoteMeta("[]()"))
	// query = r.ReplaceAllString(query, " ")
	query = strings.Replace(query, "\"", " ", -1)
	query = strings.Replace(query, "[", " ", -1)
	query = strings.Replace(query, "]", " ", -1)
	query = strings.Replace(query, "(", " ", -1)
	query = strings.Replace(query, ")", " ", -1)
	query = strings.Replace(query, "?", " ", -1)
	query = strings.Replace(query, "!", " ", -1)
	query = strings.Replace(query, ",", " ", -1)
	query = strings.Replace(query, "-", " ", -1)
	query = strings.Replace(query, ":", " ", -1)
	query = strings.Replace(query, ";", " ", -1)
	query = strings.Replace(query, ",", " ", -1)
	query = strings.Replace(query, "'", " ", -1)
	split := strings.Fields(strings.Trim(query, " "))
	terms := make([]string, len(split))
	for i, term := range split {
		terms[i] = strings.ToLower(strings.Trim(strings.Trim(term, " "), "."))
	}
	return terms
}

func encodeString(str string) string {
	h := md5.New()
	h.Write([]byte(str))
	return hex.EncodeToString(h.Sum(nil))[0:8]
}

func getPrefixes(title string) []string {
	var prefixes []string
	for _, word := range cleanWords(title) {
		for i := 1; i <= len(word); i++ {
			prefixes = append(prefixes, word[0:i])
		}
	}
	return prefixes
}

// func QueryScore(terms []string, title) float32 {
// 	return 1.0
// }

func errHndlr(err error) {
	if err != nil {
		fmt.Println("error:", err)
		panic(err)
		os.Exit(1)
	}
}

func IndexHandler(w http.ResponseWriter, req *http.Request) {
	// this assumes there's a `templates/index.tmpl` file
	renderer.HTML(w, http.StatusOK, "index", nil)
}

/* POSTING new stuff in */
func UpdateHandler(w http.ResponseWriter, req *http.Request) {
	values := make(map[string]string)

	required := []string{"domain", "url", "title"}
	errors := make(map[string]string)
	var value string
	for _, key := range required {
		value = strings.Trim(req.FormValue(key), " ")
		values[key] = value
		if value == "" {
			errors[key] = "Missing"
		}
	}

	optional := []string{"groups", "popularity"}
	for _, key := range optional {
		value = strings.Trim(req.FormValue(key), " ")
		if key == "groups" {
			// values[key] = strings.Split(value, ",")
			values[key] = value
		} else if key == "popularity" {
			if value == "" {
				values[key] = "0"
			} else {
				_, err := strconv.ParseFloat(value, 0)
				if err != nil {
					errors[key] = "Not a number"
				} else {
					values[key] = value
				}
			}
		} else {
			values[key] = value
		}
	}
	// fmt.Println("errors=", errors, len(errors))
	if len(errors) > 0 {
		error := make(map[string]interface{})
		error["error"] = errors
		renderer.JSON(w, http.StatusBadRequest, error)
		return
	}
	// popularity, _ := strconv.ParseInt(values["popularity"], 10, 0)
	popularity, _ := strconv.ParseFloat(values["popularity"], 0)

	encoded := encodeString(values["domain"])
	values["url_encoded"] = encodeString(values["url"])

	c, err := redis_pool.Get()
	errHndlr(err)
	defer redis_pool.Put(c)
	// c.Cmd("FLUSHALL")
	// fmt.Println("CAREFUL! Always flushing the database")
	piped_commands := 0
	for _, prefix := range getPrefixes(values["title"]) {
		c.Append("ZADD", encoded+prefix, popularity, values["url_encoded"])
		piped_commands += 1
	}
	c.Append("HSET", encoded+"$titles", values["url_encoded"], values["title"])
	piped_commands += 1
	c.Append("HSET", encoded+"$urls", values["url_encoded"], values["url"])
	piped_commands += 1
	for i := 1; i <= piped_commands; i++ {
		if err := c.GetReply().Err; err != nil {
			errHndlr(err)
		}
	}

	output := map[string]string{"message": "OK"}
	renderer.JSON(w, http.StatusCreated, output)
}

func FetchHandler(w http.ResponseWriter, req *http.Request) {

	n_str := req.FormValue("n")

	var n int64
	if n_str == "" {
		n = 10
	} else {
		n, err := strconv.ParseInt(n_str, 10, 0)
		if err != nil {
			// error := make(map[string]string{"error": "Not a number"})
			error := map[string]string{"error": "Not a number"}
			renderer.JSON(w, http.StatusBadRequest, error)
			return
		}
		if n <= 0 {
			error := map[string]string{"error": "Number too small"}
			renderer.JSON(w, http.StatusBadRequest, error)
			return
		}
		if n > 100 {
			error := map[string]string{"error": "Number too big"}
			renderer.JSON(w, http.StatusBadRequest, error)
			return
		}
	}

	domain := strings.Trim(req.FormValue("domain"), " ")
	if domain == "" {
		error := map[string]string{"error": "'domain' missing"}
		renderer.JSON(w, http.StatusBadRequest, error)
		return
	}

	encoded := encodeString(domain)
	// fmt.Println(domain, encoded)

	q := strings.Trim(req.FormValue("q"), " ")
	terms := cleanWords(q)

	c, err := redis_pool.Get()
	errHndlr(err)
	defer redis_pool.CarefullyPut(c, &err)

	encoded_terms := make([]string, len(terms))
	for i, term := range terms {
		encoded_terms[i] = encoded + term
	}
	// NOTE! Maybe we don't need the ZINTERSTORE if there's only 1 command
	c.Append("ZINTERSTORE", "$tmp", len(terms), encoded_terms, "AGGREGATE", "max")
	c.Append("ZREVRANGE", "$tmp", 0, n-1, "WITHSCORES")

	c.GetReply() // the ZINTERSTORE
	replies, err := c.GetReply().List()
	// fmt.Println("replies", replies, len(replies))
	errHndlr(err)

	encoded_urls := make([]string, n+1)
	scores := make([]string, n+1)
	evens := 0
	for i, element := range replies {
		if i%2 == 0 {
			encoded_urls[evens] = element
			evens = evens + 1
		} else {
			scores[evens-1] = element
		}
	}
	encoded_urls = encoded_urls[:evens]
	scores = scores[:evens]

	var titles []string
	var urls []string
	if len(encoded_urls) == 0 {
	} else {
		titles, err = c.Cmd("HMGET", encoded+"$titles", encoded_urls).List()
		errHndlr(err)
		urls, err = c.Cmd("HMGET", encoded+"$urls", encoded_urls).List()
		errHndlr(err)
	}
	rows := make([]interface{}, len(titles))
	for i, title := range titles {
		row := make([]string, 2)
		row[0] = urls[i]
		// fmt.Println("scores", scores[i]+ 1000)
		// row[1] = scores[i] * QueryScore(terms, title)
		row[1] = title
		rows[i] = row
	}
	rows = rows[:len(titles)]

	output := make(map[string]interface{})
	output["terms"] = terms
	output["results"] = rows
	// fmt.Println(output)
	w.Header().Set("Access-Control-Allow-Origin", "*")
	renderer.JSON(w, http.StatusOK, output)
}

var renderer *render.Render
var redis_pool *pool.Pool

func main() {
	fmt.Println(runtime.NumCPU())
	// runtime.GOMAXPROCS(8)

	is_debug := os.Getenv("DEBUG")
	// fmt.Println("debug", is_debug)
	var debug = true
	if is_debug == "" {
		debug = false
	}
	renderer = render.New(render.Options{
		IndentJSON:    debug,
		IsDevelopment: debug,
	})

	REDIS_URL := os.Getenv("REDIS_URL")
	if REDIS_URL == "" {
		REDIS_URL = "127.0.0.1:6379"
	}
	REDIS_DATABASE := os.Getenv("REDIS_DATABASE")
	if REDIS_DATABASE == "" {
		REDIS_DATABASE = "0"
	}

	df := func(network, addr string) (*redis.Client, error) {
		client, err := redis.Dial(network, addr)
		// fmt.Println("DIaling")
		if err != nil {
			return nil, err
		}
		err = client.Cmd("SELECT", REDIS_DATABASE).Err
		if err != nil {
			return nil, err
		}
		// if err = client.Cmd("AUTH", "SUPERSECRET").Err; err != nil {
		// 	client.Close()
		// 	return nil, err
		// }
		return client, nil
	}

	var err error
	redis_pool, err = pool.NewCustomPool("tcp", REDIS_URL, 100, df)
	errHndlr(err)

	mux := mux.NewRouter()
	// mux.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
	// 	// this assumes there's a `templates/index.tmpl` file
	// 	renderer.HTML(w, http.StatusOK, "index", nil)
	// })
	mux.HandleFunc("/", IndexHandler).Methods("GET", "HEAD")
	mux.HandleFunc("/v1", FetchHandler).Methods("GET", "HEAD")
	mux.HandleFunc("/v1", UpdateHandler).Methods("POST", "PUT")

	n := negroni.Classic()
	n.UseHandler(mux)
	port := os.Getenv("PORT")
	if port == "" {
		port = "3001"
	}
	n.Run(":" + port)
}
<|endoftext|>


<commit_before>package veneur

import (
	"bytes"
	"net"
	"net/http"
	"sync"
	"time"

	"github.com/DataDog/datadog-go/statsd"
	"github.com/Sirupsen/logrus"
	"github.com/getsentry/raven-go"
	"github.com/zenazn/goji/bind"
	"github.com/zenazn/goji/graceful"
)

// VERSION stores the current veneur version.
// It must be a var so it can be set at link time.
var VERSION = "dirty"

// A Server is the actual veneur instance that will be run.
type Server struct {
	Workers     []*Worker
	EventWorker *EventWorker

	statsd *statsd.Client
	logger *logrus.Logger
	sentry *raven.Client

	Hostname string
	Tags     []string

	DDHostname string
	DDAPIKey   string
	HTTPClient *http.Client

	HTTPAddr    string
	ForwardAddr string
	UDPAddr     *net.UDPAddr
	RcvbufBytes int

	HistogramPercentiles []float64
}

// NewFromConfig creates a new veneur server from a configuration specification.
func NewFromConfig(conf Config) (ret Server, err error) {
	ret.Hostname = conf.Hostname
	ret.Tags = conf.Tags
	ret.DDHostname = conf.APIHostname
	ret.DDAPIKey = conf.Key
	ret.HistogramPercentiles = conf.Percentiles

	ret.HTTPClient = &http.Client{
		// make sure that POSTs to datadog do not overflow the flush interval
		Timeout: conf.Interval * 9 / 10,
		// we're fine with using the default transport and redirect behavior
	}

	ret.statsd, err = statsd.NewBuffered(conf.StatsAddr, 1024)
	if err != nil {
		return
	}
	ret.statsd.Namespace = "veneur."
	ret.statsd.Tags = append(ret.Tags, "veneurlocalonly")

	// nil is a valid sentry client that noops all methods, if there is no DSN
	// we can just leave it as nil
	if conf.SentryDSN != "" {
		ret.sentry, err = raven.New(conf.SentryDSN)
		if err != nil {
			return
		}
	}

	ret.logger = logrus.New()
	if conf.Debug {
		ret.logger.Level = logrus.DebugLevel
	}
	ret.logger.Hooks.Add(sentryHook{
		c:        ret.sentry,
		hostname: ret.Hostname,
		lv: []logrus.Level{
			logrus.ErrorLevel,
			logrus.FatalLevel,
			logrus.PanicLevel,
		},
	})
	ret.logger.WithField("version", VERSION).Info("Starting server")

	ret.logger.WithField("number", conf.NumWorkers).Info("Starting workers")
	ret.Workers = make([]*Worker, conf.NumWorkers)
	for i := range ret.Workers {
		ret.Workers[i] = NewWorker(i+1, ret.statsd, ret.logger)
		// do not close over loop index
		go func(w *Worker) {
			defer func() {
				ret.ConsumePanic(recover())
			}()
			w.Work()
		}(ret.Workers[i])
	}

	ret.EventWorker = NewEventWorker(ret.statsd)
	go func() {
		defer func() {
			ret.ConsumePanic(recover())
		}()
		ret.EventWorker.Work()
	}()

	ret.UDPAddr, err = net.ResolveUDPAddr("udp", conf.UDPAddr)
	if err != nil {
		return
	}
	ret.RcvbufBytes = conf.ReadBufferSizeBytes
	ret.HTTPAddr = conf.HTTPAddr
	ret.ForwardAddr = conf.ForwardAddr

	conf.Key = "REDACTED"
	conf.SentryDSN = "REDACTED"
	ret.logger.WithField("config", conf).Debug("Initialized server")

	return
}

// HandlePacket processes each packet that is sent to the server, and sends to an
// appropriate worker (EventWorker or Worker).
func (s *Server) HandlePacket(packet []byte) {
	// This is a very performance-sensitive function
	// and packets may be dropped if it gets slowed down.
	// Keep that in mind when modifying!

	if len(packet) == 0 {
		// a lot of clients send packets that accidentally have a trailing
		// newline, it's easier to just let them be
		return
	}

	if bytes.HasPrefix(packet, []byte{'_', 'e', '{'}) {
		event, err := ParseEvent(packet)
		if err != nil {
			s.logger.WithFields(logrus.Fields{
				logrus.ErrorKey: err,
				"packet":        string(packet),
			}).Error("Could not parse packet")
			s.statsd.Count("packet.error_total", 1, []string{"packet_type:event"}, 1.0)
			return
		}
		s.EventWorker.EventChan <- *event
	} else if bytes.HasPrefix(packet, []byte{'_', 's', 'c'}) {
		svcheck, err := ParseServiceCheck(packet)
		if err != nil {
			s.logger.WithFields(logrus.Fields{
				logrus.ErrorKey: err,
				"packet":        string(packet),
			}).Error("Could not parse packet")
			s.statsd.Count("packet.error_total", 1, []string{"packet_type:service_check"}, 1.0)
			return
		}
		s.EventWorker.ServiceCheckChan <- *svcheck
	} else {
		metric, err := ParseMetric(packet)
		if err != nil {
			s.logger.WithFields(logrus.Fields{
				logrus.ErrorKey: err,
				"packet":        string(packet),
			}).Error("Could not parse packet")
			s.statsd.Count("packet.error_total", 1, []string{"packet_type:metric"}, 1.0)
			return
		}
		s.Workers[metric.Digest%uint32(len(s.Workers))].PacketChan <- *metric
	}
}

// ReadSocket listens for available packets to handle.
func (s *Server) ReadSocket(packetPool *sync.Pool, reuseport bool) {
	// each goroutine gets its own socket
	// if the sockets support SO_REUSEPORT, then this will cause the
	// kernel to distribute datagrams across them, for better read
	// performance
	serverConn, err := NewSocket(s.UDPAddr, s.RcvbufBytes, reuseport)
	if err != nil {
		// if any goroutine fails to create the socket, we can't really
		// recover, so we just blow up
		// this probably indicates a systemic issue, eg lack of
		// SO_REUSEPORT support
		s.logger.WithError(err).Fatal("Error listening for UDP")
	}
	s.logger.WithField("address", s.UDPAddr).Info("UDP server listening")

	for {
		buf := packetPool.Get().([]byte)
		n, _, err := serverConn.ReadFrom(buf)
		if err != nil {
			s.logger.WithError(err).Error("Error reading from UDP")
			continue
		}

		// statsd allows multiple packets to be joined by newlines and sent as
		// one larger packet
		// note that spurious newlines are not allowed in this format, it has
		// to be exactly one newline between each packet, with no leading or
		// trailing newlines
		splitPacket := NewSplitBytes(buf[:n], '\n')
		for splitPacket.Next() {
			s.HandlePacket(splitPacket.Chunk())
		}

		// the Metric struct created by HandlePacket has no byte slices in it,
		// only strings
		// therefore there are no outstanding references to this byte slice, we
		// can return it to the pool
		packetPool.Put(buf)
	}
}

// HTTPServe starts the HTTP server and listens perpetually until it encounters an unrecoverable error.
func (s *Server) HTTPServe() {
	httpSocket := bind.Socket(s.HTTPAddr)
	graceful.Timeout(10 * time.Second)
	graceful.PreHook(func() {
		s.logger.Info("Terminating HTTP listener")
	})
	graceful.HandleSignals()
	s.logger.WithField("address", s.HTTPAddr).Info("HTTP server listening")
	bind.Ready()

	if err := graceful.Serve(httpSocket, s.Handler()); err != nil {
		s.logger.WithError(err).Error("HTTP server shut down due to error")
	}

	graceful.Shutdown()
}

// Shutdown signals the server to shut down after closing all
// current connections.
func (s *Server) Shutdown() {
	graceful.Shutdown()
}

// SplitBytes iterates over a byte buffer, returning chunks split by a given
// delimiter byte. It does not perform any allocations, and does not modify the
// buffer it is given. It is not safe for use by concurrent goroutines.
//
//     sb := NewSplitBytes(buf, '\n')
//     for sb.Next() {
//         fmt.Printf("%q\n", sb.Chunk())
//     }
//
// The sequence of chunks returned by SplitBytes is equivalent to calling
// bytes.Split, except without allocating an intermediate slice.
type SplitBytes struct {
	buf          []byte
	delim        byte
	currentChunk []byte
	lastChunk    bool
}

// NewSplitBytes initializes a SplitBytes struct with the provided buffer and delimiter.
func NewSplitBytes(buf []byte, delim byte) *SplitBytes {
	return &SplitBytes{
		buf:   buf,
		delim: delim,
	}
}

// Next advances SplitBytes to the next chunk, returning true if a new chunk
// actually exists and false otherwise.
func (sb *SplitBytes) Next() bool {
	if sb.lastChunk {
		// we do not check the length here, this ensures that we return the
		// last chunk in the sequence (even if it's empty)
		return false
	}

	next := bytes.IndexByte(sb.buf, sb.delim)
	if next == -1 {
		// no newline, consume the entire buffer
		sb.currentChunk = sb.buf
		sb.buf = nil
		sb.lastChunk = true
	} else {
		sb.currentChunk = sb.buf[:next]
		sb.buf = sb.buf[next+1:]
	}
	return true
}

// Chunk returns the current chunk.
func (sb *SplitBytes) Chunk() []byte {
	return sb.currentChunk
}
<commit_msg>Log server shutdowns<commit_after>package veneur

import (
	"bytes"
	"net"
	"net/http"
	"sync"
	"time"

	"github.com/DataDog/datadog-go/statsd"
	"github.com/Sirupsen/logrus"
	"github.com/getsentry/raven-go"
	"github.com/zenazn/goji/bind"
	"github.com/zenazn/goji/graceful"
)

// VERSION stores the current veneur version.
// It must be a var so it can be set at link time.
var VERSION = "dirty"

// A Server is the actual veneur instance that will be run.
type Server struct {
	Workers     []*Worker
	EventWorker *EventWorker

	statsd *statsd.Client
	logger *logrus.Logger
	sentry *raven.Client

	Hostname string
	Tags     []string

	DDHostname string
	DDAPIKey   string
	HTTPClient *http.Client

	HTTPAddr    string
	ForwardAddr string
	UDPAddr     *net.UDPAddr
	RcvbufBytes int

	HistogramPercentiles []float64
}

// NewFromConfig creates a new veneur server from a configuration specification.
func NewFromConfig(conf Config) (ret Server, err error) {
	ret.Hostname = conf.Hostname
	ret.Tags = conf.Tags
	ret.DDHostname = conf.APIHostname
	ret.DDAPIKey = conf.Key
	ret.HistogramPercentiles = conf.Percentiles

	ret.HTTPClient = &http.Client{
		// make sure that POSTs to datadog do not overflow the flush interval
		Timeout: conf.Interval * 9 / 10,
		// we're fine with using the default transport and redirect behavior
	}

	ret.statsd, err = statsd.NewBuffered(conf.StatsAddr, 1024)
	if err != nil {
		return
	}
	ret.statsd.Namespace = "veneur."
	ret.statsd.Tags = append(ret.Tags, "veneurlocalonly")

	// nil is a valid sentry client that noops all methods, if there is no DSN
	// we can just leave it as nil
	if conf.SentryDSN != "" {
		ret.sentry, err = raven.New(conf.SentryDSN)
		if err != nil {
			return
		}
	}

	ret.logger = logrus.New()
	if conf.Debug {
		ret.logger.Level = logrus.DebugLevel
	}
	ret.logger.Hooks.Add(sentryHook{
		c:        ret.sentry,
		hostname: ret.Hostname,
		lv: []logrus.Level{
			logrus.ErrorLevel,
			logrus.FatalLevel,
			logrus.PanicLevel,
		},
	})
	ret.logger.WithField("version", VERSION).Info("Starting server")

	ret.logger.WithField("number", conf.NumWorkers).Info("Starting workers")
	ret.Workers = make([]*Worker, conf.NumWorkers)
	for i := range ret.Workers {
		ret.Workers[i] = NewWorker(i+1, ret.statsd, ret.logger)
		// do not close over loop index
		go func(w *Worker) {
			defer func() {
				ret.ConsumePanic(recover())
			}()
			w.Work()
		}(ret.Workers[i])
	}

	ret.EventWorker = NewEventWorker(ret.statsd)
	go func() {
		defer func() {
			ret.ConsumePanic(recover())
		}()
		ret.EventWorker.Work()
	}()

	ret.UDPAddr, err = net.ResolveUDPAddr("udp", conf.UDPAddr)
	if err != nil {
		return
	}
	ret.RcvbufBytes = conf.ReadBufferSizeBytes
	ret.HTTPAddr = conf.HTTPAddr
	ret.ForwardAddr = conf.ForwardAddr

	conf.Key = "REDACTED"
	conf.SentryDSN = "REDACTED"
	ret.logger.WithField("config", conf).Debug("Initialized server")

	return
}

// HandlePacket processes each packet that is sent to the server, and sends to an
// appropriate worker (EventWorker or Worker).
func (s *Server) HandlePacket(packet []byte) {
	// This is a very performance-sensitive function
	// and packets may be dropped if it gets slowed down.
	// Keep that in mind when modifying!

	if len(packet) == 0 {
		// a lot of clients send packets that accidentally have a trailing
		// newline, it's easier to just let them be
		return
	}

	if bytes.HasPrefix(packet, []byte{'_', 'e', '{'}) {
		event, err := ParseEvent(packet)
		if err != nil {
			s.logger.WithFields(logrus.Fields{
				logrus.ErrorKey: err,
				"packet":        string(packet),
			}).Error("Could not parse packet")
			s.statsd.Count("packet.error_total", 1, []string{"packet_type:event"}, 1.0)
			return
		}
		s.EventWorker.EventChan <- *event
	} else if bytes.HasPrefix(packet, []byte{'_', 's', 'c'}) {
		svcheck, err := ParseServiceCheck(packet)
		if err != nil {
			s.logger.WithFields(logrus.Fields{
				logrus.ErrorKey: err,
				"packet":        string(packet),
			}).Error("Could not parse packet")
			s.statsd.Count("packet.error_total", 1, []string{"packet_type:service_check"}, 1.0)
			return
		}
		s.EventWorker.ServiceCheckChan <- *svcheck
	} else {
		metric, err := ParseMetric(packet)
		if err != nil {
			s.logger.WithFields(logrus.Fields{
				logrus.ErrorKey: err,
				"packet":        string(packet),
			}).Error("Could not parse packet")
			s.statsd.Count("packet.error_total", 1, []string{"packet_type:metric"}, 1.0)
			return
		}
		s.Workers[metric.Digest%uint32(len(s.Workers))].PacketChan <- *metric
	}
}

// ReadSocket listens for available packets to handle.
func (s *Server) ReadSocket(packetPool *sync.Pool, reuseport bool) {
	// each goroutine gets its own socket
	// if the sockets support SO_REUSEPORT, then this will cause the
	// kernel to distribute datagrams across them, for better read
	// performance
	serverConn, err := NewSocket(s.UDPAddr, s.RcvbufBytes, reuseport)
	if err != nil {
		// if any goroutine fails to create the socket, we can't really
		// recover, so we just blow up
		// this probably indicates a systemic issue, eg lack of
		// SO_REUSEPORT support
		s.logger.WithError(err).Fatal("Error listening for UDP")
	}
	s.logger.WithField("address", s.UDPAddr).Info("UDP server listening")

	for {
		buf := packetPool.Get().([]byte)
		n, _, err := serverConn.ReadFrom(buf)
		if err != nil {
			s.logger.WithError(err).Error("Error reading from UDP")
			continue
		}

		// statsd allows multiple packets to be joined by newlines and sent as
		// one larger packet
		// note that spurious newlines are not allowed in this format, it has
		// to be exactly one newline between each packet, with no leading or
		// trailing newlines
		splitPacket := NewSplitBytes(buf[:n], '\n')
		for splitPacket.Next() {
			s.HandlePacket(splitPacket.Chunk())
		}

		// the Metric struct created by HandlePacket has no byte slices in it,
		// only strings
		// therefore there are no outstanding references to this byte slice, we
		// can return it to the pool
		packetPool.Put(buf)
	}
}

// HTTPServe starts the HTTP server and listens perpetually until it encounters an unrecoverable error.
func (s *Server) HTTPServe() {
	httpSocket := bind.Socket(s.HTTPAddr)
	graceful.Timeout(10 * time.Second)
	graceful.PreHook(func() {
		s.logger.Info("Terminating HTTP listener")
	})
	graceful.HandleSignals()
	s.logger.WithField("address", s.HTTPAddr).Info("HTTP server listening")
	bind.Ready()

	if err := graceful.Serve(httpSocket, s.Handler()); err != nil {
		s.logger.WithError(err).Error("HTTP server shut down due to error")
	}

	graceful.Shutdown()
}

// Shutdown signals the server to shut down after closing all
// current connections.
func (s *Server) Shutdown() {
	s.logger.Info("Shutting down server gracefully")
	graceful.Shutdown()
}

// SplitBytes iterates over a byte buffer, returning chunks split by a given
// delimiter byte. It does not perform any allocations, and does not modify the
// buffer it is given. It is not safe for use by concurrent goroutines.
//
//     sb := NewSplitBytes(buf, '\n')
//     for sb.Next() {
//         fmt.Printf("%q\n", sb.Chunk())
//     }
//
// The sequence of chunks returned by SplitBytes is equivalent to calling
// bytes.Split, except without allocating an intermediate slice.
type SplitBytes struct {
	buf          []byte
	delim        byte
	currentChunk []byte
	lastChunk    bool
}

// NewSplitBytes initializes a SplitBytes struct with the provided buffer and delimiter.
func NewSplitBytes(buf []byte, delim byte) *SplitBytes {
	return &SplitBytes{
		buf:   buf,
		delim: delim,
	}
}

// Next advances SplitBytes to the next chunk, returning true if a new chunk
// actually exists and false otherwise.
func (sb *SplitBytes) Next() bool {
	if sb.lastChunk {
		// we do not check the length here, this ensures that we return the
		// last chunk in the sequence (even if it's empty)
		return false
	}

	next := bytes.IndexByte(sb.buf, sb.delim)
	if next == -1 {
		// no newline, consume the entire buffer
		sb.currentChunk = sb.buf
		sb.buf = nil
		sb.lastChunk = true
	} else {
		sb.currentChunk = sb.buf[:next]
		sb.buf = sb.buf[next+1:]
	}
	return true
}

// Chunk returns the current chunk.
func (sb *SplitBytes) Chunk() []byte {
	return sb.currentChunk
}
<|endoftext|>


<commit_before>package main

import (
	//"fmt"
	"net/http"
	"text/template"
)

var (
	ChartHandlers = make(map[string]ChartIf)
	ChartFiles    []string
	Index         int
)

func handler(w http.ResponseWriter, r *http.Request) {
	tt, err := Parse(ChartFiles[Index])
	if err != nil {
		w.Write([]byte(err.Error()))
		return
	}

	Index++
	Index = Index % len(ChartFiles)

	if t, err := template.New("foo").Parse(tt.tmpl); err != nil {
		w.Write([]byte(err.Error()))
	} else {
		if err = t.ExecuteTemplate(w, "T", tt.args); err != nil {
			w.Write([]byte(err.Error()))
		}
	}
}

func ListenAndServe(addr string) error {
	http.HandleFunc("/", handler)
	http.HandleFunc("/favicon.ico", func(w http.ResponseWriter, r *http.Request) {})

	var err error
	ChartFiles, err = LookupChartFiles(".")
	if err != nil {
		return err
	}

	// Register chart handlders
	ChartHandlers["spline"] = new(SplineChart)
	ChartHandlers["column"] = new(SplineChart)
	ChartHandlers["area"] = new(SplineChart)
	ChartHandlers["bar"] = new(SplineChart)
	ChartHandlers["line"] = new(SplineChart)
	ChartHandlers["pie"] = new(PieChart)

	return http.ListenAndServe(addr, nil)
}
<commit_msg>Add index paramenter in the URL path<commit_after>package main

import (
	//"fmt"
	"net/http"
	"text/template"
	"strconv"
	"log"
)

var (
	ChartHandlers = make(map[string]ChartIf)
	ChartFiles    []string
	Index         int
)

func handler(w http.ResponseWriter, r *http.Request) {
	values := r.URL.Query()
	i := values.Get("index")
	if len(i) > 0 {
		Index, _ = strconv.Atoi(i)
	} else {
		Index++
	}

	Index = Index % len(ChartFiles)

	log.Printf("Index=%v rendering %v", Index, ChartFiles[Index])
	tt, err := Parse(ChartFiles[Index])
	if err != nil {
		w.Write([]byte(err.Error()))
		return
	}

	if t, err := template.New("foo").Parse(tt.tmpl); err != nil {
		w.Write([]byte(err.Error()))
	} else {
		if err = t.ExecuteTemplate(w, "T", tt.args); err != nil {
			w.Write([]byte(err.Error()))
		}
	}
}

func ListenAndServe(addr string) error {
	http.HandleFunc("/", handler)
	http.HandleFunc("/favicon.ico", func(w http.ResponseWriter, r *http.Request) {})

	var err error
	ChartFiles, err = LookupChartFiles(".")
	if err != nil {
		return err
	}

	// Register chart handlders
	ChartHandlers["spline"] = new(SplineChart)
	ChartHandlers["column"] = new(SplineChart)
	ChartHandlers["area"] = new(SplineChart)
	ChartHandlers["bar"] = new(SplineChart)
	ChartHandlers["line"] = new(SplineChart)
	ChartHandlers["pie"] = new(PieChart)

	return http.ListenAndServe(addr, nil)
}
<|endoftext|>


<commit_before>package main

import (
	"code.google.com/p/goweb/goweb"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"launchpad.net/mgo"
	"launchpad.net/mgo/bson"
	"regexp"
	"runtime"
	"strconv"
	"strings"
	"syscall"
	"time"
)

// types
type Msg struct {
	Iv      string `json:"iv,omitempty"`
	Message string `json:"message"`
}

type MsgWrap struct {
	Content   Msg       `json:"content"`
	Timestamp string    `json:"timestamp"`
	Time      time.Time `json:"-"`
	Id        int       `bson:"_id"`
}

type Counter struct {
	Id string `bson:"_id"`
	N  int
}

type LastMessage struct {
	LastMessage int `json:"lastMessage"`
}

// utility functions
func updateCounter(id string, d *mgo.Database) int {
	var counter Counter
	col := d.C("counters")
	change := mgo.Change{Update: bson.M{"$inc": bson.M{"n": 1}}, New: true, Upsert: true}
	err := col.Find(bson.M{"_id": "C" + id}).Modify(change, &counter)
	if err != nil {
		panic("Failed to update counter")
	}
	fmt.Println(counter.N)
	return counter.N
}

func getCounter(id string, d *mgo.Database) int {
	var counter Counter
	col := d.C("counters")
	err := col.Find(bson.M{"_id": "C" + id}).One(&counter)
	if err != nil {
		return (0)
	}
	fmt.Println(counter.N)
	return counter.N
}

func dropPrivs(uid int) {
	if syscall.Getuid() == 0 {
		fmt.Printf("Dropping privileges.")
		syscall.Setuid(uid)
	}
}

func validateHash(hash string) {
	match, _ := regexp.MatchString("^\\w{40}$", hash)
	if !match {
		panic("Invalid hash circle key.")
	}
}

func catchPanic(c *goweb.Context, s *mgo.Session) {

	if r := recover(); r != nil {
		fmt.Printf("%s\n", r)
		s.Close()
		goweb.AddFormatter(&goweb.JsonFormatter{})
		c.RespondWithError(400)
		recover()
	}
	s.Close()
}

// handler functions
func PostMsg(c *goweb.Context, d *mgo.Database) {
	var m Msg
	var mw MsgWrap

	col := d.C("C" + c.PathParams["hash"])
	body, _ := ioutil.ReadAll(c.Request.Body)
	json.Unmarshal(body, &m)
	mw.Content = m
	mw.Id = updateCounter(c.PathParams["hash"], d)
	mw.Time = time.Now()
	mw.Timestamp = mw.Time.Format(time.RFC1123)
	mw.Timestamp = strings.Replace(mw.Timestamp, "UTC", "GMT", -1)
	col.Insert(mw)
}

func GetMsgRange(c *goweb.Context, d *mgo.Database) {
	ids := strings.Split(c.PathParams["id"], "-")
	top, _ := strconv.Atoi(ids[0])
	bottom, _ := strconv.Atoi(ids[1])

	if bottom > top {
		goweb.AddFormatter(&goweb.JsonFormatter{})
		c.RespondWithStatus(403)
		return
	}

	delta := top - bottom
	if delta > 30 {
		goweb.AddFormatter(&goweb.JsonFormatter{})
		c.RespondWithStatus(403)
		return
	}

	var msgs []MsgWrap
	col := d.C("C" + c.PathParams["hash"])
	msgQuery := col.Find(bson.M{"_id": bson.M{"$gte": bottom, "$lte": top}}).Sort("-_id")
	msgQuery.All(&msgs)

	msgBytes, _ := json.Marshal(msgs)
	c.ResponseWriter.Header().Set("Content-Type", "application/json")
	c.ResponseWriter.Write(msgBytes)
}

func GetOneMsg(c *goweb.Context, d *mgo.Database) {
	var mw MsgWrap
	col := d.C("C" + c.PathParams["hash"])
	id, _ := strconv.Atoi(c.PathParams["id"])
	err := col.Find(bson.M{"_id": id}).One(&mw)
	if err != nil {
		fmt.Printf("Error fetching %s: %s", c.PathParams["hash"], err)
		goweb.AddFormatter(&goweb.JsonFormatter{})
		c.RespondWithNotFound()
		return
	}

	c.ResponseWriter.Header().Set("Last-Modified", mw.Timestamp)
	msgBytes, _ := json.Marshal(mw.Content)
	c.ResponseWriter.Header().Set("Content-Type", "application/json")
	c.ResponseWriter.Write(msgBytes)
}

func GetMsg(c *goweb.Context, d *mgo.Database) {

	if strings.Contains(c.PathParams["id"], "-") {
		GetMsgRange(c, d)
	} else {
		GetOneMsg(c, d)
	}

}

func GetLastMsg(c *goweb.Context, d *mgo.Database) {

	mostRecent := getCounter(c.PathParams["hash"], d)

	lastMsg := &LastMessage{LastMessage: mostRecent}

	b, _ := json.Marshal(lastMsg)
	c.ResponseWriter.Header().Set("Content-Type", "application/json")
	c.ResponseWriter.Write(b)
}

// expire goroutine
func ExpireMessageLoop(d *mgo.Database) {
	for true {
		fmt.Printf("Searching for expired messages...\n")
		collections, _ := d.CollectionNames()
		for _, collection := range collections {
			if !strings.HasPrefix(collection, "C") {
				continue
			}

			var msgs []MsgWrap
			fmt.Printf("Expiring %s\n", collection)
			col := d.C(collection)

			// 3  months old and they are gone
			delBoundary := time.Now().Add(-time.Hour * 2160)
			msgQuery := col.Find(bson.M{"time": bson.M{"$lte": delBoundary}})
			msgQuery.All(&msgs)

			for _, msgW := range msgs {
				fmt.Printf("Deleted id %d timestamp %s\n", msgW.Id, msgW.Timestamp)
				err := col.Remove(bson.M{"_id": msgW.Id})
				if err != nil {
					fmt.Printf("Error expiring %d in %s.", msgW.Id, collection)
					continue
				}
			}

		}
		time.Sleep(time.Hour)
	}
}

func main() {

	runtime.GOMAXPROCS(runtime.NumCPU())

	var (
		port int
		ip   string
		db   string
		uid  int
	)
	flag.IntVar(&port, "port", 80, "Port to bind on.")
	flag.StringVar(&ip, "ip", "127.0.0.1", "IP to bind to")
	flag.StringVar(&db, "db", "muteswan", "MongoDB database to use")
	flag.IntVar(&uid, "uid", 1001, "User to drop privileges")
	flag.Parse()

	fmt.Printf("Port is %d\n", port)
	fmt.Printf("IP is %s\n", ip)
	fmt.Printf("DB is %s\n", db)
	fmt.Printf("UID is %d\n", uid)

	session, err := mgo.Dial("localhost")
	if err != nil {
		panic(err)
	}
	defer session.Close()

	session.SetMode(mgo.Monotonic, true)

	go ExpireMessageLoop(session.DB(db))

	// fetch a message, or a message range
	// e.g. GET /1e7db1c00e4036d9ea426e5875c355184578ab2d/200
	// or   GET /1e7db1c00e4036d9ea426e5875c355184578ab2d/200-190
	goweb.MapFunc("/{hash}/{id}", func(c *goweb.Context) {
		s := session.Copy()
		defer catchPanic(c, s)
		dropPrivs(uid)

		validateHash(c.PathParams["hash"])
		fmt.Printf("GET %s\n", c.Request.URL.String())
		GetMsg(c, s.DB(db))
	})

	// Get the last message in the circle
	// e.g. GET /1e7db1c00e4036d9ea426e5875c355184578ab2d
	goweb.MapFunc("/{hash}", func(c *goweb.Context) {
		s := session.Copy()
		defer catchPanic(c, s)
		dropPrivs(uid)

		validateHash(c.PathParams["hash"])
		fmt.Printf("GET %s\n", c.Request.URL.String())
		GetLastMsg(c, s.DB(db))
	}, goweb.GetMethod)

	// POST a new message to a circle
	// e.g. POST /1e7db1c00e4036d9ea426e5875c355184578ab2d
	goweb.MapFunc("/{hash}", func(c *goweb.Context) {
		s := session.Copy()
		defer catchPanic(c, s)
		dropPrivs(uid)

		validateHash(c.PathParams["hash"])
		fmt.Printf("POST %s\n", c.Request.URL.String())
		PostMsg(c, s.DB(db))
	}, goweb.PostMethod)

	goweb.ListenAndServe(fmt.Sprintf("%s:%d", ip, port))

}
<commit_msg>Add filestore/mongostore options with MtsnStore interface.<commit_after>package main

import (
	"code.google.com/p/goweb/goweb"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"launchpad.net/mgo"
	"launchpad.net/mgo/bson"
	"regexp"
	"runtime"
	"strconv"
	"strings"
	"syscall"
	"time"
)

// types
type Msg struct {
	Iv      string `json:"iv,omitempty"`
	Message string `json:"message"`
}

type MsgWrap struct {
	Content   Msg       `json:"content"`
	Timestamp string    `json:"timestamp"`
	Time      time.Time `json:"-"`
	Id        int       `bson:"_id"`
}

type Counter struct {
	Id string `bson:"_id"`
	N  int
}

type LastMessage struct {
	LastMessage int `json:"lastMessage"`
}

type MtsnStore interface {
	GetMsg(id int) (MsgWrap, error)
	GetMsgs(top int, bottom int) ([]MsgWrap, error)
	GetLastMsg() (LastMessage, error)
	PostMsg(msgw MsgWrap) error
}

type MongoStore struct {
	Circle string
	Db *mgo.Database
}

type FileStore struct {
	Circle string
	Db *mgo.Database
}

func (ms *MongoStore) GetMsg(id int) (MsgWrap, error) {
	var mw MsgWrap
        col := ms.Db.C("C" + ms.Circle)

        err := col.Find(bson.M{"_id": id}).One(&mw)
        if err != nil {
                fmt.Printf("Error fetching %s: %s", id, err)
		return mw, err
        }

	return mw, nil
}

func (ms *MongoStore) GetMsgs(top int, bottom int) ([]MsgWrap, error) {
	var msgs []MsgWrap

        if bottom > top {
                return msgs,nil
        }

        col := ms.Db.C("C" + ms.Circle)
        msgQuery := col.Find(bson.M{"_id": bson.M{"$gte": bottom, "$lte": top}}).Sort("-_id")
        err := msgQuery.All(&msgs)

	if err != nil {
		return msgs, err
	}

	return msgs,nil
}

func (ms *MongoStore) GetLastMsg() (LastMessage, error) {
        var lastMessage LastMessage
        mostRecent := getCounter(ms.Circle, ms.Db)
        lastMessage = LastMessage{LastMessage: mostRecent}
	return lastMessage,nil
}

func (ms *MongoStore) PostMsg(msgw MsgWrap) error {

        msgw.Id = updateCounter(ms.Circle,ms.Db)
        msgw.Time = time.Now()
        msgw.Timestamp = msgw.Time.Format(time.RFC1123)
        msgw.Timestamp = strings.Replace(msgw.Timestamp, "UTC", "GMT", -1)

        col := ms.Db.C(ms.Circle)
        err := col.Insert(msgw)

	if err != nil {
		return err
	}

	return nil

}

func (ms *FileStore) GetMsg(id int) (MsgWrap, error) {
        var mw MsgWrap
        return mw,nil
}

func (ms *FileStore) GetMsgs(top int, bottom int) ([]MsgWrap, error) {
        var msgs []MsgWrap
        return msgs,nil
}

func (ms *FileStore) GetLastMsg() (LastMessage, error) {
        var lastMessage LastMessage

        //path := fmt.Sprintf("/home/junger/gowebtest/muteswansrv-hidden/data/%s", ms.Circle)
        //dir,_ := os.Open(path)
        //files,_ := dir.Readdirnames(10000)
        //var mostRecent int
        //for i := range files {
        //        id,_ := strconv.Atoi(files[i])
        //        if id > mostRecent {
        //                mostRecent = id
        //        }
//
 //       }
  //      return(mostRecent)

	return lastMessage,nil
}

func (ms *FileStore) PostMsg(msgw MsgWrap) error {
	return nil
}

// utility functions
func updateCounter(id string, d *mgo.Database) int {
	var counter Counter
	col := d.C("counters")
	change := mgo.Change{Update: bson.M{"$inc": bson.M{"n": 1}}, New: true, Upsert: true}
	err := col.Find(bson.M{"_id": "C" + id}).Modify(change, &counter)
	if err != nil {
		panic("Failed to update counter")
	}
	fmt.Println(counter.N)
	return counter.N
}

func getCounter(id string, d *mgo.Database) int {
	var counter Counter
	col := d.C("counters")
	err := col.Find(bson.M{"_id": "C" + id}).One(&counter)
	if err != nil {
		return (0)
	}
	fmt.Println(counter.N)
	return counter.N
}

func dropPrivs(uid int) {
	if syscall.Getuid() == 0 {
		fmt.Printf("Dropping privileges.")
		syscall.Setuid(uid)
	}
}

func validateHash(hash string) {
	match, _ := regexp.MatchString("^\\w{40}$", hash)
	if !match {
		panic("Invalid hash circle key.")
	}
}

func catchPanic(c *goweb.Context, s *mgo.Session) {

	if r := recover(); r != nil {
		fmt.Printf("%s\n", r)
		s.Close()
		goweb.AddFormatter(&goweb.JsonFormatter{})
		c.RespondWithError(400)
		recover()
	}
	s.Close()
}

// handler functions
func PostMsg(c *goweb.Context, store MtsnStore) {
	var m Msg
	var mw MsgWrap

	body, _ := ioutil.ReadAll(c.Request.Body)
	json.Unmarshal(body, &m)
	mw.Content = m
	err := store.PostMsg(mw)
	if err != nil {
		goweb.AddFormatter(&goweb.JsonFormatter{})
		c.RespondWithStatus(500)
		return
	}
}

func GetMsgRange(c *goweb.Context, store MtsnStore) {
	ids := strings.Split(c.PathParams["id"], "-")
	top, _ := strconv.Atoi(ids[0])
	bottom, _ := strconv.Atoi(ids[1])

	if bottom > top {
		goweb.AddFormatter(&goweb.JsonFormatter{})
		c.RespondWithStatus(403)
		return
	}

	delta := top - bottom
	if delta > 30 {
		goweb.AddFormatter(&goweb.JsonFormatter{})
		c.RespondWithStatus(403)
		return
	}

	msgs,err := store.GetMsgs(top,bottom)

	if err != nil {
		goweb.AddFormatter(&goweb.JsonFormatter{})
		c.RespondWithStatus(500)
		return
	}

	msgBytes, _ := json.Marshal(msgs)
	c.ResponseWriter.Header().Set("Content-Type", "application/json")
	c.ResponseWriter.Write(msgBytes)
}

func GetOneMsg(c *goweb.Context, store MtsnStore) {
	id, _ := strconv.Atoi(c.PathParams["id"])

	mw,err := store.GetMsg(id)

	if err != nil {
		goweb.AddFormatter(&goweb.JsonFormatter{})
		c.RespondWithNotFound()
		return
	}

	c.ResponseWriter.Header().Set("Last-Modified", mw.Timestamp)
	msgBytes, _ := json.Marshal(mw.Content)
	c.ResponseWriter.Header().Set("Content-Type", "application/json")
	c.ResponseWriter.Write(msgBytes)
}

func GetMsg(c *goweb.Context, store MtsnStore) {

	if strings.Contains(c.PathParams["id"], "-") {
		GetMsgRange(c, store)
	} else {
		GetOneMsg(c, store)
	}

}

func GetLastMsg(c *goweb.Context, store MtsnStore) {

	lastMsg,_ := store.GetLastMsg()
	b, _ := json.Marshal(lastMsg)
	c.ResponseWriter.Header().Set("Content-Type", "application/json")
	c.ResponseWriter.Write(b)
}

// expire goroutine
func ExpireMessageLoop(d *mgo.Database) {
	for true {
		fmt.Printf("Searching for expired messages...\n")
		collections, _ := d.CollectionNames()
		for _, collection := range collections {
			if !strings.HasPrefix(collection, "C") {
				continue
			}

			var msgs []MsgWrap
			fmt.Printf("Expiring %s\n", collection)
			col := d.C(collection)

			// 3  months old and they are gone
			delBoundary := time.Now().Add(-time.Hour * 2160)
			msgQuery := col.Find(bson.M{"time": bson.M{"$lte": delBoundary}})
			msgQuery.All(&msgs)

			for _, msgW := range msgs {
				fmt.Printf("Deleted id %d timestamp %s\n", msgW.Id, msgW.Timestamp)
				err := col.Remove(bson.M{"_id": msgW.Id})
				if err != nil {
					fmt.Printf("Error expiring %d in %s.", msgW.Id, collection)
					continue
				}
			}

		}
		time.Sleep(time.Hour)
	}
}

func main() {

	runtime.GOMAXPROCS(runtime.NumCPU())

	var (
		port int
		ip   string
		db   string
		uid  int
	)
	flag.IntVar(&port, "port", 80, "Port to bind on.")
	flag.StringVar(&ip, "ip", "127.0.0.1", "IP to bind to")
	flag.StringVar(&db, "db", "muteswan", "MongoDB database to use")
	flag.IntVar(&uid, "uid", 1001, "User to drop privileges")
	flag.Parse()

	fmt.Printf("Port is %d\n", port)
	fmt.Printf("IP is %s\n", ip)
	fmt.Printf("DB is %s\n", db)
	fmt.Printf("UID is %d\n", uid)

	session, err := mgo.Dial("localhost")
	if err != nil {
		panic(err)
	}
	defer session.Close()

	session.SetMode(mgo.Monotonic, true)

	go ExpireMessageLoop(session.DB(db))

	// fetch a message, or a message range
	// e.g. GET /1e7db1c00e4036d9ea426e5875c355184578ab2d/200
	// or   GET /1e7db1c00e4036d9ea426e5875c355184578ab2d/200-190
	goweb.MapFunc("/{hash}/{id}", func(c *goweb.Context) {
		s := session.Copy()
		defer catchPanic(c, s)
		dropPrivs(uid)

		validateHash(c.PathParams["hash"])
		mongoStore := &MongoStore{Circle : c.PathParams["hash"], Db : s.DB(db)}

		fmt.Printf("GET %s\n", c.Request.URL.String())
		GetMsg(c, mongoStore)
	})

	// Get the last message in the circle
	// e.g. GET /1e7db1c00e4036d9ea426e5875c355184578ab2d
	goweb.MapFunc("/{hash}", func(c *goweb.Context) {
		s := session.Copy()
		defer catchPanic(c, s)
		dropPrivs(uid)

		validateHash(c.PathParams["hash"])
		mongoStore := &MongoStore{Circle : c.PathParams["hash"], Db : s.DB(db)}
		fmt.Printf("GET %s\n", c.Request.URL.String())
		GetLastMsg(c, mongoStore)
	}, goweb.GetMethod)

	// POST a new message to a circle
	// e.g. POST /1e7db1c00e4036d9ea426e5875c355184578ab2d
	goweb.MapFunc("/{hash}", func(c *goweb.Context) {
		s := session.Copy()
		defer catchPanic(c, s)
		dropPrivs(uid)

		validateHash(c.PathParams["hash"])
		mongoStore := &MongoStore{Circle : c.PathParams["hash"], Db : s.DB(db)}
		fmt.Printf("POST %s\n", c.Request.URL.String())
		PostMsg(c, mongoStore)
	}, goweb.PostMethod)

	goweb.ListenAndServe(fmt.Sprintf("%s:%d", ip, port))

}
<|endoftext|>


<commit_before>package incus

import (
	"crypto/md5"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"os/signal"
	"syscall"
	"time"

	"github.com/alexjlockwood/gcm"
	apns "github.com/anachronistic/apns"
	"github.com/gorilla/websocket"
)

const (
	writeWait                         = 5 * time.Second
	pongWait                          = 1 * time.Second
	abnormalCloseControlWriteDeadline = 1 * time.Second
	websocketReadBufferSize           = 1024
	websocketWriteBufferSize          = 1024

	// RFC 6455 Section 7
	closeCodeNormal          = 1000
	closeCodeGoingAway       = 1001
	closeCodeUnexpectedError = 1011
)

type GCMClient interface {
	Send(*gcm.Message, int) (*gcm.Response, error)
}

type Server struct {
	ID     string
	Config *Configuration
	Store  *Storage
	Stats  RuntimeStats

	timeout      time.Duration
	apnsProvider func(string) apns.APNSClient
	gcmProvider  func() GCMClient
}

func NewServer(conf *Configuration, store *Storage) *Server {
	hash := md5.New()
	io.WriteString(hash, time.Now().String())
	id := string(hash.Sum(nil))

	timeout := time.Duration(conf.GetInt("connection_timeout"))

	var runtimeStats RuntimeStats

	if conf.GetBool("datadog_enabled") {
		runtimeStats, _ = NewDatadogStats(conf.Get("datadog_host"))
		runtimeStats.LogStartup()
	} else {
		runtimeStats = &DiscardStats{}
	}

	apnsProvider := func(build string) apns.APNSClient {
		return apns.NewClient(conf.Get("apns_"+build+"_url"), conf.Get("apns_"+build+"_cert"), conf.Get("apns_"+build+"_private_key"))
	}

	gcmProvider := func() GCMClient {
		return &gcm.Sender{ApiKey: conf.Get("gcm_api_key")}
	}

	return &Server{
		ID:           id,
		Config:       conf,
		Store:        store,
		timeout:      timeout,
		Stats:        runtimeStats,
		apnsProvider: apnsProvider,
		gcmProvider:  gcmProvider,
	}
}

func (this *Server) ListenFromSockets() {
	Connect := func(w http.ResponseWriter, r *http.Request) {
		writtenCloseMessage := false

		exitSignals := make(chan os.Signal)
		signal.Notify(exitSignals, os.Interrupt, syscall.SIGTERM)
		defer signal.Stop(exitSignals)

		if r.Method != "GET" {
			http.Error(w, "Method not allowed", 405)
			return
		}
		//if r.Header.Get("Origin") != "http://"+r.Host {
		//        http.Error(w, "Origin not allowed", 403)
		//        return
		// }

		ws, err := websocket.Upgrade(w, r, nil, websocketReadBufferSize, websocketWriteBufferSize)
		if _, ok := err.(websocket.HandshakeError); ok {
			http.Error(w, "Not a websocket handshake", 400)
			return
		} else if err != nil {
			log.Println(err)
			return
		}

		defer func() {
			if !writtenCloseMessage {
				closeMessage := websocket.FormatCloseMessage(closeCodeUnexpectedError, "")
				ws.WriteControl(websocket.CloseMessage, closeMessage, time.Now().Add(abnormalCloseControlWriteDeadline))
			}

			ws.Close()
			this.Stats.LogWebsocketDisconnection()
			if DEBUG {
				log.Println("Socket Closed")
			}
		}()

		sock := newSocket(ws, nil, this, "")

		this.Stats.LogWebsocketConnection()
		if DEBUG {
			log.Printf("Socket connected via %s\n", ws.RemoteAddr())
		}
		if err := sock.Authenticate(""); err != nil {
			if DEBUG {
				log.Printf("Error: %s\n", err.Error())
			}
			return
		}

		go sock.listenForMessages()
		go sock.listenForWrites()

		if this.timeout <= 0 { // if timeout is 0 then wait forever and return when socket is done.
			select {
			case <-sock.done:
				writtenCloseMessage = closeWebsocket(closeCodeNormal, ws)
				return
			case <-exitSignals:
				writtenCloseMessage = closeWebsocket(closeCodeGoingAway, ws)
				return
			}
		}

		select {
		case <-time.After(this.timeout * time.Second):
			sock.Close()
			writtenCloseMessage = closeWebsocket(closeCodeNormal, ws)
			return
		case <-sock.done:
			writtenCloseMessage = closeWebsocket(closeCodeNormal, ws)
			return
		case <-exitSignals:
			writtenCloseMessage = closeWebsocket(closeCodeGoingAway, ws)
			return
		}
	}

	http.HandleFunc("/socket", Connect)
}

func closeWebsocket(closeCode int, ws *websocket.Conn) bool {
	closeMessage := websocket.FormatCloseMessage(closeCode, "")
	err := ws.WriteControl(websocket.CloseMessage, closeMessage, time.Now().Add(1*time.Second))
	return err == nil
}

func (this *Server) ListenFromLongpoll() {
	LpConnect := func(w http.ResponseWriter, r *http.Request) {
		exitSignals := make(chan os.Signal)
		signal.Notify(exitSignals, os.Interrupt, syscall.SIGTERM)
		defer signal.Stop(exitSignals)

		defer func() {
			r.Body.Close()
			if DEBUG {
				log.Println("Socket Closed")
			}
		}()

		sock := newSocket(nil, w, this, "")
		w.Header().Set("Access-Control-Allow-Origin", "*")
		w.Header().Set("Content-Type", "application/json")
		w.Header().Set("Cache-Control", "private, no-store, no-cache, must-revalidate, post-check=0, pre-check=0")
		w.Header().Set("Connection", "keep-alive")
		//w.Header().Set("Content-Encoding", "gzip")

		if DEBUG {
			log.Printf("Long poll connected via \n")
		}

		if err := sock.Authenticate(r.FormValue("user")); err != nil {
			if DEBUG {
				log.Printf("Error: %s\n", err.Error())
			}
			return
		}

		page := r.FormValue("page")
		if page != "" {
			if sock.Page != "" {
				this.Store.UnsetPage(sock) //remove old page if it exists
			}

			sock.Page = page
			this.Store.SetPage(sock)
		}

		command := r.FormValue("command")
		if command != "" {
			var cmd = new(CommandMsg)
			json.Unmarshal([]byte(command), cmd)

			go cmd.FromSocket(sock)
		}

		go sock.listenForWrites()

		if this.timeout > 0 {
			select {
			case <-exitSignals:
				sock.Close()
				w.WriteHeader(503)
				return
			case <-time.After(this.timeout * time.Second):
				sock.Close()
				w.WriteHeader(204)
				return
			case <-sock.done:
				w.WriteHeader(200)
				return
			}
		} else {
			select {
			case <-exitSignals:
				sock.Close()
				w.WriteHeader(503)
				return
			case <-sock.done:
				w.WriteHeader(200)
				return
			}
		}

	}

	http.HandleFunc("/lp", LpConnect)
}

func (this *Server) ListenFromRedis() {
	if !this.Config.GetBool("redis_enabled") {
		return
	}

	subReciever := make(chan []byte, 10000)
	queueReciever := make(chan []byte, 10000)

	_, err := this.Store.redis.Subscribe(subReciever, this.Config.Get("redis_message_channel"))
	if err != nil {
		log.Fatal("Couldn't subscribe to redis channel")
	}

	err = this.Store.redis.Poll(queueReciever, this.Config.Get("redis_message_queue"))
	if err != nil {
		log.Fatal("Couldn't start polling of redis queue")
	}

	if DEBUG {
		log.Println("LISENING FOR REDIS MESSAGE")
	}

	var subMessage []byte
	var pollMessage []byte
	for {
		var cmd = new(CommandMsg)

		select {
		case subMessage = <-subReciever:
			err = json.Unmarshal(subMessage, cmd)
		case pollMessage = <-queueReciever:
			err = json.Unmarshal(pollMessage, cmd)
		}

		if err != nil {
			log.Printf("Error decoding JSON: %s", err.Error())
			this.Stats.LogInvalidJSON()
		} else {
			go cmd.FromRedis(this)
		}
	}
}

func (this *Server) ListenForHTTPPings() {
	pingHandler := func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprint(w, "OK")
	}

	http.HandleFunc("/ping", pingHandler)
}

func (this *Server) SendHeartbeatsPeriodically(period time.Duration) {

	for {
		time.Sleep(period)

		clients := this.Store.Clients()

		for _, user := range clients {
			for _, sock := range user {

				if sock.isWebsocket() {
					if !sock.isClosed() {
						sock.ws.WriteControl(websocket.PingMessage, []byte{}, time.Now().Add(pongWait))
					}
				}

			}
		}
	}
}

func (this *Server) RecordStats(period time.Duration) {
	for {
		this.Stats.LogClientCount(this.Store.memory.clientCount)
		time.Sleep(period)
	}
}

func (this *Server) LogConnectedClientsPeriodically(period time.Duration) {
	for {
		log.Printf("There are %d connected clients\n", this.Store.memory.clientCount)
		time.Sleep(period)
	}
}

func (this *Server) GetAPNSClient(build string) apns.APNSClient {
	return this.apnsProvider(build)
}

func (this *Server) GetGCMClient() GCMClient {
	return this.gcmProvider()
}
<commit_msg>fix: Don't write header 200 on success<commit_after>package incus

import (
	"crypto/md5"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"os/signal"
	"syscall"
	"time"

	"github.com/alexjlockwood/gcm"
	apns "github.com/anachronistic/apns"
	"github.com/gorilla/websocket"
)

const (
	writeWait                         = 5 * time.Second
	pongWait                          = 1 * time.Second
	abnormalCloseControlWriteDeadline = 1 * time.Second
	websocketReadBufferSize           = 1024
	websocketWriteBufferSize          = 1024

	// RFC 6455 Section 7
	closeCodeNormal          = 1000
	closeCodeGoingAway       = 1001
	closeCodeUnexpectedError = 1011
)

type GCMClient interface {
	Send(*gcm.Message, int) (*gcm.Response, error)
}

type Server struct {
	ID     string
	Config *Configuration
	Store  *Storage
	Stats  RuntimeStats

	timeout      time.Duration
	apnsProvider func(string) apns.APNSClient
	gcmProvider  func() GCMClient
}

func NewServer(conf *Configuration, store *Storage) *Server {
	hash := md5.New()
	io.WriteString(hash, time.Now().String())
	id := string(hash.Sum(nil))

	timeout := time.Duration(conf.GetInt("connection_timeout"))

	var runtimeStats RuntimeStats

	if conf.GetBool("datadog_enabled") {
		runtimeStats, _ = NewDatadogStats(conf.Get("datadog_host"))
		runtimeStats.LogStartup()
	} else {
		runtimeStats = &DiscardStats{}
	}

	apnsProvider := func(build string) apns.APNSClient {
		return apns.NewClient(conf.Get("apns_"+build+"_url"), conf.Get("apns_"+build+"_cert"), conf.Get("apns_"+build+"_private_key"))
	}

	gcmProvider := func() GCMClient {
		return &gcm.Sender{ApiKey: conf.Get("gcm_api_key")}
	}

	return &Server{
		ID:           id,
		Config:       conf,
		Store:        store,
		timeout:      timeout,
		Stats:        runtimeStats,
		apnsProvider: apnsProvider,
		gcmProvider:  gcmProvider,
	}
}

func (this *Server) ListenFromSockets() {
	Connect := func(w http.ResponseWriter, r *http.Request) {
		writtenCloseMessage := false

		exitSignals := make(chan os.Signal)
		signal.Notify(exitSignals, os.Interrupt, syscall.SIGTERM)
		defer signal.Stop(exitSignals)

		if r.Method != "GET" {
			http.Error(w, "Method not allowed", 405)
			return
		}
		//if r.Header.Get("Origin") != "http://"+r.Host {
		//        http.Error(w, "Origin not allowed", 403)
		//        return
		// }

		ws, err := websocket.Upgrade(w, r, nil, websocketReadBufferSize, websocketWriteBufferSize)
		if _, ok := err.(websocket.HandshakeError); ok {
			http.Error(w, "Not a websocket handshake", 400)
			return
		} else if err != nil {
			log.Println(err)
			return
		}

		defer func() {
			if !writtenCloseMessage {
				closeMessage := websocket.FormatCloseMessage(closeCodeUnexpectedError, "")
				ws.WriteControl(websocket.CloseMessage, closeMessage, time.Now().Add(abnormalCloseControlWriteDeadline))
			}

			ws.Close()
			this.Stats.LogWebsocketDisconnection()
			if DEBUG {
				log.Println("Socket Closed")
			}
		}()

		sock := newSocket(ws, nil, this, "")

		this.Stats.LogWebsocketConnection()
		if DEBUG {
			log.Printf("Socket connected via %s\n", ws.RemoteAddr())
		}
		if err := sock.Authenticate(""); err != nil {
			if DEBUG {
				log.Printf("Error: %s\n", err.Error())
			}
			return
		}

		go sock.listenForMessages()
		go sock.listenForWrites()

		if this.timeout <= 0 { // if timeout is 0 then wait forever and return when socket is done.
			select {
			case <-sock.done:
				writtenCloseMessage = closeWebsocket(closeCodeNormal, ws)
				return
			case <-exitSignals:
				writtenCloseMessage = closeWebsocket(closeCodeGoingAway, ws)
				return
			}
		}

		select {
		case <-time.After(this.timeout * time.Second):
			sock.Close()
			writtenCloseMessage = closeWebsocket(closeCodeNormal, ws)
			return
		case <-sock.done:
			writtenCloseMessage = closeWebsocket(closeCodeNormal, ws)
			return
		case <-exitSignals:
			writtenCloseMessage = closeWebsocket(closeCodeGoingAway, ws)
			return
		}
	}

	http.HandleFunc("/socket", Connect)
}

func closeWebsocket(closeCode int, ws *websocket.Conn) bool {
	closeMessage := websocket.FormatCloseMessage(closeCode, "")
	err := ws.WriteControl(websocket.CloseMessage, closeMessage, time.Now().Add(1*time.Second))
	return err == nil
}

func (this *Server) ListenFromLongpoll() {
	LpConnect := func(w http.ResponseWriter, r *http.Request) {
		exitSignals := make(chan os.Signal)
		signal.Notify(exitSignals, os.Interrupt, syscall.SIGTERM)
		defer signal.Stop(exitSignals)

		defer func() {
			r.Body.Close()
			if DEBUG {
				log.Println("Socket Closed")
			}
		}()

		sock := newSocket(nil, w, this, "")
		w.Header().Set("Access-Control-Allow-Origin", "*")
		w.Header().Set("Content-Type", "application/json")
		w.Header().Set("Cache-Control", "private, no-store, no-cache, must-revalidate, post-check=0, pre-check=0")
		w.Header().Set("Connection", "keep-alive")
		//w.Header().Set("Content-Encoding", "gzip")

		if DEBUG {
			log.Printf("Long poll connected via \n")
		}

		if err := sock.Authenticate(r.FormValue("user")); err != nil {
			if DEBUG {
				log.Printf("Error: %s\n", err.Error())
			}
			return
		}

		page := r.FormValue("page")
		if page != "" {
			if sock.Page != "" {
				this.Store.UnsetPage(sock) //remove old page if it exists
			}

			sock.Page = page
			this.Store.SetPage(sock)
		}

		command := r.FormValue("command")
		if command != "" {
			var cmd = new(CommandMsg)
			json.Unmarshal([]byte(command), cmd)

			go cmd.FromSocket(sock)
		}

		go sock.listenForWrites()

		if this.timeout > 0 {
			select {
			case <-exitSignals:
				sock.Close()
				w.WriteHeader(503)
				return
			case <-time.After(this.timeout * time.Second):
				sock.Close()
				w.WriteHeader(204)
				return
			case <-sock.done:
				// No need to write 200, as it's already written implicitly.
				return
			}
		} else {
			select {
			case <-exitSignals:
				sock.Close()
				w.WriteHeader(503)
				return
			case <-sock.done:
				// No need to write 200, as it's already written implicitly.
				return
			}
		}

	}

	http.HandleFunc("/lp", LpConnect)
}

func (this *Server) ListenFromRedis() {
	if !this.Config.GetBool("redis_enabled") {
		return
	}

	subReciever := make(chan []byte, 10000)
	queueReciever := make(chan []byte, 10000)

	_, err := this.Store.redis.Subscribe(subReciever, this.Config.Get("redis_message_channel"))
	if err != nil {
		log.Fatal("Couldn't subscribe to redis channel")
	}

	err = this.Store.redis.Poll(queueReciever, this.Config.Get("redis_message_queue"))
	if err != nil {
		log.Fatal("Couldn't start polling of redis queue")
	}

	if DEBUG {
		log.Println("LISENING FOR REDIS MESSAGE")
	}

	var subMessage []byte
	var pollMessage []byte
	for {
		var cmd = new(CommandMsg)

		select {
		case subMessage = <-subReciever:
			err = json.Unmarshal(subMessage, cmd)
		case pollMessage = <-queueReciever:
			err = json.Unmarshal(pollMessage, cmd)
		}

		if err != nil {
			log.Printf("Error decoding JSON: %s", err.Error())
			this.Stats.LogInvalidJSON()
		} else {
			go cmd.FromRedis(this)
		}
	}
}

func (this *Server) ListenForHTTPPings() {
	pingHandler := func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprint(w, "OK")
	}

	http.HandleFunc("/ping", pingHandler)
}

func (this *Server) SendHeartbeatsPeriodically(period time.Duration) {

	for {
		time.Sleep(period)

		clients := this.Store.Clients()

		for _, user := range clients {
			for _, sock := range user {

				if sock.isWebsocket() {
					if !sock.isClosed() {
						sock.ws.WriteControl(websocket.PingMessage, []byte{}, time.Now().Add(pongWait))
					}
				}

			}
		}
	}
}

func (this *Server) RecordStats(period time.Duration) {
	for {
		this.Stats.LogClientCount(this.Store.memory.clientCount)
		time.Sleep(period)
	}
}

func (this *Server) LogConnectedClientsPeriodically(period time.Duration) {
	for {
		log.Printf("There are %d connected clients\n", this.Store.memory.clientCount)
		time.Sleep(period)
	}
}

func (this *Server) GetAPNSClient(build string) apns.APNSClient {
	return this.apnsProvider(build)
}

func (this *Server) GetGCMClient() GCMClient {
	return this.gcmProvider()
}
<|endoftext|>


<commit_before>// Package server offers a simple server with logging and configs
package server

import (
	"fmt"
	"log"
	"net/http"
	"os"
)

// TODO: Graceful restart would be nice - http://grisha.org/blog/2014/06/03/graceful-restart-in-golang/

// Logger interface for a simple logger (the stdlib log pkg and the fragmenta log pkg conform)
type Logger interface {
	Printf(format string, args ...interface{})
}

// Server holds the config and logger for the app
type Server struct {
	// Our internal logger instance
	Logger Logger

	// Which port to serve on
	port int

	// Which env mode we're in, read from ENV variable
	production bool

	// Server configs - access with Config(string)
	configProduction  map[string]string
	configDevelopment map[string]string
	configTest        map[string]string
}

// New creates a new server instance
func New() (*Server, error) {

	// Check environment variable to see if we are in production mode
	prod := false
	if os.Getenv("FRAG_ENV") == "production" {
		prod = true
	}

	// Set up a new server
	s := &Server{
		port:              3000,
		production:        prod,
		configProduction:  make(map[string]string),
		configDevelopment: make(map[string]string),
		configTest:        make(map[string]string),
		Logger:            log.New(os.Stderr, "fragmenta: ", log.LstdFlags), // default to a stderr logger
	}

	err := s.readConfig()
	if err != nil {
		// Run the setup, till we have collected enough information
		s.runSetup()
		return s, err
	}
	err = s.readArguments()
	if err != nil {
		return s, err
	}

	return s, err
}

// Logf logs the message with the given arguments to our internal logger
func (s *Server) Logf(format string, v ...interface{}) {
	s.Logger.Printf(format, v...)
}

// Log logs the message to our internal logger
func (s *Server) Log(message string) {
	s.Logf(message)
}

// Fatalf the message with the given arguments to our internal logger, and then exits with status 1
func (s *Server) Fatalf(format string, v ...interface{}) {
	s.Logger.Printf(format, v...)

	// Now exit
	os.Exit(1)
}

// Fatal logs the message, and then exits with status 1
func (s *Server) Fatal(format string) {
	s.Fatalf(format)
}

// Start starts the http server on our given port
func (s *Server) Start() error {
	p := fmt.Sprintf(":%d", s.port)
	return http.ListenAndServe(p, nil)

}
<commit_msg>Added server.Timef function<commit_after>// Package server offers a simple server with logging and configs
package server

import (
	"fmt"
	"log"
	"net/http"
	"os"
	"time"
)

// TODO: Graceful restart would be nice - http://grisha.org/blog/2014/06/03/graceful-restart-in-golang/

// Logger interface for a simple logger (the stdlib log pkg and the fragmenta log pkg conform)
type Logger interface {
	Printf(format string, args ...interface{})
}

// Server holds the config and logger for the app
type Server struct {
	// Our internal logger instance
	Logger Logger

	// Which port to serve on
	port int

	// Which env mode we're in, read from ENV variable
	production bool

	// Server configs - access with Config(string)
	configProduction  map[string]string
	configDevelopment map[string]string
	configTest        map[string]string
}

// New creates a new server instance
func New() (*Server, error) {

	// Check environment variable to see if we are in production mode
	prod := false
	if os.Getenv("FRAG_ENV") == "production" {
		prod = true
	}

	// Set up a new server
	s := &Server{
		port:              3000,
		production:        prod,
		configProduction:  make(map[string]string),
		configDevelopment: make(map[string]string),
		configTest:        make(map[string]string),
		Logger:            log.New(os.Stderr, "fragmenta: ", log.LstdFlags), // default to a stderr logger
	}

	err := s.readConfig()
	if err != nil {
		// Run the setup, till we have collected enough information
		s.runSetup()
		return s, err
	}
	err = s.readArguments()
	if err != nil {
		return s, err
	}

	return s, err
}

// Logf logs the message with the given arguments to our internal logger
func (s *Server) Logf(format string, v ...interface{}) {
	s.Logger.Printf(format, v...)
}

// Log logs the message to our internal logger
func (s *Server) Log(message string) {
	s.Logf(message)
}

// Fatalf the message with the given arguments to our internal logger, and then exits with status 1
func (s *Server) Fatalf(format string, v ...interface{}) {
	s.Logger.Printf(format, v...)

	// Now exit
	os.Exit(1)
}

// Fatal logs the message, and then exits with status 1
func (s *Server) Fatal(format string) {
	s.Fatalf(format)
}

// Timef logs a time since starting, when used with defer at the start of a function to time
// Usage: defer s.Timef("My message %s",time.Now())
func (s *Server) Timef(format string, start time.Time) {
	end := time.Since(start).String()
	s.Logf(format, end)
}

// Start starts the http server on our given port
func (s *Server) Start() error {
	p := fmt.Sprintf(":%d", s.port)
	return http.ListenAndServe(p, nil)

}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/boltdb/bolt"
	"log"
	"net/http"
	"os"
	"time"
)

const version = "0.0.1"

func main() {
	fmt.Println("JAMA Server version ", version)
	fmt.Println("[*] Starting server...")

	StartServer()
}

func StartServer() {
	fmt.Println("[+] Loading BoltDB")
	_, err := bolt.Open("my.db", 0600, &bolt.Options{Timeout: 1 * time.Second})
	if err != nil {
		fmt.Println("[!] Error opening BoltDB database file")
		os.Exit(1)
	}
	fmt.Println("[+] BoltDB loaded")

	http.HandleFunc("/", HomeHandler)            // Return index.html
	http.HandleFunc("/favicon.ico", iconHandler) // Return favicon

	http.HandleFunc("/api/", apiHandler)               // Return API reference
	http.HandleFunc("/api/user", apiUserHandler)       // Handle all user API requests
	http.HandleFunc("/api/message", apiMessageHandler) // Handle all message API requests
	http.HandleFunc("/api/auth", apiAuthHandler)       // Handle all auth API requests

	log.Fatal(http.ListenAndServe(":8080", nil))
	return
}

func HomeHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Println("[+] Serving index.html")
	filename := "index.html"
	http.ServeFile(w, r, filename)
}

func iconHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Println("[+] Serving favicon.ico")
	filename := "favicon.ico"
	http.ServeFile(w, r, filename)
}

func apiHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Println("[+] Serving api.html")
	filename := "api.html"
	http.ServeFile(w, r, filename)
}
<commit_msg>Store BoltDB instange globally.<commit_after>package main

import (
	"fmt"
	"github.com/boltdb/bolt"
	"log"
	"net/http"
	"os"
	"time"
)

const version = "0.0.1"

var db *bolt.DB

func main() {
	fmt.Println("JAMA Server version ", version)
	fmt.Println("[*] Starting server...")

	StartServer()
}

func StartServer() {
	fmt.Println("[+] Loading BoltDB")
	d, err := bolt.Open("my.db", 0600, &bolt.Options{Timeout: 1 * time.Second})
	if err != nil {
		fmt.Println("[!] Error opening BoltDB database file")
		os.Exit(1)
	}
	db = d
	fmt.Println("[+] BoltDB loaded")

	http.HandleFunc("/", HomeHandler)            // Return index.html
	http.HandleFunc("/favicon.ico", iconHandler) // Return favicon

	http.HandleFunc("/api/", apiHandler)               // Return API reference
	http.HandleFunc("/api/user", apiUserHandler)       // Handle all user API requests
	http.HandleFunc("/api/message", apiMessageHandler) // Handle all message API requests
	http.HandleFunc("/api/auth", apiAuthHandler)       // Handle all auth API requests

	log.Fatal(http.ListenAndServe(":8080", nil))
	return
}

func HomeHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Println("[+] Serving index.html")
	filename := "index.html"
	http.ServeFile(w, r, filename)
}

func iconHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Println("[+] Serving favicon.ico")
	filename := "favicon.ico"
	http.ServeFile(w, r, filename)
}

func apiHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Println("[+] Serving api.html")
	filename := "api.html"
	http.ServeFile(w, r, filename)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/speps/go-hashids"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"
)

const DEFAULT_RESPONSE = `{"default": "response"}`
const MAX_DELAY = 99
const MIN_STATUS = 100
const MAX_STATUS = 599

var REDIRECT_STATUS map[int]bool = map[int]bool{301: true, 302: true}

const MAX_GENERATE_SITE_NAME_ATTEMPTS = 5
const DURATION_BETWEEN_GENERATE_SITE_NAME_ATTEMPTS = time.Duration(10) * time.Millisecond

type Server struct {
	config  *Config
	storage *Storage
	hasher  *hashids.HashID
}

func NewServer(config *Config) *Server {
	storage, err := NewStorage(config.driver, config.dataSource)
	if err != nil {
		log.Fatal(err)
	}

	server := &Server{config: config, storage: storage,
		hasher: NewHasher(config.siteSalt, config.minSiteLength)}
	if config.createDefaultRules {
		server.CreateDefaultRules()
	}
	return server
}

func NewHasher(salt string, minLength int) *hashids.HashID {
	hd := hashids.NewData()
	hd.Salt = salt
	hd.MinLength = minLength
	hd.Alphabet = "abcdefghijklmnopqrstuvwxyz1234567890"
	return hashids.NewWithData(hd)
}

func (server *Server) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	log.Printf("%s %s", req.Method, req.URL.Path)
	allowCrossDomainRequests(w, req)
	switch {
	case server.isOptionsRequest(req):
		// do nothing
	case server.isCreateRequest(req):
		server.createSite(w, req)
	case server.isConfigRequest(req):
		server.createRuleFromRequest(server.GetKey(req), w, req)
	default:
		server.respondFromRule(w, req)
	}
}

func (server *Server) respondFromRule(w http.ResponseWriter, req *http.Request) {
	rule, found, err := server.storage.FindRuleMatching(server.GetKey(req), req)
	if err != nil {
		log.Print(err)
		http.Error(w, "Internal error. For real.", 500)
		return
	}

	if found {
		ApplyRule(rule, w)
	} else {
		http.Error(w, "No rule. For real.", 404)
	}
}

func (server *Server) isOptionsRequest(req *http.Request) bool {
	return req.Method == "OPTIONS"
}

func (server *Server) isCreateRequest(r *http.Request) bool {
	if r.Method != "POST" {
		return false
	}

	if server.IsSingleDomain() {
		return false
	}
	return GetSubdomain(r.Host) == "create"
}

func (server *Server) isConfigRequest(r *http.Request) bool {
	if r.Method != "POST" {
		return false
	}
	if server.IsSingleDomain() {
		return strings.HasPrefix(r.URL.Path, server.config.singleDomainUrlPath)

	}
	return strings.HasPrefix(GetSubdomain(r.Host), "admin-")
}

func (server *Server) IsSingleDomain() bool {
	return server.config.singleDomainUrlPath != ""
}

func (server *Server) GetKey(r *http.Request) string {
	if server.IsSingleDomain() {
		return ""
	}
	if server.isConfigRequest(r) {
		return strings.TrimPrefix(GetSubdomain(r.Host), "admin-")
	}
	return GetSubdomain(r.Host)
}

// TODO: check crossbrowser compatibility
func allowCrossDomainRequests(w http.ResponseWriter, r *http.Request) {
	header := w.Header()
	header.Set("Access-Control-Allow-Origin", "*")
	header.Set("Access-Control-Allow-Credentials", "true")
	header["Access-Control-Allow-Headers"] = r.Header["Access-Control-Request-Headers"]
}

func GetSubdomain(url string) string {
	return strings.Split(url, ".")[0]
}

func (server *Server) createSite(w http.ResponseWriter, req *http.Request) {
	site, err := server.generateSiteName(MAX_GENERATE_SITE_NAME_ATTEMPTS)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	server.createRuleFromRequest(site, w, req)
}

func (server *Server) createRuleFromRequest(subdomain string, w http.ResponseWriter, r *http.Request) {
	payload, err := ioutil.ReadAll(r.Body)
	if err != nil {
		io.WriteString(w, fmt.Sprintf("ERROR: %s", err))
		return
	}

	values, err := url.ParseQuery(r.URL.RawQuery)
	if err != nil {
		io.WriteString(w, fmt.Sprintf("ERROR: %s", err))
		return
	}
	delay := 0
	path := server.GetConfigPath(r)
	delay, _ = strconv.Atoi(values.Get("delay"))
	err = server.storage.UpsertRule(&Rule{site: subdomain, responseStatus: 200, headers: EmptyHeader(),
		path: path, method: values.Get("method"),
		responseBody: string(payload), delay: time.Duration(delay) * time.Second})
	log.Print(err)
	io.WriteString(w, fmt.Sprintf("Created domain %s\n", subdomain))
}

func (server *Server) GetConfigPath(r *http.Request) string {
	if server.IsSingleDomain() {
		return "/" + strings.TrimPrefix(r.URL.Path, server.config.singleDomainUrlPath)
	}
	return r.URL.Path
}

func (server *Server) generateSiteName(maxAttempts uint) (string, error) {
	var err error
	for ; maxAttempts > 0; maxAttempts-- {
		site, err := server.generateSiteNameFrom(time.Now().UTC().UnixNano())
		if err != nil {
			break
		}
		err = server.storage.CreateSite(site)
		if err == nil {
			return site, nil
		}
		time.Sleep(DURATION_BETWEEN_GENERATE_SITE_NAME_ATTEMPTS)
	}
	return "", err
}

func (server *Server) generateSiteNameFrom(nanoseconds int64) (string, error) {
	totalSeconds := int(nanoseconds / 1000000000)
	millisecondsPart := int((nanoseconds / 1000000) % 1000)
	return server.hasher.Encode([]int{totalSeconds, millisecondsPart})
}

func ApplyRule(rule *Rule, w http.ResponseWriter) {
	log.Printf("sleeping for %v", rule.delay)
	time.Sleep(rule.delay)

	AddHeaders(rule.headers, w)
	w.WriteHeader(rule.responseStatus)
	io.WriteString(w, rule.responseBody)
}

func AddHeaders(header map[string]string, w http.ResponseWriter) {
	responseHeader := w.Header()
	for key, value := range header {
		responseHeader.Add(key, value)
	}
}

func (server *Server) CreateDefaultRules() {
	server.CreateDelayRules()
	server.CreateStatusRules()
}

func (server *Server) CreateDelayRules() {
	for delay := 0; delay <= MAX_DELAY; delay++ {
		delayHost := strconv.Itoa(delay)
		delayInSeconds := time.Duration(delay) * time.Second

		server.storage.UpsertRule(&Rule{site: delayHost, headers: EmptyHeader(), delay: delayInSeconds,
			responseStatus: 200, responseBody: DEFAULT_RESPONSE,
		})
	}
}

func EmptyHeader() map[string]string {
	return make(map[string]string)
}

func (server *Server) CreateStatusRules() {
	for status := MIN_STATUS; status <= MAX_STATUS; status++ {
		statusHost := strconv.Itoa(status)
		header := server.HeaderFor(status)
		server.storage.UpsertRule(&Rule{site: statusHost, responseStatus: status,
			headers: header, responseBody: DEFAULT_RESPONSE})
	}
}

func (server *Server) HeaderFor(status int) map[string]string {
	_, isRedirect := REDIRECT_STATUS[status]
	if isRedirect {
		// TODO: check that protocol-independent location is legal HTTP
		// TODO: header should respect current port
		host := fmt.Sprintf("//0.goslow.link")
		return map[string]string{"Location": host}
	}
	return EmptyHeader()
}

func (server *Server) ListenAndServe() error {
	log.Printf("listening on %s", server.config.address)
	return http.ListenAndServe(server.config.address, server)
}
<commit_msg>Refactor server<commit_after>package main

import (
	"fmt"
	"github.com/speps/go-hashids"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"
)

const DEFAULT_RESPONSE = `{"default": "response"}`
const MAX_DELAY = 99
const MIN_STATUS = 100
const MAX_STATUS = 599

var REDIRECT_STATUS map[int]bool = map[int]bool{301: true, 302: true}
var EMPTY_HEADERS map[string]string = make(map[string]string)

const MAX_GENERATE_SITE_NAME_ATTEMPTS = 5
const DURATION_BETWEEN_GENERATE_SITE_NAME_ATTEMPTS = time.Duration(10) * time.Millisecond

type Server struct {
	config  *Config
	storage *Storage
	hasher  *hashids.HashID
}

func NewServer(config *Config) *Server {
	storage, err := NewStorage(config.driver, config.dataSource)
	if err != nil {
		log.Fatal(err)
	}

	server := &Server{config: config, storage: storage,
		hasher: NewHasher(config.siteSalt, config.minSiteLength)}
	if config.createDefaultRules {
		server.createDefaultRules()
	}
	return server
}

func NewHasher(salt string, minLength int) *hashids.HashID {
	hd := hashids.NewData()
	hd.Salt = salt
	hd.MinLength = minLength
	hd.Alphabet = "abcdefghijklmnopqrstuvwxyz1234567890"
	return hashids.NewWithData(hd)
}

func (server *Server) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	log.Printf("%s %s", req.Method, req.URL.Path)
	allowCrossDomainRequests(w, req)

	switch {
	case server.isOptionsRequest(req):
		// do nothing
	case server.isCreateRequest(req):
		server.createSite(w, req)
	case server.isConfigRequest(req):
		server.createRuleFromRequest(server.GetKey(req), w, req)
	default:
		server.respondFromRule(w, req)
	}
}

func (server *Server) respondFromRule(w http.ResponseWriter, req *http.Request) {
	rule, found, err := server.storage.FindRuleMatching(server.GetKey(req), req)
	if err != nil {
		log.Print(err)
		http.Error(w, "Internal error. For real.", 500)
		return
	}

	if found {
		ApplyRule(rule, w)
	} else {
		http.Error(w, "No rule. For real.", 404)
	}
}

func (server *Server) isOptionsRequest(req *http.Request) bool {
	return req.Method == "OPTIONS"
}

func (server *Server) isCreateRequest(r *http.Request) bool {
	if r.Method != "POST" {
		return false
	}

	if server.IsSingleDomain() {
		return false
	}
	return GetSubdomain(r.Host) == "create"
}

func (server *Server) isConfigRequest(r *http.Request) bool {
	if r.Method != "POST" {
		return false
	}
	if server.IsSingleDomain() {
		return strings.HasPrefix(r.URL.Path, server.config.singleDomainUrlPath)

	}
	return strings.HasPrefix(GetSubdomain(r.Host), "admin-")
}

func (server *Server) IsSingleDomain() bool {
	return server.config.singleDomainUrlPath != ""
}

func (server *Server) GetKey(r *http.Request) string {
	if server.IsSingleDomain() {
		return ""
	}
	if server.isConfigRequest(r) {
		return strings.TrimPrefix(GetSubdomain(r.Host), "admin-")
	}
	return GetSubdomain(r.Host)
}

// TODO: check crossbrowser compatibility
func allowCrossDomainRequests(w http.ResponseWriter, r *http.Request) {
	header := w.Header()
	header.Set("Access-Control-Allow-Origin", "*")
	header.Set("Access-Control-Allow-Credentials", "true")
	header["Access-Control-Allow-Headers"] = r.Header["Access-Control-Request-Headers"]
}

func GetSubdomain(url string) string {
	return strings.Split(url, ".")[0]
}

func (server *Server) createSite(w http.ResponseWriter, req *http.Request) {
	site, err := server.generateSiteName(MAX_GENERATE_SITE_NAME_ATTEMPTS)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	server.createRuleFromRequest(site, w, req)
}

func (server *Server) createRuleFromRequest(subdomain string, w http.ResponseWriter, r *http.Request) {
	payload, err := ioutil.ReadAll(r.Body)
	if err != nil {
		io.WriteString(w, fmt.Sprintf("ERROR: %s", err))
		return
	}

	values, err := url.ParseQuery(r.URL.RawQuery)
	if err != nil {
		io.WriteString(w, fmt.Sprintf("ERROR: %s", err))
		return
	}
	delay := 0
	path := server.GetConfigPath(r)
	delay, _ = strconv.Atoi(values.Get("delay"))
	err = server.storage.UpsertRule(&Rule{site: subdomain, responseStatus: http.StatusOK, headers: EMPTY_HEADERS,
		path: path, method: values.Get("method"),
		responseBody: string(payload), delay: time.Duration(delay) * time.Second})
	log.Print(err)
	io.WriteString(w, fmt.Sprintf("Created domain %s\n", subdomain))
}

func (server *Server) GetConfigPath(r *http.Request) string {
	if server.IsSingleDomain() {
		return "/" + strings.TrimPrefix(r.URL.Path, server.config.singleDomainUrlPath)
	}
	return r.URL.Path
}

func (server *Server) generateSiteName(maxAttempts uint) (string, error) {
	var err error
	for ; maxAttempts > 0; maxAttempts-- {
		site, err := server.generateSiteNameFrom(time.Now().UTC().UnixNano())
		if err != nil {
			break
		}
		err = server.storage.CreateSite(site)
		if err == nil {
			return site, nil
		}
		time.Sleep(DURATION_BETWEEN_GENERATE_SITE_NAME_ATTEMPTS)
	}
	return "", err
}

func (server *Server) generateSiteNameFrom(nanoseconds int64) (string, error) {
	totalSeconds := int(nanoseconds / 1000000000)
	millisecondsPart := int((nanoseconds / 1000000) % 1000)
	return server.hasher.Encode([]int{totalSeconds, millisecondsPart})
}

func ApplyRule(rule *Rule, w http.ResponseWriter) {
	log.Printf("sleeping for %v", rule.delay)
	time.Sleep(rule.delay)

	AddHeaders(rule.headers, w)
	w.WriteHeader(rule.responseStatus)
	io.WriteString(w, rule.responseBody)
}

func AddHeaders(header map[string]string, w http.ResponseWriter) {
	responseHeader := w.Header()
	for key, value := range header {
		responseHeader.Add(key, value)
	}
}

func (server *Server) createDefaultRules() {
	server.createDelayRules()
	server.createStatusRules()
}

func (server *Server) createDelayRules() {
	for i := 0; i <= MAX_DELAY; i++ {
		delaySite := strconv.Itoa(i)
		delay := time.Duration(i) * time.Second

		server.storage.UpsertRule(&Rule{site: delaySite, headers: EMPTY_HEADERS, delay: delay,
			responseStatus: http.StatusOK, responseBody: DEFAULT_RESPONSE,
		})
	}
}

func (server *Server) createStatusRules() {
	for status := MIN_STATUS; status <= MAX_STATUS; status++ {
		statusHost := strconv.Itoa(status)
		header := server.headerFor(status)
		server.storage.UpsertRule(&Rule{site: statusHost, responseStatus: status,
			headers: header, responseBody: DEFAULT_RESPONSE})
	}
}

func (server *Server) headerFor(status int) map[string]string {
	_, isRedirect := REDIRECT_STATUS[status]
	if isRedirect {
		// TODO: check that protocol-independent location is legal HTTP
		// TODO: header should respect current port
		host := fmt.Sprintf("//0.goslow.link")
		return map[string]string{"Location": host}
	}
	return EMPTY_HEADERS
}

func (server *Server) ListenAndServe() error {
	log.Printf("listening on %s", server.config.address)
	return http.ListenAndServe(server.config.address, server)
}
<|endoftext|>


<commit_before>package play

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"

	"github.com/gernest/utron/controller"
)

// maxSnippetSize value taken from
// https://github.com/golang/playground/blob/master/app/goplay/share.go
const maxSnippetSize = 64 * 1024

type Service struct {
	controller.BaseController
	basePath string
	Routes   []string
}

func New() controller.Controller {
	return &Service{
		basePath: "https://play.golang.org",
		Routes: []string{
			"post;/fmt;Format",
			"post;/compile;Compile",
			"post;/share;Share",
		},
	}
}

func (s *Service) Format() {
	r := s.Ctx.Request()
	b, err := ioutil.ReadAll(r.Body)
	if err != nil {
		log.Fatal(err)
	}
	o, err := s.format(b)
	if err != nil {
		log.Fatal(err)
	}
	s.Ctx.Response().Write(o)
}

func (s *Service) format(src []byte) ([]byte, error) {
	u := make(url.Values)
	u.Add("imports", "true")
	u.Add("body", string(src))
	res, err := http.PostForm(s.basePath+"/fmt", u)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	return ioutil.ReadAll(res.Body)
}

func (s *Service) compile(src []byte) ([]byte, error) {
	u := make(url.Values)
	u.Add("body", string(src))
	u.Add("version", "2")
	res, err := http.PostForm(s.basePath+"/compile", u)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	return ioutil.ReadAll(res.Body)
}

func (s *Service) Compile() {
	r := s.Ctx.Request()
	b, err := ioutil.ReadAll(r.Body)
	if err != nil {
		log.Fatal(err)
	}
	nb, err := s.format(b)
	if err != nil {
		log.Fatal(err)
	}
	fres := make(map[string]interface{})
	json.Unmarshal(nb, &fres)
	o, err := s.compile([]byte(fres["Body"].(string)))
	if err != nil {
		log.Fatal(err)
	}
	s.Ctx.Response().Write(o)
}

func (s *Service) Share() {
	r := s.Ctx.Request()
	b, err := ioutil.ReadAll(r.Body)
	if err != nil {
		log.Fatal(err)
	}
	nb, err := s.share(b)
	if err != nil {
		log.Fatal(err)
	}
	s.Ctx.Response().Write(nb)
}

func (s *Service) share(src []byte) ([]byte, error) {
	res, err := http.Post(s.basePath+"/share", "text/plain", bytes.NewReader(src))
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	return ioutil.ReadAll(res.Body)
}
<commit_msg>cleanup<commit_after>package play

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"

	"github.com/gernest/utron/controller"
)

type Service struct {
	controller.BaseController
	basePath string
	Routes   []string
}

func New() controller.Controller {
	return &Service{
		basePath: "https://play.golang.org",
		Routes: []string{
			"post;/fmt;Format",
			"post;/compile;Compile",
			"post;/share;Share",
		},
	}
}

func (s *Service) Format() {
	r := s.Ctx.Request()
	b, err := ioutil.ReadAll(r.Body)
	if err != nil {
		log.Fatal(err)
	}
	o, err := s.format(b)
	if err != nil {
		log.Fatal(err)
	}
	s.Ctx.Response().Write(o)
}

func (s *Service) format(src []byte) ([]byte, error) {
	u := make(url.Values)
	u.Add("imports", "true")
	u.Add("body", string(src))
	res, err := http.PostForm(s.basePath+"/fmt", u)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	return ioutil.ReadAll(res.Body)
}

func (s *Service) compile(src []byte) ([]byte, error) {
	u := make(url.Values)
	u.Add("body", string(src))
	u.Add("version", "2")
	res, err := http.PostForm(s.basePath+"/compile", u)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	return ioutil.ReadAll(res.Body)
}

func (s *Service) Compile() {
	r := s.Ctx.Request()
	b, err := ioutil.ReadAll(r.Body)
	if err != nil {
		log.Fatal(err)
	}
	nb, err := s.format(b)
	if err != nil {
		log.Fatal(err)
	}
	fres := make(map[string]interface{})
	json.Unmarshal(nb, &fres)
	o, err := s.compile([]byte(fres["Body"].(string)))
	if err != nil {
		log.Fatal(err)
	}
	s.Ctx.Response().Write(o)
}

func (s *Service) Share() {
	r := s.Ctx.Request()
	b, err := ioutil.ReadAll(r.Body)
	if err != nil {
		log.Fatal(err)
	}
	nb, err := s.share(b)
	if err != nil {
		log.Fatal(err)
	}
	s.Ctx.Response().Write(nb)
}

func (s *Service) share(src []byte) ([]byte, error) {
	res, err := http.Post(s.basePath+"/share", "text/plain", bytes.NewReader(src))
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	return ioutil.ReadAll(res.Body)
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/VividCortex/pm"

	"flag"
	"fmt"
	"math/rand"
	"sync"
	"time"
)

var statuses = []string{"starting",
	"waiting for requests",
	"accepting request",
	"receiving data",
	"querying database",
	"sending data",
	"sending response",
}

var pid = 0
var mutex sync.Mutex

var wg sync.WaitGroup

func SomeProcess() {
	wg.Add(1)
	go func() {
		mutex.Lock()
		pid++
		id := fmt.Sprint(pid)
		mutex.Unlock()

		defer func() {
			if r := recover(); r != nil {
				fmt.Printf("pid [%d] cancelled\n", pid)
			}
			wg.Done()
		}()

		pm.Start(id, nil, nil)
		defer pm.Done(id)

		for _, status := range statuses {
			time.Sleep(time.Duration((rand.Int()) % 7000 * int(time.Millisecond)))
			pm.Status(id, status)
		}

		SomeProcess()
	}()

}

func main() {
	port := flag.String("port", ":8081", "port string (ex. :8081)")
	flag.Parse()

	opts := pm.DefaultProclist.Options()
	opts.HttpHeaders = map[string]string{
		"Access-Control-Allow-Origin": "*",
	}
	pm.DefaultProclist.SetOptions(opts)
	go pm.ListenAndServe(*port)

	fmt.Printf("Listening on localhost%s\n", *port)

	for i := 0; i < 5; i++ {
		SomeProcess()
	}

	wg.Wait()
}
<commit_msg>deleted Options now since it's handled in pm<commit_after>package main

import (
	"github.com/VividCortex/pm"

	"flag"
	"fmt"
	"math/rand"
	"sync"
	"time"
)

var statuses = []string{"starting",
	"waiting for requests",
	"accepting request",
	"receiving data",
	"querying database",
	"sending data",
	"sending response",
}

var pid = 0
var mutex sync.Mutex

var wg sync.WaitGroup

func SomeProcess() {
	wg.Add(1)
	go func() {
		mutex.Lock()
		pid++
		id := fmt.Sprint(pid)
		mutex.Unlock()

		defer func() {
			if r := recover(); r != nil {
				fmt.Printf("pid [%d] cancelled\n", pid)
			}
			wg.Done()
		}()

		pm.Start(id, nil, nil)
		defer pm.Done(id)

		for _, status := range statuses {
			time.Sleep(time.Duration((rand.Int()) % 7000 * int(time.Millisecond)))
			pm.Status(id, status)
		}

		SomeProcess()
	}()

}

func main() {
	port := flag.String("port", ":8081", "port string (ex. :8081)")
	flag.Parse()

	go pm.ListenAndServe(*port)

	fmt.Printf("Listening on localhost%s\n", *port)

	for i := 0; i < 5; i++ {
		SomeProcess()
	}

	wg.Wait()
}
<|endoftext|>