<commit_before>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"runtime"

	"github.com/pointlander/peg/tree"
)

var (
	inline   = flag.Bool("inline", false, "parse rule inlining")
	_switch  = flag.Bool("switch", false, "replace if-else if-else like blocks with switch blocks")
	print    = flag.Bool("print", false, "directly dump the syntax tree")
	syntax   = flag.Bool("syntax", false, "print out the syntax tree")
	noast    = flag.Bool("noast", false, "disable AST")
	strict   = flag.Bool("strict", false, "treat compiler warnings as errors")
	filename = flag.String("output", "", "specify name of output file")
)

func main() {
	runtime.GOMAXPROCS(2)
	flag.Parse()

	if flag.NArg() != 1 {
		flag.Usage()
		log.Fatalf("FILE: the peg file to compile")
	}
	file := flag.Arg(0)

	buffer, err := ioutil.ReadFile(file)
	if err != nil {
		log.Fatal(err)
	}

	p := &Peg{Tree: tree.New(*inline, *_switch, *noast), Buffer: string(buffer)}
	p.Init(Pretty(true), Size(1<<15))
	if err := p.Parse(); err != nil {
		log.Fatal(err)
	}

	p.Execute()

	if *print {
		p.Print()
	}
	if *syntax {
		p.PrintSyntaxTree()
	}

	if *filename == "" {
		*filename = file + ".go"
	}
	out, err := os.OpenFile(*filename, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
	if err != nil {
		fmt.Printf("%v: %v\n", *filename, err)
		return
	}
	defer out.Close()

	p.Strict = *strict
	if err = p.Compile(*filename, os.Args, out); err != nil {
		log.Fatal(err)
	}
}
<commit_msg>added -version flag<commit_after>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"runtime"

	"github.com/pointlander/peg/tree"
)

const VERSION string = "v1.1.0"

var (
	inline   = flag.Bool("inline", false, "parse rule inlining")
	_switch  = flag.Bool("switch", false, "replace if-else if-else like blocks with switch blocks")
	print    = flag.Bool("print", false, "directly dump the syntax tree")
	syntax   = flag.Bool("syntax", false, "print out the syntax tree")
	noast    = flag.Bool("noast", false, "disable AST")
	strict   = flag.Bool("strict", false, "treat compiler warnings as errors")
	filename = flag.String("output", "", "specify name of output file")
	showVersion = flag.Bool("version", false, "print the version and exit")
)

func main() {
	runtime.GOMAXPROCS(2)
	flag.Parse()

	if *showVersion {
		fmt.Println("version:",VERSION)
		return
	}

	if flag.NArg() != 1 {
		flag.Usage()
		log.Fatalf("FILE: the peg file to compile")
	}
	file := flag.Arg(0)

	buffer, err := ioutil.ReadFile(file)
	if err != nil {
		log.Fatal(err)
	}

	p := &Peg{Tree: tree.New(*inline, *_switch, *noast), Buffer: string(buffer)}
	p.Init(Pretty(true), Size(1<<15))
	if err := p.Parse(); err != nil {
		log.Fatal(err)
	}

	p.Execute()

	if *print {
		p.Print()
	}
	if *syntax {
		p.PrintSyntaxTree()
	}

	if *filename == "" {
		*filename = file + ".go"
	}
	out, err := os.OpenFile(*filename, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
	if err != nil {
		fmt.Printf("%v: %v\n", *filename, err)
		return
	}
	defer out.Close()

	p.Strict = *strict
	if err = p.Compile(*filename, os.Args, out); err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"math/rand"
	"sync"
	"time"
)

var wg sync.WaitGroup
var wga sync.WaitGroup

type alarm struct {
	bool
	*sync.Mutex
}

func (a *alarm) Set(value bool) {
	a.Lock()
	a.bool = value
	a.Unlock()
}

type person struct {
	Name    string
	AllDone bool
	*sync.Mutex
}

func (p *person) setAllDone(value bool) {
	defer p.Unlock()
	p.Lock()
	p.AllDone = value
}

type people []person

var alrm = alarm{false, &sync.Mutex{}}
var hsld = []person{}

func main() {
	fmt.Println("Let's go for walk!")
	Bob := person{"Bob", false, &sync.Mutex{}}
	Alice := person{"Alice", false, &sync.Mutex{}}
	hsld = []person{Bob, Alice}

	for k := range hsld {
		wg.Add(1)
		go hsld[k].dotask("getting ready", 1, 3, false)
	}

	wg.Wait()

	wga.Add(1)
	go setAlarm(5)

	for k := range hsld {
		wg.Add(1)
		go hsld[k].dotask("putting on shoes", 1, 3, true)
	}
	wg.Wait()
	alrm.Lock()
	if alrm.bool {
		fmt.Println("Crap!  The alarm is already set.")
	} else {
		fmt.Println("Exiting and locking door.")
	}
	alrm.Unlock()
	wga.Wait()
}

func setAlarm(delay int) {
	fmt.Println("Arming alarm.")
	fmt.Println("Alarm is counting down.")
	time.Sleep(time.Duration(delay) * time.Second)
	alrm.Set(true)
	fmt.Println("Alarm armed.")
	for _, v := range hsld {
		v.Lock()
		if v.AllDone == false {
			fmt.Printf("Alarm set before %v was ready.\n", v.Name)
		}
		v.Unlock()
	}
	defer func() {
		wga.Done()
	}()
}

func (p *person) dotask(task string, min int, max int, setdone bool) {
	defer wg.Done()
	s := random(min, max)
	fmt.Println(p.Name, "started", task)
	time.Sleep(time.Duration(s) * time.Second)
	fmt.Println(p.Name, "spent", s, "seconds", task)
	if setdone {
		p.setAllDone(true)
	}
}

func random(min, max int) int {
	rand.Seed(time.Now().Unix())
	time.Sleep(10 * time.Microsecond)
	return rand.Intn(max-min) + min
}
<commit_msg>moved defers around<commit_after>package main

import (
	"fmt"
	"math/rand"
	"sync"
	"time"
)

var wg sync.WaitGroup
var wga sync.WaitGroup

type alarm struct {
	bool
	*sync.Mutex
}

func (a *alarm) Set(value bool) {
	defer a.Unlock()
	a.Lock()
	a.bool = value
}

type person struct {
	Name    string
	AllDone bool
	*sync.Mutex
}

func (p *person) setAllDone(value bool) {
	defer p.Unlock()
	p.Lock()
	p.AllDone = value
}

type people []person

var alrm = alarm{false, &sync.Mutex{}}
var hsld = []person{}

func main() {
	fmt.Println("Let's go for walk!")
	Bob := person{"Bob", false, &sync.Mutex{}}
	Alice := person{"Alice", false, &sync.Mutex{}}
	hsld = []person{Bob, Alice}

	for k := range hsld {
		wg.Add(1)
		go hsld[k].dotask("getting ready", 1, 3, false)
	}

	wg.Wait()

	wga.Add(1)
	go setAlarm(5)

	for k := range hsld {
		wg.Add(1)
		go hsld[k].dotask("putting on shoes", 1, 3, true)
	}
	wg.Wait()
	alrm.Lock()
	if alrm.bool {
		fmt.Println("Crap!  The alarm is already set.")
	} else {
		fmt.Println("Exiting and locking door.")
	}
	alrm.Unlock()
	wga.Wait()
}

func setAlarm(delay int) {
	defer wga.Done()
	fmt.Println("Arming alarm.")
	fmt.Println("Alarm is counting down.")
	time.Sleep(time.Duration(delay) * time.Second)
	alrm.Set(true)
	fmt.Println("Alarm armed.")
	for _, v := range hsld {
		v.Lock()
		if v.AllDone == false {
			fmt.Printf("Alarm set before %v was ready.\n", v.Name)
		}
		v.Unlock()
	}
}

func (p *person) dotask(task string, min int, max int, setdone bool) {
	defer wg.Done()
	s := random(min, max)
	fmt.Println(p.Name, "started", task)
	time.Sleep(time.Duration(s) * time.Second)
	fmt.Println(p.Name, "spent", s, "seconds", task)
	if setdone {
		p.setAllDone(true)
	}
}

func random(min, max int) int {
	rand.Seed(time.Now().Unix())
	time.Sleep(10 * time.Microsecond)
	return rand.Intn(max-min) + min
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"os/exec"
)

func main() {
	runOrQuit("docker", "pull", "busybox")
	runOrQuit("docker", "run", "busybox", "echo", "Dockerception!")
	runOrQuit("docker", "ps", "-a")
}

func runOrQuit(command string, args ...string) {
	cmd := exec.Command(command, args...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	err := cmd.Run()
	quitOnErr(err)
}

func quitOnErr(err error) {
	if err != nil {
		fmt.Printf("error ocurred: %v", err)
		os.Exit(1)
	}
}
<commit_msg>some ascii art and call docker version<commit_after>package main

import (
	"fmt"
	"os"
	"os/exec"
)

// Echo this from within the a container
const dc = `
______               _                                    _    _
|  _  \             | |                                  | |  (_)
| | | |  ___    ___ | | __  ___  _ __   ___   ___  _ __  | |_  _   ___   _ __
| | | | / _ \  / __|| |/ / / _ \| '__| / __| / _ \| '_ \ | __|| | / _ \ | '_ \
| |/ / | (_) || (__ |   < |  __/| |   | (__ |  __/| |_) || |_ | || (_) || | | |
|___/   \___/  \___||_|\_\ \___||_|    \___| \___|| .__/  \__||_| \___/ |_| |_|
                                                  | |

`

func main() {
	runOrQuit("docker", "version")
	runOrQuit("docker", "pull", "busybox")
	runOrQuit("docker", "run", "busybox", "echo", dc)
	runOrQuit("docker", "ps", "-a")
}

func runOrQuit(command string, args ...string) {
	cmd := exec.Command(command, args...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	err := cmd.Run()
	quitOnErr(err)
}

func quitOnErr(err error) {
	if err != nil {
		fmt.Printf("error ocurred: %v", err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"io/ioutil"
	"net/http"
	"os"
	//grain_capnp "zenhack.net/go/sandstorm/capnp/grain"
	//ws_capnp "zenhack.net/go/sandstorm/capnp/websession"
	"zenhack.net/go/sandstorm/grain"
	"zenhack.net/go/sandstorm/websession"
)

func chkfatal(err error) {
	if err != nil {
		panic(err)
	}
}

func getAction() string {
	if len(os.Args) != 2 {
		panic("len(os.Args) != 2")
	}
	action := os.Args[1]
	if action == "restore" {
		// We previously saved our on-creation action; load it
		// from the file.
		file, err := os.Open("/var/action")
		chkfatal(err)
		defer file.Close()
		data, err := ioutil.ReadAll(file)
		chkfatal(err)
		action = string(data)
	} else {
		// Save the action so we can figure out what it was when
		// we're restored.
		file, err := os.Create("/var/action")
		chkfatal(err)
		defer file.Close()
		data := []byte(action)
		n, err := file.Write(data)
		chkfatal(err)
		if n != len(data) {
			panic("Short read")
		}
	}
	return action
}

func main() {
	action := getAction()

	http.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
		w.Write([]byte(action))
	})
	ctx := context.Background()
	ws := websession.FromHandler(ctx, http.DefaultServeMux)
	_, err := grain.ConnectAPI(ctx, ws)
	if err != nil {
		panic(err)
	}
	<-ctx.Done()
}
<commit_msg>Add a comment<commit_after>package main

import (
	"context"
	"io/ioutil"
	"net/http"
	"os"
	//grain_capnp "zenhack.net/go/sandstorm/capnp/grain"
	//ws_capnp "zenhack.net/go/sandstorm/capnp/websession"
	"zenhack.net/go/sandstorm/grain"
	"zenhack.net/go/sandstorm/websession"
)

func chkfatal(err error) {
	if err != nil {
		panic(err)
	}
}

// Figure out what "action" from sandstorm-pkgdef.capnp created this
// grain.
func getAction() string {
	if len(os.Args) != 2 {
		panic("len(os.Args) != 2")
	}
	action := os.Args[1]
	if action == "restore" {
		// We previously saved our on-creation action; load it
		// from the file.
		file, err := os.Open("/var/action")
		chkfatal(err)
		defer file.Close()
		data, err := ioutil.ReadAll(file)
		chkfatal(err)
		action = string(data)
	} else {
		// Save the action so we can figure out what it was when
		// we're restored.
		file, err := os.Create("/var/action")
		chkfatal(err)
		defer file.Close()
		data := []byte(action)
		n, err := file.Write(data)
		chkfatal(err)
		if n != len(data) {
			panic("Short read")
		}
	}
	return action
}

func main() {
	action := getAction()

	http.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
		w.Write([]byte(action))
	})
	ctx := context.Background()
	ws := websession.FromHandler(ctx, http.DefaultServeMux)
	_, err := grain.ConnectAPI(ctx, ws)
	if err != nil {
		panic(err)
	}
	<-ctx.Done()
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"encoding/json"
	"fmt"
	"math/rand"
	"net/http"
	"os"
	"strconv"
	"time"
	"strings"
	"regexp"

	_ "github.com/go-sql-driver/mysql"
	"github.com/gorilla/mux"
)

var db *sql.DB
var err error

type UrlResponse struct {
	OriginalURL interface{} `json:"original_url"`
	ShortURL    interface{} `json:"short_url"`
}

type ErrorResponse struct {
	Error interface{} `json:"error"`
}

type Config struct {
    Database struct {
        User     string `json:"user"`
        Password string `json:"password"`
    } `json:"database"`
    Host string `json:"host"`
    Port string `json:"port"`
		Name string `json:"name"`
}

func main() {
	var config = loadConfig("config.json")

	db, err = sql.Open("mysql", "" + config.Database.User + ":" + config.Database.Password + "@tcp(" + config.Host + ":" + config.Port + ")/" + config.Name + "")
	check(err)
	defer db.Close()

	err = db.Ping()
	check(err)

	port := os.Getenv("PORT")
	if port == "" {
		port = "8080"
	}

	router := mux.NewRouter()
	router.HandleFunc("/new/", index)
	router.HandleFunc("/", index)
	http.ListenAndServe(":"+port, router)
}

func loadConfig(file string) Config {
    var config Config
    configFile, err := os.Open(file)
    if err != nil {
        fmt.Println(err.Error())
    }
    jsonParser := json.NewDecoder(configFile)
    jsonParser.Decode(&config)
    configFile.Close()
    return config
}

func index(res http.ResponseWriter, req *http.Request) {
	req.URL.Path = strings.Replace(req.URL.Path, ":/", "://", -1)
	fmt.Println(req.URL.Path)
	param := strings.Split(req.URL.Path, "/")
	if len(param) > 4 {
		createURL(res, req, param)
	} else {
		fmt.Fprint(res, "Welcome to the URL Shortener Service!\n")
	}
}

func createURL(res http.ResponseWriter, req *http.Request, param []string) {

	fmt.Println("createURL")

	posURL := strings.Split(req.URL.Path, "new/")
	originalUrl := posURL[1]

	reg, _ := regexp.Compile(`https?:\/\/(www\.)[a-zA-Z0-9_\-]+\.[(com|net|org|edu|gov|mil|aero|asia|biz|cat|coop|info|int|jobs|mobi|museum|name|post|pro|tel|travel|xxx|ac|ad|ae|af|ag|ai|al|am|an|ao|aq|ar|as|at|au|aw|ax|az|ba|bb|bd|be|bf|bg|bh|bi|bj|bm|bn|bo|br|bs|bt|bv|bw|by|bz|ca|cc|cd|cf|cg|ch|ci|ck|cl|cm|cn|co|cr|cs|cu|cv|cx|cy|cz|dd|de|dj|dk|dm|do|dz|ec|ee|eg|eh|er|es|et|eu|fi|fj|fk|fm|fo|fr|ga|gb|gd|ge|gf|gg|gh|gi|gl|gm|gn|gp|gq|gr|gs|gt|gu|gw|gy|hk|hm|hn|hr|ht|hu|id|ie|il|im|in|io|iq|ir|is|it|je|jm|jo|jp|ke|kg|kh|ki|km|kn|kp|kr|kw|ky|kz|la|lb|lc|li|lk|lr|ls|lt|lu|lv|ly|ma|mc|md|me|mg|mh|mk|ml|mm|mn|mo|mp|mq|mr|ms|mt|mu|mv|mw|mx|my|mz|na|nc|ne|nf|ng|ni|nl|no|np|nr|nu|nz|om|pa|pe|pf|pg|ph|pk|pl|pm|pn|pr|ps|pt|pw|py|qa|re|ro|rs|ru|rw|sa|sb|sc|sd|se|sg|sh|si|sj|Ja|sk|sl|sm|sn|so|sr|ss|st|su|sv|sx|sy|sz|tc|td|tf|tg|th|tj|tk|tl|tm|tn|to|tp|tr|tt|tv|tw|tz|ua|ug|uk|us|uy|uz|va|vc|ve|vg|vi|vn|vu|wf|ws|ye|yt|yu|za|zm|zw)]\/?[/a-zA-Z0-9_\-]+$`)

	if reg.MatchString(originalUrl) {
		idNum := random(0, 9999)
		idStr := strconv.Itoa(idNum)
		check(err)
		shortUrl := "https://morning-retreat-24523.herokuapp.com/" + idStr
		_, err := db.Exec("INSERT INTO urls(id, original_url, short_url) VALUES(?, ?, ?)", idNum, originalUrl, shortUrl)
		check(err)
		response := UrlResponse{OriginalURL: originalUrl, ShortURL: shortUrl}
		js, err := json.Marshal(response)
		check(err)
		res.Header().Set("Content-Type", "application/json")
		res.Write(js)
	} else {
		response := ErrorResponse{Error: "Wrong url format, make sure you have a valid protocol and real site."}
		js, err := json.Marshal(response)
		check(err)
		res.Header().Set("Content-Type", "application/json")
		res.Write(js)
	}

}

func getURL(w http.ResponseWriter, r *http.Request) {

	vars := mux.Vars(r)
	var idStr string = vars["id"]
	var idNum, err = strconv.Atoi(idStr)
	var originalUrl string
	err = db.QueryRow("SELECT original_url FROM urls WHERE id = ?", idNum).Scan(&originalUrl)
	check(err)

	http.Redirect(w, r, originalUrl, 301) //This needs to have a protocol: http or https
}

func random(min, max int) int {
	rand.Seed(time.Now().Unix())
	return rand.Intn(max-min) + min
}

func check(err error) {
	if err != nil {
		fmt.Println(err)
	}
}
<commit_msg>add SkipClean method<commit_after>package main

import (
	"database/sql"
	"encoding/json"
	"fmt"
	"math/rand"
	"net/http"
	"os"
	"strconv"
	"time"
	"regexp"

	_ "github.com/go-sql-driver/mysql"
	"github.com/gorilla/mux"
)

var db *sql.DB
var err error

type UrlResponse struct {
	OriginalURL interface{} `json:"original_url"`
	ShortURL    interface{} `json:"short_url"`
}

type ErrorResponse struct {
	Error interface{} `json:"error"`
}

type Config struct {
    Database struct {
        User     string `json:"user"`
        Password string `json:"password"`
    } `json:"database"`
    Host string `json:"host"`
    Port string `json:"port"`
		Name string `json:"name"`
}

func main() {
	var config = loadConfig("config.json")

	db, err = sql.Open("mysql", "" + config.Database.User + ":" + config.Database.Password + "@tcp(" + config.Host + ":" + config.Port + ")/" + config.Name + "")
	check(err)
	defer db.Close()

	err = db.Ping()
	check(err)

	port := os.Getenv("PORT")
	if port == "" {
		port = "8080"
	}

	router := mux.NewRouter()
	router.SkipClean(true)
	router.HandleFunc("/new/", createURL)
	router.HandleFunc("/{id}", getURL)
	router.HandleFunc("/", index)
	http.ListenAndServe(":"+port, router)
}

func loadConfig(file string) Config {
    var config Config
    configFile, err := os.Open(file)
    if err != nil {
        fmt.Println(err.Error())
    }
    jsonParser := json.NewDecoder(configFile)
    jsonParser.Decode(&config)
    configFile.Close()
    return config
}

func index(res http.ResponseWriter, req *http.Request) {
	//req.URL.Path = strings.Replace(req.URL.Path, ":/", "://", -1)
	//fmt.Println(req.URL.Path)
	//param := strings.Split(req.URL.Path, "/")

	fmt.Fprint(res, "Welcome to the URL Shortener Service!\n")
}

func createURL(res http.ResponseWriter, req *http.Request) {

	vars := mux.Vars(req)
	originalUrl := vars["url"]

	fmt.Println(originalUrl)

	reg, _ := regexp.Compile(`https?:\/\/(www\.)[a-zA-Z0-9_\-]+\.[(com|net|org|edu|gov|mil|aero|asia|biz|cat|coop|info|int|jobs|mobi|museum|name|post|pro|tel|travel|xxx|ac|ad|ae|af|ag|ai|al|am|an|ao|aq|ar|as|at|au|aw|ax|az|ba|bb|bd|be|bf|bg|bh|bi|bj|bm|bn|bo|br|bs|bt|bv|bw|by|bz|ca|cc|cd|cf|cg|ch|ci|ck|cl|cm|cn|co|cr|cs|cu|cv|cx|cy|cz|dd|de|dj|dk|dm|do|dz|ec|ee|eg|eh|er|es|et|eu|fi|fj|fk|fm|fo|fr|ga|gb|gd|ge|gf|gg|gh|gi|gl|gm|gn|gp|gq|gr|gs|gt|gu|gw|gy|hk|hm|hn|hr|ht|hu|id|ie|il|im|in|io|iq|ir|is|it|je|jm|jo|jp|ke|kg|kh|ki|km|kn|kp|kr|kw|ky|kz|la|lb|lc|li|lk|lr|ls|lt|lu|lv|ly|ma|mc|md|me|mg|mh|mk|ml|mm|mn|mo|mp|mq|mr|ms|mt|mu|mv|mw|mx|my|mz|na|nc|ne|nf|ng|ni|nl|no|np|nr|nu|nz|om|pa|pe|pf|pg|ph|pk|pl|pm|pn|pr|ps|pt|pw|py|qa|re|ro|rs|ru|rw|sa|sb|sc|sd|se|sg|sh|si|sj|Ja|sk|sl|sm|sn|so|sr|ss|st|su|sv|sx|sy|sz|tc|td|tf|tg|th|tj|tk|tl|tm|tn|to|tp|tr|tt|tv|tw|tz|ua|ug|uk|us|uy|uz|va|vc|ve|vg|vi|vn|vu|wf|ws|ye|yt|yu|za|zm|zw)]\/?[/a-zA-Z0-9_\-]+$`)

	if reg.MatchString(originalUrl) {
		idNum := random(0, 9999)
		idStr := strconv.Itoa(idNum)
		check(err)
		shortUrl := "https://morning-retreat-24523.herokuapp.com/" + idStr
		_, err := db.Exec("INSERT INTO urls(id, original_url, short_url) VALUES(?, ?, ?)", idNum, originalUrl, shortUrl)
		check(err)
		response := UrlResponse{OriginalURL: originalUrl, ShortURL: shortUrl}
		js, err := json.Marshal(response)
		check(err)
		res.Header().Set("Content-Type", "application/json")
		res.Write(js)
	} else {
		response := ErrorResponse{Error: "Wrong url format, make sure you have a valid protocol and real site."}
		js, err := json.Marshal(response)
		check(err)
		res.Header().Set("Content-Type", "application/json")
		res.Write(js)
	}

}

func getURL(w http.ResponseWriter, r *http.Request) {

	vars := mux.Vars(r)
	var idStr string = vars["id"]
	var idNum, err = strconv.Atoi(idStr)
	var originalUrl string
	err = db.QueryRow("SELECT original_url FROM urls WHERE id = ?", idNum).Scan(&originalUrl)
	check(err)

	http.Redirect(w, r, originalUrl, 301) //This needs to have a protocol: http or https
}

func random(min, max int) int {
	rand.Seed(time.Now().Unix())
	return rand.Intn(max-min) + min
}

func check(err error) {
	if err != nil {
		fmt.Println(err)
	}
}
<|endoftext|>


<commit_before>// Dinit is a mini init replacement useful for use inside Docker containers.
package main

import (
	"flag"
	"fmt"
	"log"
	"math"
	"os"
	"os/exec"
	"os/signal"
	"runtime"
	"strconv"
	"syscall"
	"time"
)

var (
	verbose     bool
	timeout     time.Duration
	maxproc     float64
	start, stop string
)

func main() {
	flag.BoolVar(&verbose, "verbose", envBool("DINIT_VERBOSE", false), "be more verbose and show stdout/stderr of commands (DINIT_VERBOSE)")
	flag.DurationVar(&timeout, "timeout", envDuration("DINIT_TIMEOUT", 10*time.Second), "time in seconds between SIGTERM and SIGKILL (DINIT_TIMEOUT)")
	flag.Float64Var(&maxproc, "maxproc", 0.0, "set GOMAXPROC to runtime.NumCPU() * maxproc, when 0.0 use GOMAXPROCS")
	flag.StringVar(&start, "start", "", "command to run during startup, non-zero exit status abort dinit")
	flag.StringVar(&stop, "stop", "", "command to run during teardown")

	flag.Usage = func() {
		fmt.Fprintln(os.Stderr, "Usage: dinit [OPTION]... CMD [CMD]...")
		fmt.Fprintln(os.Stderr, "Start CMDs by passing the environment and reap any zombies.\n")
		flag.PrintDefaults()
	}
	flag.Parse()

	if len(flag.Args()) == 0 {
		logFatalf("need at least one command")
	}

	if maxproc > 0.0 {
		numcpu := strconv.Itoa(int(math.Ceil(float64(runtime.NumCPU()) * maxproc)))
		logF("using %d as GOMAXPROCS", numcpu)
		os.Setenv("GOMAXPROCS", numcpu)
	}

	if start != "" {
		startcmd := command(start)
		if err := startcmd.Run(); err != nil {
			logFatalf("start command failed: %s", err)
		}
	}
	if stop != "" {
		stopcmd := command(stop)
		defer stopcmd.Run()
	}

	done := make(chan bool)
	cmds := run(flag.Args(), done)

	defer reaper()

	wait(done, cmds)
}

func run(args []string, done chan bool) []*exec.Cmd {
	cmds := []*exec.Cmd{}
	for _, arg := range args {
		cmd := command(arg)
		cmds = append(cmds, cmd)

		go func() {
			if err := cmd.Start(); err != nil {
				logFatalf("%s", err)
			}

			logF("pid %d started: %v", cmd.Process.Pid, cmd.Args)

			err := cmd.Wait()
			if err != nil {
				logF("pid %d, finished with error: %s", cmd.Process.Pid, err)
			} else {
				logF("pid %d, finished: %v", cmd.Process.Pid, cmd.Args)
			}
			done <- true
		}()
	}
	return cmds
}

// wait waits for commands to finish.
func wait(done chan bool, cmds []*exec.Cmd) {
	i := 0

	ints := make(chan os.Signal)
	chld := make(chan os.Signal)
	signal.Notify(ints, syscall.SIGINT, syscall.SIGTERM)
	signal.Notify(chld, syscall.SIGCHLD)

	for {
		select {
		case <-chld:
			go reaper()
		case <-done:
			i++
			if len(cmds) == i {
				return
			}
		case sig := <-ints:
			// There is a race here, because the process could have died, we don't care.
			for _, cmd := range cmds {
				logF("signal %d sent to pid %d", sig, cmd.Process.Pid)
				cmd.Process.Signal(sig)
			}

			time.Sleep(timeout)

			kill := []*os.Process{}
			for _, cmd := range cmds {
				if p, err := os.FindProcess(cmd.Process.Pid); err != nil {
					kill = append(kill, p)
				}
			}
			for _, p := range kill {
				logF("SIGKILL sent to pid %d", p.Pid)
				p.Signal(syscall.SIGKILL)
			}
		}
	}
}

func reaper() {
	for {
		var wstatus syscall.WaitStatus
		pid, err := syscall.Wait4(-1, &wstatus, 0, nil)
		if err != nil {
			return
		}
		logF("pid %d reaped", pid)
	}
}

func logF(format string, v ...interface{}) {
	if !verbose {
		return
	}
	log.Printf("dinit: "+format, v...)
}

func logFatalf(format string, v ...interface{}) {
	log.Fatalf("dinit: "+format, v...)
}
<commit_msg>Count processes exiting properly<commit_after>// Dinit is a mini init replacement useful for use inside Docker containers.
package main

import (
	"flag"
	"fmt"
	"log"
	"math"
	"os"
	"os/exec"
	"os/signal"
	"runtime"
	"strconv"
	"syscall"
	"time"
)

var (
	verbose     bool
	timeout     time.Duration
	maxproc     float64
	start, stop string
)

func main() {
	flag.BoolVar(&verbose, "verbose", envBool("DINIT_VERBOSE", false), "be more verbose and show stdout/stderr of commands (DINIT_VERBOSE)")
	flag.DurationVar(&timeout, "timeout", envDuration("DINIT_TIMEOUT", 10*time.Second), "time in seconds between SIGTERM and SIGKILL (DINIT_TIMEOUT)")
	flag.Float64Var(&maxproc, "maxproc", 0.0, "set GOMAXPROC to runtime.NumCPU() * maxproc, when 0.0 use GOMAXPROCS")
	flag.StringVar(&start, "start", "", "command to run during startup, non-zero exit status abort dinit")
	flag.StringVar(&stop, "stop", "", "command to run during teardown")

	flag.Usage = func() {
		fmt.Fprintln(os.Stderr, "Usage: dinit [OPTION]... CMD [CMD]...")
		fmt.Fprintln(os.Stderr, "Start CMDs by passing the environment and reap any zombies.\n")
		flag.PrintDefaults()
	}
	flag.Parse()

	if len(flag.Args()) == 0 {
		logFatalf("need at least one command")
	}

	if maxproc > 0.0 {
		numcpu := strconv.Itoa(int(math.Ceil(float64(runtime.NumCPU()) * maxproc)))
		logF("using %d as GOMAXPROCS", numcpu)
		os.Setenv("GOMAXPROCS", numcpu)
	}

	if start != "" {
		startcmd := command(start)
		if err := startcmd.Run(); err != nil {
			logFatalf("start command failed: %s", err)
		}
	}
	if stop != "" {
		stopcmd := command(stop)
		defer stopcmd.Run()
	}

	done := make(chan bool)
	cmds := run(flag.Args(), done)

	defer reaper()

	wait(done, cmds)
}

func run(args []string, done chan bool) []*exec.Cmd {
	cmds := []*exec.Cmd{}
	for _, arg := range args {
		cmd := command(arg)
		cmds = append(cmds, cmd)

		go func() {
			if err := cmd.Start(); err != nil {
				logFatalf("%s", err)
			}

			logF("pid %d started: %v", cmd.Process.Pid, cmd.Args)

			err := cmd.Wait()
			if err != nil {
				logF("pid %d, finished with error: %s", cmd.Process.Pid, err)
			} else {
				logF("pid %d, finished: %v", cmd.Process.Pid, cmd.Args)
			}
			done <- true
		}()
	}
	return cmds
}

// wait waits for commands to finish.
func wait(done chan bool, cmds []*exec.Cmd) {
	i := 0

	ints := make(chan os.Signal)
	chld := make(chan os.Signal)
	signal.Notify(ints, syscall.SIGINT, syscall.SIGTERM)
	signal.Notify(chld, syscall.SIGCHLD)

	for {
		select {
		case <-chld:
			go reaper()
			i++
			if len(cmds) == i {
				return
			}
		case <-done:
			i++
			if len(cmds) == i {
				return
			}
		case sig := <-ints:
			// There is a race here, because the process could have died, we don't care.
			for _, cmd := range cmds {
				logF("signal %d sent to pid %d", sig, cmd.Process.Pid)
				cmd.Process.Signal(sig)
			}

			time.Sleep(time.Second)

			kill := []*os.Process{}
			for _, cmd := range cmds {
				if p, err := os.FindProcess(cmd.Process.Pid); err != nil {
					kill = append(kill, p)
				}
			}

			if len(kill) > 0 {
				time.Sleep(timeout)
			}
			for _, p := range kill {
				logF("SIGKILL sent to pid %d", p.Pid)
				p.Signal(syscall.SIGKILL)
			}
		}
	}
}

func reaper() {
	for {
		var wstatus syscall.WaitStatus
		pid, err := syscall.Wait4(-1, &wstatus, 0, nil)
		if err != nil {
			return
		}
		logF("pid %d reaped", pid)
	}
}

func logF(format string, v ...interface{}) {
	if !verbose {
		return
	}
	log.Printf("dinit: "+format, v...)
}

func logFatalf(format string, v ...interface{}) {
	log.Fatalf("dinit: "+format, v...)
}
<|endoftext|>


<commit_before>// HTTP/2 web server with built-in support for Lua, Markdown, GCSS, Amber and JSX.
package main

import (
	"errors"
	"fmt"
	log "github.com/sirupsen/logrus"
	"io/ioutil"
	internallog "log"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"runtime/pprof"
	"strings"
	"time"

	"github.com/xyproto/pinterface"
	"github.com/xyproto/unzip"
	"github.com/yuin/gopher-lua"
)

const (
	versionString = "Algernon 0.92"
	description   = "HTTP/2 Web Server"
)

var (
	// For convenience. Set in the main function.
	serverHost      string
	dbName          string
	refreshDuration time.Duration
	fs              *FileStat
)

func main() {
	var err error

	// Will be default in Go 1.5
	runtime.GOMAXPROCS(runtime.NumCPU())

	// Temporary directory that might be used for logging, databases or file extraction
	serverTempDir, err := ioutil.TempDir("", "algernon")
	if err != nil {
		log.Fatalln(err)
	}
	defer os.RemoveAll(serverTempDir)

	// Set several configuration variables, based on the given flags and arguments
	serverHost = handleFlags(serverTempDir)

	// Version
	if showVersion {
		if !quietMode {
			fmt.Println(versionString)
		}
		os.Exit(0)
	}

	// CPU profiling
	if profileCPU != "" {
		f, err := os.Create(profileCPU)
		if err != nil {
			log.Fatal(err)
		}
		go func() {
			log.Info("Profiling CPU usage")
			pprof.StartCPUProfile(f)
		}()
		atShutdown(func() {
			pprof.StopCPUProfile()
			log.Info("Done profiling CPU usage")
		})
	}

	// Memory profiling at server shutdown
	if profileMem != "" {
		atShutdown(func() {
			f, err := os.Create(profileMem)
			defer f.Close()
			if err != nil {
				log.Fatal(err)
			}
			log.Info("Saving heap profile to ", profileMem)
			pprof.WriteHeapProfile(f)
		})
	}

	// Request handlers
	mux := http.NewServeMux()

	// Read mime data from the system, if available
	initializeMime()

	// Log to a file as JSON, if a log file has been specified
	if serverLogFile != "" {
		f, err := os.OpenFile(serverLogFile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, defaultPermissions)
		if err != nil {
			log.Error("Could not log to", serverLogFile)
			fatalExit(err)
		}
		log.SetFormatter(&log.JSONFormatter{})
		log.SetOutput(f)
	} else if quietMode {
		// If quiet mode is enabled and no log file has been specified, disable logging
		log.SetOutput(ioutil.Discard)
	}

	if quietMode {
		os.Stdout.Close()
		os.Stderr.Close()
	}

	// Create a new FileStat struct, with optional caching (for speed).
	// Clear the cache every 10 minutes.
	fs = NewFileStat(cacheFileStat, time.Minute*10)

	// Check if the given directory really is a directory
	if !fs.isDir(serverDir) {
		// Possibly a file
		filename := serverDir
		// Check if the file exists
		if fs.exists(filename) {
			// Switch based on the lowercase filename extension
			switch strings.ToLower(filepath.Ext(filename)) {
			case ".md", ".markdown":
				// Serve the given Markdown file as a static HTTP server
				serveStaticFile(filename, defaultWebColonPort)
				return
			case ".zip", ".alg":
				// Assume this to be a compressed Algernon application
				err := unzip.Extract(filename, serverTempDir)
				if err != nil {
					log.Fatalln(err)
				}
				// Use the directory where the file was extracted as the server directory
				serverDir = serverTempDir
				// If there is only one directory there, assume it's the
				// directory of the newly extracted ZIP file.
				if filenames := getFilenames(serverDir); len(filenames) == 1 {
					fullPath := filepath.Join(serverDir, filenames[0])
					if fs.isDir(fullPath) {
						// Use this as the server directory instead
						serverDir = fullPath
					}
				}
				// If there are server configuration files in the extracted
				// directory, register them.
				for _, filename := range serverConfigurationFilenames {
					configFilename := filepath.Join(serverDir, filename)
					if fs.exists(configFilename) {
						serverConfigurationFilenames = append(serverConfigurationFilenames, configFilename)
					}
				}
				// Disregard all configuration files from the current directory
				// (filenames without a path separator), since we are serving a
				// ZIP file.
				for i, filename := range serverConfigurationFilenames {
					if strings.Count(filepath.ToSlash(filename), "/") == 0 {
						// Remove the filename from the slice
						serverConfigurationFilenames = append(serverConfigurationFilenames[:i], serverConfigurationFilenames[i+1:]...)
					}
				}
			default:
				singleFileMode = true
			}
		} else {
			fatalExit(errors.New("File does not exist: " + filename))
		}
	}

	// Make a few changes to the defaults if we are serving a single file
	if singleFileMode {
		debugMode = true
		serveJustHTTP = true
	}

	// Console output
	if !quietMode && !singleFileMode && !simpleMode {
		fmt.Println(banner())
	}

	// Dividing line between the banner and output from any of the configuration scripts
	if len(serverConfigurationFilenames) > 0 && !quietMode {
		fmt.Println("--------------------------------------- - - · ·")
	}

	// Disable the database backend if the BoltDB filename is /dev/null
	if boltFilename == "/dev/null" {
		useNoDatabase = true
	}

	var perm pinterface.IPermissions // nil by default
	if !useNoDatabase {
		// Connect to a database and retrieve a Permissions struct
		perm, err = aquirePermissions()
		if err != nil {
			log.Fatalln("Could not find a usable database backend.")
		}
	}

	// Lua LState pool
	luapool := &lStatePool{saved: make([]*lua.LState, 0, 4)}
	atShutdown(func() {
		luapool.Shutdown()
	})

	// TODO: save repl history + close luapool + close logs ++ at shutdown

	// Create a cache struct for reading files (contains functions that can
	// be used for reading files, also when caching is disabled).
	// The final argument is for compressing with "fast" instead of "best".
	cache := newFileCache(cacheSize, cacheCompression, cacheMaxEntitySize)

	if singleFileMode && filepath.Ext(serverDir) == ".lua" {
		luaServerFilename = serverDir
		if luaServerFilename == "index.lua" || luaServerFilename == "data.lua" {
			log.Warn("Using " + luaServerFilename + " as a standalone server!\nYou might wish to serve a directory instead.")
		}
		serverDir = filepath.Dir(serverDir)
		singleFileMode = false
	}

	// Read server configuration script, if present.
	// The scripts may change global variables.
	var ranConfigurationFilenames []string
	for _, filename := range serverConfigurationFilenames {
		if fs.exists(filename) {
			if verboseMode {
				fmt.Println("Running configuration file: " + filename)
			}
			if err := runConfiguration(filename, perm, luapool, cache, mux, false); err != nil {
				log.Error("Could not use configuration script: " + filename)
				if perm != nil {
					fatalExit(err)
				} else {
					log.Warn("Ignoring script error since the database backend is disabled.")
				}
			}
			ranConfigurationFilenames = append(ranConfigurationFilenames, filename)
		}
	}
	// Only keep the active ones. Used when outputting server information.
	serverConfigurationFilenames = ranConfigurationFilenames

	// Run the standalone Lua server, if specified
	if luaServerFilename != "" {
		// Run the Lua server file and set up handlers
		if verboseMode {
			fmt.Println("Running Lua Server File")
		}
		if err := runConfiguration(luaServerFilename, perm, luapool, cache, mux, true); err != nil {
			log.Error("Error in Lua server script: " + luaServerFilename)
			fatalExit(err)
		}
	} else {
		// Register HTTP handler functions
		registerHandlers(mux, "/", serverDir, perm, luapool, cache, serverAddDomain)
	}

	// Set the values that has not been set by flags nor scripts
	// (and can be set by both)
	ranServerReadyFunction := finalConfiguration(serverHost)

	// If no configuration files were being ran successfully,
	// output basic server information.
	if len(serverConfigurationFilenames) == 0 {
		if !quietMode {
			fmt.Println(serverInfo())
		}
		ranServerReadyFunction = true
	}

	// Dividing line between the banner and output from any of the
	// configuration scripts. Marks the end of the configuration output.
	if ranServerReadyFunction && !quietMode {
		fmt.Println("--------------------------------------- - - · ·")
	}

	// Direct internal logging elsewhere
	internalLogFile, err := os.Open(internalLogFilename)
	defer internalLogFile.Close()

	if err != nil {
		// Could not open the internalLogFilename filename, try using another filename
		internalLogFile, err = os.OpenFile("internal.log", os.O_CREATE|os.O_APPEND|os.O_WRONLY, defaultPermissions)
		atShutdown(func() {
			// TODO This one is is special and should be closed after the other shutdown functions.
			//      Set up a "done" channel instead of sleeping.
			time.Sleep(100 * time.Millisecond)
			internalLogFile.Close()
		})
		if err != nil {
			fatalExit(fmt.Errorf("Could not write to %s nor %s.", internalLogFilename, "internal.log"))
		}
	}
	internallog.SetOutput(internalLogFile)

	// Serve filesystem events in the background.
	// Used for reloading pages when the sources change.
	// Can also be used when serving a single file.
	if autoRefreshMode {
		refreshDuration, err = time.ParseDuration(eventRefresh)
		if err != nil {
			log.Warn(fmt.Sprintf("%s is an invalid duration. Using %s instead.", eventRefresh, defaultEventRefresh))
			// Ignore the error, since defaultEventRefresh is a constant and must be parseable
			refreshDuration, _ = time.ParseDuration(defaultEventRefresh)
		}
		if autoRefreshDir != "" {
			// Only watch the autoRefreshDir, recursively
			EventServer(eventAddr, defaultEventPath, autoRefreshDir, refreshDuration, "*")
		} else {
			// Watch everything in the server directory, recursively
			EventServer(eventAddr, defaultEventPath, serverDir, refreshDuration, "*")
		}
	}

	// For communicating to and from the REPL
	ready := make(chan bool) // for when the server is up and running
	done := make(chan bool)  // for when the user wish to quit the server

	// The Lua REPL
	if !serverMode {
		// If the REPL uses readline, the SIGWINCH signal is handled
		go REPL(perm, luapool, cache, ready, done)
	} else {
		// Ignore SIGWINCH if we are not going to use a REPL
		ignoreTerminalResizeSignal()
	}

	shutdownTimeout := 10 * time.Second

	conf := &algernonServerConfig{
		productionMode:      productionMode,
		serverHost:          serverHost,
		serverAddr:          serverAddr,
		serverCert:          serverCert,
		serverKey:           serverKey,
		serveJustHTTP:       serveJustHTTP,
		serveJustHTTP2:      serveJustHTTP2,
		shutdownTimeout:     shutdownTimeout,
		internalLogFilename: internalLogFilename,
	}

	// Run the shutdown functions if graceful does not
	defer generateShutdownFunction(nil)()

	// Serve HTTP, HTTP/2 and/or HTTPS
	if err := serve(conf, mux, done, ready); err != nil {
		fatalExit(err)
	}
}
<commit_msg>Assume the given file is Markdown when -m is given<commit_after>// HTTP/2 web server with built-in support for Lua, Markdown, GCSS, Amber and JSX.
package main

import (
	"errors"
	"fmt"
	log "github.com/sirupsen/logrus"
	"io/ioutil"
	internallog "log"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"runtime/pprof"
	"strings"
	"time"

	"github.com/xyproto/pinterface"
	"github.com/xyproto/unzip"
	"github.com/yuin/gopher-lua"
)

const (
	versionString = "Algernon 0.92"
	description   = "HTTP/2 Web Server"
)

var (
	// For convenience. Set in the main function.
	serverHost      string
	dbName          string
	refreshDuration time.Duration
	fs              *FileStat
)

func main() {
	var err error

	// Will be default in Go 1.5
	runtime.GOMAXPROCS(runtime.NumCPU())

	// Temporary directory that might be used for logging, databases or file extraction
	serverTempDir, err := ioutil.TempDir("", "algernon")
	if err != nil {
		log.Fatalln(err)
	}
	defer os.RemoveAll(serverTempDir)

	// Set several configuration variables, based on the given flags and arguments
	serverHost = handleFlags(serverTempDir)

	// Version
	if showVersion {
		if !quietMode {
			fmt.Println(versionString)
		}
		os.Exit(0)
	}

	// CPU profiling
	if profileCPU != "" {
		f, err := os.Create(profileCPU)
		if err != nil {
			log.Fatal(err)
		}
		go func() {
			log.Info("Profiling CPU usage")
			pprof.StartCPUProfile(f)
		}()
		atShutdown(func() {
			pprof.StopCPUProfile()
			log.Info("Done profiling CPU usage")
		})
	}

	// Memory profiling at server shutdown
	if profileMem != "" {
		atShutdown(func() {
			f, err := os.Create(profileMem)
			defer f.Close()
			if err != nil {
				log.Fatal(err)
			}
			log.Info("Saving heap profile to ", profileMem)
			pprof.WriteHeapProfile(f)
		})
	}

	// Request handlers
	mux := http.NewServeMux()

	// Read mime data from the system, if available
	initializeMime()

	// Log to a file as JSON, if a log file has been specified
	if serverLogFile != "" {
		f, err := os.OpenFile(serverLogFile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, defaultPermissions)
		if err != nil {
			log.Error("Could not log to", serverLogFile)
			fatalExit(err)
		}
		log.SetFormatter(&log.JSONFormatter{})
		log.SetOutput(f)
	} else if quietMode {
		// If quiet mode is enabled and no log file has been specified, disable logging
		log.SetOutput(ioutil.Discard)
	}

	if quietMode {
		os.Stdout.Close()
		os.Stderr.Close()
	}

	// Create a new FileStat struct, with optional caching (for speed).
	// Clear the cache every 10 minutes.
	fs = NewFileStat(cacheFileStat, time.Minute*10)

	// Check if the given directory really is a directory
	if !fs.isDir(serverDir) {
		// Possibly a file
		filename := serverDir
		// Check if the file exists
		if fs.exists(filename) {
			if markdownMode {
				// Serve the given Markdown file as a static HTTP server
				serveStaticFile(filename, defaultWebColonPort)
				return
			}
			// Switch based on the lowercase filename extension
			switch strings.ToLower(filepath.Ext(filename)) {
			case ".md", ".markdown":
				// Serve the given Markdown file as a static HTTP server
				serveStaticFile(filename, defaultWebColonPort)
				return
			case ".zip", ".alg":
				// Assume this to be a compressed Algernon application
				err := unzip.Extract(filename, serverTempDir)
				if err != nil {
					log.Fatalln(err)
				}
				// Use the directory where the file was extracted as the server directory
				serverDir = serverTempDir
				// If there is only one directory there, assume it's the
				// directory of the newly extracted ZIP file.
				if filenames := getFilenames(serverDir); len(filenames) == 1 {
					fullPath := filepath.Join(serverDir, filenames[0])
					if fs.isDir(fullPath) {
						// Use this as the server directory instead
						serverDir = fullPath
					}
				}
				// If there are server configuration files in the extracted
				// directory, register them.
				for _, filename := range serverConfigurationFilenames {
					configFilename := filepath.Join(serverDir, filename)
					if fs.exists(configFilename) {
						serverConfigurationFilenames = append(serverConfigurationFilenames, configFilename)
					}
				}
				// Disregard all configuration files from the current directory
				// (filenames without a path separator), since we are serving a
				// ZIP file.
				for i, filename := range serverConfigurationFilenames {
					if strings.Count(filepath.ToSlash(filename), "/") == 0 {
						// Remove the filename from the slice
						serverConfigurationFilenames = append(serverConfigurationFilenames[:i], serverConfigurationFilenames[i+1:]...)
					}
				}
			default:
				singleFileMode = true
			}
		} else {
			fatalExit(errors.New("File does not exist: " + filename))
		}
	}

	// Make a few changes to the defaults if we are serving a single file
	if singleFileMode {
		debugMode = true
		serveJustHTTP = true
	}

	// Console output
	if !quietMode && !singleFileMode && !simpleMode {
		fmt.Println(banner())
	}

	// Dividing line between the banner and output from any of the configuration scripts
	if len(serverConfigurationFilenames) > 0 && !quietMode {
		fmt.Println("--------------------------------------- - - · ·")
	}

	// Disable the database backend if the BoltDB filename is /dev/null
	if boltFilename == "/dev/null" {
		useNoDatabase = true
	}

	var perm pinterface.IPermissions // nil by default
	if !useNoDatabase {
		// Connect to a database and retrieve a Permissions struct
		perm, err = aquirePermissions()
		if err != nil {
			log.Fatalln("Could not find a usable database backend.")
		}
	}

	// Lua LState pool
	luapool := &lStatePool{saved: make([]*lua.LState, 0, 4)}
	atShutdown(func() {
		luapool.Shutdown()
	})

	// TODO: save repl history + close luapool + close logs ++ at shutdown

	// Create a cache struct for reading files (contains functions that can
	// be used for reading files, also when caching is disabled).
	// The final argument is for compressing with "fast" instead of "best".
	cache := newFileCache(cacheSize, cacheCompression, cacheMaxEntitySize)

	if singleFileMode && filepath.Ext(serverDir) == ".lua" {
		luaServerFilename = serverDir
		if luaServerFilename == "index.lua" || luaServerFilename == "data.lua" {
			log.Warn("Using " + luaServerFilename + " as a standalone server!\nYou might wish to serve a directory instead.")
		}
		serverDir = filepath.Dir(serverDir)
		singleFileMode = false
	}

	// Read server configuration script, if present.
	// The scripts may change global variables.
	var ranConfigurationFilenames []string
	for _, filename := range serverConfigurationFilenames {
		if fs.exists(filename) {
			if verboseMode {
				fmt.Println("Running configuration file: " + filename)
			}
			if err := runConfiguration(filename, perm, luapool, cache, mux, false); err != nil {
				log.Error("Could not use configuration script: " + filename)
				if perm != nil {
					fatalExit(err)
				} else {
					log.Warn("Ignoring script error since the database backend is disabled.")
				}
			}
			ranConfigurationFilenames = append(ranConfigurationFilenames, filename)
		}
	}
	// Only keep the active ones. Used when outputting server information.
	serverConfigurationFilenames = ranConfigurationFilenames

	// Run the standalone Lua server, if specified
	if luaServerFilename != "" {
		// Run the Lua server file and set up handlers
		if verboseMode {
			fmt.Println("Running Lua Server File")
		}
		if err := runConfiguration(luaServerFilename, perm, luapool, cache, mux, true); err != nil {
			log.Error("Error in Lua server script: " + luaServerFilename)
			fatalExit(err)
		}
	} else {
		// Register HTTP handler functions
		registerHandlers(mux, "/", serverDir, perm, luapool, cache, serverAddDomain)
	}

	// Set the values that has not been set by flags nor scripts
	// (and can be set by both)
	ranServerReadyFunction := finalConfiguration(serverHost)

	// If no configuration files were being ran successfully,
	// output basic server information.
	if len(serverConfigurationFilenames) == 0 {
		if !quietMode {
			fmt.Println(serverInfo())
		}
		ranServerReadyFunction = true
	}

	// Dividing line between the banner and output from any of the
	// configuration scripts. Marks the end of the configuration output.
	if ranServerReadyFunction && !quietMode {
		fmt.Println("--------------------------------------- - - · ·")
	}

	// Direct internal logging elsewhere
	internalLogFile, err := os.Open(internalLogFilename)
	defer internalLogFile.Close()

	if err != nil {
		// Could not open the internalLogFilename filename, try using another filename
		internalLogFile, err = os.OpenFile("internal.log", os.O_CREATE|os.O_APPEND|os.O_WRONLY, defaultPermissions)
		atShutdown(func() {
			// TODO This one is is special and should be closed after the other shutdown functions.
			//      Set up a "done" channel instead of sleeping.
			time.Sleep(100 * time.Millisecond)
			internalLogFile.Close()
		})
		if err != nil {
			fatalExit(fmt.Errorf("Could not write to %s nor %s.", internalLogFilename, "internal.log"))
		}
	}
	internallog.SetOutput(internalLogFile)

	// Serve filesystem events in the background.
	// Used for reloading pages when the sources change.
	// Can also be used when serving a single file.
	if autoRefreshMode {
		refreshDuration, err = time.ParseDuration(eventRefresh)
		if err != nil {
			log.Warn(fmt.Sprintf("%s is an invalid duration. Using %s instead.", eventRefresh, defaultEventRefresh))
			// Ignore the error, since defaultEventRefresh is a constant and must be parseable
			refreshDuration, _ = time.ParseDuration(defaultEventRefresh)
		}
		if autoRefreshDir != "" {
			// Only watch the autoRefreshDir, recursively
			EventServer(eventAddr, defaultEventPath, autoRefreshDir, refreshDuration, "*")
		} else {
			// Watch everything in the server directory, recursively
			EventServer(eventAddr, defaultEventPath, serverDir, refreshDuration, "*")
		}
	}

	// For communicating to and from the REPL
	ready := make(chan bool) // for when the server is up and running
	done := make(chan bool)  // for when the user wish to quit the server

	// The Lua REPL
	if !serverMode {
		// If the REPL uses readline, the SIGWINCH signal is handled
		go REPL(perm, luapool, cache, ready, done)
	} else {
		// Ignore SIGWINCH if we are not going to use a REPL
		ignoreTerminalResizeSignal()
	}

	shutdownTimeout := 10 * time.Second

	conf := &algernonServerConfig{
		productionMode:      productionMode,
		serverHost:          serverHost,
		serverAddr:          serverAddr,
		serverCert:          serverCert,
		serverKey:           serverKey,
		serveJustHTTP:       serveJustHTTP,
		serveJustHTTP2:      serveJustHTTP2,
		shutdownTimeout:     shutdownTimeout,
		internalLogFilename: internalLogFilename,
	}

	// Run the shutdown functions if graceful does not
	defer generateShutdownFunction(nil)()

	// Serve HTTP, HTTP/2 and/or HTTPS
	if err := serve(conf, mux, done, ready); err != nil {
		fatalExit(err)
	}
}
<|endoftext|>


<commit_before>package sift4

import (
	"math"
	"strings"
)

// Models

type Sift4 struct {
	maxOffset                  int
	maxDistance                int
	tokenizer                  func(string) []string
	tokenMatcher               func(string, string) bool
	matchingEvaluator          func(string, string) float64
	localLengthEvaluator       func(float64) float64
	transpositionCostEvaluator func(int, int) float64
	transpositionsEvaluator    func(float64, float64) float64
}

type offset struct {
	c1    int
	c2    int
	trans bool
}

// Initialization

func New() *Sift4 {
	return &Sift4{
		maxOffset:                  0,
		maxDistance:                0,
		tokenizer:                  defaultTokenizer,
		tokenMatcher:               defaultTokenMatcher,
		matchingEvaluator:          defaultMatchingEvaluator,
		localLengthEvaluator:       defaultLocalLengthEvaluator,
		transpositionCostEvaluator: defaultTranspositionCostEvaluator,
		transpositionsEvaluator:    defaultTranspositionsEvaluator,
	}
}

// Defaults

func defaultTokenizer(s string) []string {
	return strings.Split(s, "")
}

func defaultTokenMatcher(t1, t2 string) bool {
	return t1 == t2
}

func defaultMatchingEvaluator(t1, t2 string) float64 {
	return 1.0
}

func defaultLocalLengthEvaluator(i float64) float64 {
	return i
}

func defaultTranspositionCostEvaluator(c1, c2 int) float64 {
	return 1.0
}

func defaultTranspositionsEvaluator(lcss, trans float64) float64 {
	return lcss - trans
}

// Getters & Setters

func (s *Sift4) SetMaxOffset(i int) *Sift4 {
	s.maxOffset = i
	return s
}
func (s *Sift4) GetMaxOffset() int {
	return s.maxOffset
}

func (s *Sift4) SetTokenizer(fn func(string) []string) *Sift4 {
	s.tokenizer = fn
	return s
}
func (s *Sift4) GetTokenizer() func(string) []string {
	return s.tokenizer
}

func (s *Sift4) SetTokenMatcher(fn func(string, string) bool) *Sift4 {
	s.tokenMatcher = fn
	return s
}
func (s *Sift4) GetTokenMatcher() func(string, string) bool {
	return s.tokenMatcher
}

func (s *Sift4) SetMatchingEvaluator(fn func(string, string) float64) *Sift4 {
	s.matchingEvaluator = fn
	return s
}
func (s *Sift4) GetMatchingEvaluator() func(string, string) float64 {
	return s.matchingEvaluator
}

func (s *Sift4) SetLocalLengthEvaluator(fn func(float64) float64) *Sift4 {
	s.localLengthEvaluator = fn
	return s
}
func (s *Sift4) GetLocalLengthEvaluator() func(float64) float64 {
	return s.localLengthEvaluator
}

func (s *Sift4) SetTranspositionCostEvaluator(fn func(int, int) float64) *Sift4 {
	s.transpositionCostEvaluator = fn
	return s
}
func (s *Sift4) GetTranspositionCostEvaluator() func(int, int) float64 {
	return s.transpositionCostEvaluator
}

func (s *Sift4) SetTranspositionsEvaluator(fn func(float64, float64) float64) *Sift4 {
	s.transpositionsEvaluator = fn
	return s
}
func (s *Sift4) GetTranspositionsEvaluator() func(float64, float64) float64 {
	return s.transpositionsEvaluator
}

// Methods

func (s *Sift4) Distance(s1, s2 string) float64 {

	// Tokenize strings
	t1 := s.tokenizer(s1)
	t2 := s.tokenizer(s2)

	// Length of token sets
	l1 := len(t1)
	l2 := len(t2)

	if l1 == 0 {
		return float64(l2)
	}
	if l2 == 0 {
		return float64(l1)
	}

	c1 := 0         // cursor for string 1
	c2 := 0         // cursor for string 2
	lcss := 0.0     // largest common subsequence
	local_cs := 0.0 // local common substring
	trans := 0.0    // number of transpositions

	offset_arr := []offset{}

	for (c1 < l1) && (c2 < l2) {
		if s.tokenMatcher(t1[c1], t2[c2]) {
			is_trans := false

			local_cs += s.matchingEvaluator(t1[c1], t2[c2])

			// check if current pair is a transposition
			i := 0
			for i < len(offset_arr) {
				ofs := offset_arr[i]
				if c1 <= ofs.c1 || c2 <= ofs.c2 {
					is_trans = math.Abs(float64(c2-c1)) >= math.Abs(float64(ofs.c2-ofs.c1))
					if is_trans {
						trans += s.transpositionCostEvaluator(c1, c2)
					} else if !ofs.trans {
						ofs.trans = true
						trans += s.transpositionCostEvaluator(ofs.c1, ofs.c2)
					}
					break
				} else {
					if c1 > ofs.c2 && c2 > ofs.c1 {
						offset_arr = append(offset_arr[:i], offset_arr[i+1:]...)
					} else {
						i++
					}
				}
			}
			offset_arr = append(offset_arr, offset{c1, c2, is_trans})
		} else {
			lcss += s.localLengthEvaluator(local_cs)
			local_cs = 0
			if c1 != c2 {
				c1 = int(math.Min(float64(c1), float64(c2)))
				c2 = c1
			}

			for i := 0; i < s.maxOffset && (c1+i < l1 || c2+i < l2); i++ {
				if c1+i < l1 && s.tokenMatcher(t1[c1+i], t2[c2]) {
					c1 += i - 1
					c2--
					break
				}
				if c2+i < l2 && s.tokenMatcher(t1[c1], t2[c2+i]) {
					c1--
					c2 += i - 1
					break
				}
			}
		}

		c1++
		c2++

		if s.maxDistance > 0 {
			tmp_dist := s.localLengthEvaluator(math.Max(float64(c1), float64(c2))) - s.transpositionsEvaluator(lcss, trans)
			if tmp_dist >= float64(s.maxDistance) {
				return math.Floor(tmp_dist + .5)
			}
		}

		if c1 >= l1 || c2 >= l2 {
			lcss += s.localLengthEvaluator(local_cs)
			local_cs = 0
			c1 = int(math.Min(float64(c1), float64(c2)))
			c2 = c1
		}
	}

	lcss += s.localLengthEvaluator(local_cs)
	return math.Floor(s.localLengthEvaluator(math.Max(float64(l1), float64(l2))) - s.transpositionsEvaluator(lcss, trans) + .5)
}
<commit_msg>sensible maxOffset default<commit_after>package sift4

import (
	"math"
	"strings"
)

// Models

type Sift4 struct {
	maxOffset                  int
	maxDistance                int
	tokenizer                  func(string) []string
	tokenMatcher               func(string, string) bool
	matchingEvaluator          func(string, string) float64
	localLengthEvaluator       func(float64) float64
	transpositionCostEvaluator func(int, int) float64
	transpositionsEvaluator    func(float64, float64) float64
}

type offset struct {
	c1    int
	c2    int
	trans bool
}

// Initialization

func New() *Sift4 {
	return &Sift4{
		maxOffset:                  5,
		maxDistance:                0,
		tokenizer:                  defaultTokenizer,
		tokenMatcher:               defaultTokenMatcher,
		matchingEvaluator:          defaultMatchingEvaluator,
		localLengthEvaluator:       defaultLocalLengthEvaluator,
		transpositionCostEvaluator: defaultTranspositionCostEvaluator,
		transpositionsEvaluator:    defaultTranspositionsEvaluator,
	}
}

// Defaults

func defaultTokenizer(s string) []string {
	return strings.Split(s, "")
}

func defaultTokenMatcher(t1, t2 string) bool {
	return t1 == t2
}

func defaultMatchingEvaluator(t1, t2 string) float64 {
	return 1.0
}

func defaultLocalLengthEvaluator(i float64) float64 {
	return i
}

func defaultTranspositionCostEvaluator(c1, c2 int) float64 {
	return 1.0
}

func defaultTranspositionsEvaluator(lcss, trans float64) float64 {
	return lcss - trans
}

// Getters & Setters

func (s *Sift4) SetMaxOffset(i int) *Sift4 {
	s.maxOffset = i
	return s
}
func (s *Sift4) GetMaxOffset() int {
	return s.maxOffset
}

func (s *Sift4) SetTokenizer(fn func(string) []string) *Sift4 {
	s.tokenizer = fn
	return s
}
func (s *Sift4) GetTokenizer() func(string) []string {
	return s.tokenizer
}

func (s *Sift4) SetTokenMatcher(fn func(string, string) bool) *Sift4 {
	s.tokenMatcher = fn
	return s
}
func (s *Sift4) GetTokenMatcher() func(string, string) bool {
	return s.tokenMatcher
}

func (s *Sift4) SetMatchingEvaluator(fn func(string, string) float64) *Sift4 {
	s.matchingEvaluator = fn
	return s
}
func (s *Sift4) GetMatchingEvaluator() func(string, string) float64 {
	return s.matchingEvaluator
}

func (s *Sift4) SetLocalLengthEvaluator(fn func(float64) float64) *Sift4 {
	s.localLengthEvaluator = fn
	return s
}
func (s *Sift4) GetLocalLengthEvaluator() func(float64) float64 {
	return s.localLengthEvaluator
}

func (s *Sift4) SetTranspositionCostEvaluator(fn func(int, int) float64) *Sift4 {
	s.transpositionCostEvaluator = fn
	return s
}
func (s *Sift4) GetTranspositionCostEvaluator() func(int, int) float64 {
	return s.transpositionCostEvaluator
}

func (s *Sift4) SetTranspositionsEvaluator(fn func(float64, float64) float64) *Sift4 {
	s.transpositionsEvaluator = fn
	return s
}
func (s *Sift4) GetTranspositionsEvaluator() func(float64, float64) float64 {
	return s.transpositionsEvaluator
}

// Methods

func (s *Sift4) Distance(s1, s2 string) float64 {

	// Tokenize strings
	t1 := s.tokenizer(s1)
	t2 := s.tokenizer(s2)

	// Length of token sets
	l1 := len(t1)
	l2 := len(t2)

	if l1 == 0 {
		return float64(l2)
	}
	if l2 == 0 {
		return float64(l1)
	}

	c1 := 0         // cursor for string 1
	c2 := 0         // cursor for string 2
	lcss := 0.0     // largest common subsequence
	local_cs := 0.0 // local common substring
	trans := 0.0    // number of transpositions

	offset_arr := []offset{}

	for (c1 < l1) && (c2 < l2) {
		if s.tokenMatcher(t1[c1], t2[c2]) {
			is_trans := false

			local_cs += s.matchingEvaluator(t1[c1], t2[c2])

			// check if current pair is a transposition
			i := 0
			for i < len(offset_arr) {
				ofs := offset_arr[i]
				if c1 <= ofs.c1 || c2 <= ofs.c2 {
					is_trans = math.Abs(float64(c2-c1)) >= math.Abs(float64(ofs.c2-ofs.c1))
					if is_trans {
						trans += s.transpositionCostEvaluator(c1, c2)
					} else if !ofs.trans {
						ofs.trans = true
						trans += s.transpositionCostEvaluator(ofs.c1, ofs.c2)
					}
					break
				} else {
					if c1 > ofs.c2 && c2 > ofs.c1 {
						offset_arr = append(offset_arr[:i], offset_arr[i+1:]...)
					} else {
						i++
					}
				}
			}
			offset_arr = append(offset_arr, offset{c1, c2, is_trans})
		} else {
			lcss += s.localLengthEvaluator(local_cs)
			local_cs = 0
			if c1 != c2 {
				c1 = int(math.Min(float64(c1), float64(c2)))
				c2 = c1
			}

			for i := 0; i < s.maxOffset && (c1+i < l1 || c2+i < l2); i++ {
				if c1+i < l1 && s.tokenMatcher(t1[c1+i], t2[c2]) {
					c1 += i - 1
					c2--
					break
				}
				if c2+i < l2 && s.tokenMatcher(t1[c1], t2[c2+i]) {
					c1--
					c2 += i - 1
					break
				}
			}
		}

		c1++
		c2++

		if s.maxDistance > 0 {
			tmp_dist := s.localLengthEvaluator(math.Max(float64(c1), float64(c2))) - s.transpositionsEvaluator(lcss, trans)
			if tmp_dist >= float64(s.maxDistance) {
				return math.Floor(tmp_dist + .5)
			}
		}

		if c1 >= l1 || c2 >= l2 {
			lcss += s.localLengthEvaluator(local_cs)
			local_cs = 0
			c1 = int(math.Min(float64(c1), float64(c2)))
			c2 = c1
		}
	}

	lcss += s.localLengthEvaluator(local_cs)
	return math.Floor(s.localLengthEvaluator(math.Max(float64(l1), float64(l2))) - s.transpositionsEvaluator(lcss, trans) + .5)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/csv"
	"fmt"
	"github.com/ActiveState/tail"
	"github.com/prometheus/client_golang/prometheus"
	"log"
	"net/http"
	"strconv"
	"strings"
	// "regexp"
	// "time"
)

var (
	loadEvents = prometheus.NewCounter(prometheus.CounterOpts{
		Name: "datalogger_rows_loaded",
		Help: "the number of rows loaded into the database",
	})
	batteryVoltage = prometheus.NewGauge(prometheus.GaugeOpts{
		Name: "datalogger_main_battery_voltage",
		Help: "The current main battery voltage",
	})
)

func init() {
	prometheus.MustRegister(loadEvents)
	prometheus.MustRegister(batteryVoltage)
}

func loadData(fileName string) {

	t, err := tail.TailFile(fileName, tail.Config{
		Follow: true,
		ReOpen: true,
	})
	if err != nil {
		log.Fatal(err)
	}

	// Read title and variables and units
	skip := 0
	for line := range t.Lines {
		if strings.Contains(line.Text, "TOA5") {
			skip = 4
		}
		if skip != 0 {
			fmt.Println(line.Text)
			skip = skip - 1
			continue
		}
		// Read data
		reader := csv.NewReader(strings.NewReader(line.Text))
		fields, err := reader.Read()
		if err != nil {
			log.Fatal(err)
		}
		voltage, err := strconv.ParseFloat(fields[2], 64)
		if err != nil {
			log.Fatal(err)
		}
		fmt.Println(voltage)
		batteryVoltage.Set(voltage)
	}
}

func main() {

	go loadData("raingauge_Table1.dat")

	http.Handle("/metrics", prometheus.Handler())
	http.ListenAndServe(":9094", nil)

}
<commit_msg>remove printing for now<commit_after>package main

import (
	"encoding/csv"
	"github.com/ActiveState/tail"
	"github.com/prometheus/client_golang/prometheus"
	"log"
	"net/http"
	"strconv"
	"strings"
)

var (
	loadEvents = prometheus.NewCounter(prometheus.CounterOpts{
		Name: "datalogger_rows_loaded",
		Help: "the number of rows loaded into the database",
	})
	batteryVoltage = prometheus.NewGauge(prometheus.GaugeOpts{
		Name: "datalogger_main_battery_voltage",
		Help: "The current main battery voltage",
	})
)

func init() {
	prometheus.MustRegister(loadEvents)
	prometheus.MustRegister(batteryVoltage)
}

func loadData(fileName string) {

	t, err := tail.TailFile(fileName, tail.Config{
		Follow: true,
		ReOpen: true,
	})
	if err != nil {
		log.Fatal(err)
	}

	// Read title and variables and units
	skip := 0
	for line := range t.Lines {
		if strings.Contains(line.Text, "TOA5") {
			skip = 4
		}
		if skip != 0 {
			// decode headers here
			skip = skip - 1
			continue
		}
		// Read data
		reader := csv.NewReader(strings.NewReader(line.Text))
		fields, err := reader.Read()
		if err != nil {
			log.Fatal(err)
		}
		voltage, err := strconv.ParseFloat(fields[2], 64)
		if err != nil {
			log.Fatal(err)
		}
		batteryVoltage.Set(voltage)
	}
}

func main() {

	go loadData("raingauge_Table1.dat")

	http.Handle("/metrics", prometheus.Handler())
	http.ListenAndServe(":9094", nil)

}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"github.com/nats-io/nats"
	"github.com/pivotal-golang/lager"
	"log/syslog"
	"os"
	"os/signal"
	"strings"
	"syscall"
)

var stop chan bool
var logger lager.Logger

func main() {
	logger = lager.NewLogger("nats-to-syslog")

	stop = make(chan bool)
	signals := make(chan os.Signal, 1)
	signal.Notify(signals, syscall.SIGINT)
	signal.Notify(signals, syscall.SIGKILL)
	signal.Notify(signals, syscall.SIGTERM)

	go func() {
		for signal := range signals {
			logger.Info("signal-caught", lager.Data{"signal": signal})
			stop <- true
		}
	}()

	var natsUri = flag.String("nats-uri", "nats://localhost:4222", "The NATS server URI")
	var natsSubject = flag.String("nats-subject", ">", "The NATS subject to subscribe to")
	var syslogEndpoint = flag.String("syslog-endpoint", "localhost:514", "The remote syslog server host:port")
	var debug = flag.Bool("debug", false, "debug logging true/false")
	flag.Parse()

	if *debug {
		logger.RegisterSink(lager.NewWriterSink(os.Stdout, lager.DEBUG))
	} else {
		logger.RegisterSink(lager.NewWriterSink(os.Stdout, lager.INFO))
	}

	syslog, err := syslog.Dial("tcp", *syslogEndpoint, syslog.LOG_INFO, "nats-to-syslog")
	handleError(err, "connecting to syslog")
	logger.Info("connected-to-syslog", lager.Data{"endpoint": syslogEndpoint})
	defer syslog.Close()

	natsClient, err := nats.Connect(*natsUri)
	handleError(err, "connecting to nats")
	logger.Info("connected-to-nats", lager.Data{"uri": natsUri})
	defer natsClient.Close()

	buffer := make(chan *nats.Msg, 1000)

	go func() {
		for message := range buffer {
			logEntry := buildLogEntry(message)
			logger.Debug("message-sent-to-syslog", lager.Data{"message": logEntry})
			err = syslog.Info(logEntry)
			if err != nil {
				logger.Error("logging-to-syslog-failed", err)
				stop <- true
			}
		}
	}()

	natsClient.Subscribe(*natsSubject, func(message *nats.Msg) {
		buffer <- message
	})
	logger.Info("subscribed-to-subject", lager.Data{"subject": *natsSubject})

	<-stop
	logger.Info("bye.")
}

func handleError(err error, context string) {
	if err != nil {
		context = strings.Replace(context, " ", "-", -1)
		errorLogger := logger.Session(context)
		errorLogger.Error("error", err)
		os.Exit(1)
	}
}

func buildLogEntry(message *nats.Msg) string {
	entry := struct {
		Data    string
		Reply   string
		Subject string
	}{
		string(message.Data),
		message.Reply,
		message.Subject,
	}

	data, err := json.Marshal(entry)
	if err != nil {
		logger.Error("unmarshalling-log-failed", err, lager.Data{"data": string(message.Data)})
		return ""
	}

	return string(data)
}
<commit_msg>extract functions<commit_after>package main

import (
	"encoding/json"
	"flag"
	"github.com/nats-io/nats"
	"github.com/pivotal-golang/lager"
	"log/syslog"
	"os"
	"os/signal"
	"strings"
	"syscall"
)

var stop chan bool
var logger lager.Logger

func main() {
	logger = lager.NewLogger("nats-to-syslog")
	stop = make(chan bool)
	buffer := make(chan *nats.Msg, 1000)

	trapSignals()

	var natsUri = flag.String("nats-uri", "nats://localhost:4222", "The NATS server URI")
	var natsSubject = flag.String("nats-subject", ">", "The NATS subject to subscribe to")
	var syslogEndpoint = flag.String("syslog-endpoint", "localhost:514", "The remote syslog server host:port")
	var debug = flag.Bool("debug", false, "debug logging true/false")
	flag.Parse()

	setupLogger(*debug)

	syslog := connectToSyslog(*syslogEndpoint)
	defer syslog.Close()

	natsClient := connectToNATS(*natsUri)
	defer natsClient.Close()

	go func() {
		for message := range buffer {
			sendToSyslog(message, syslog)
		}
	}()

	natsClient.Subscribe(*natsSubject, func(message *nats.Msg) {
		buffer <- message
	})
	logger.Info("subscribed-to-subject", lager.Data{"subject": *natsSubject})

	<-stop
	logger.Info("bye.")
}

func handleError(err error, context string) {
	if err != nil {
		context = strings.Replace(context, " ", "-", -1)
		errorLogger := logger.Session(context)
		errorLogger.Error("error", err)
		os.Exit(1)
	}
}

func buildLogEntry(message *nats.Msg) string {
	entry := struct {
		Data    string
		Reply   string
		Subject string
	}{
		string(message.Data),
		message.Reply,
		message.Subject,
	}

	data, err := json.Marshal(entry)
	if err != nil {
		logger.Error("unmarshalling-log-failed", err, lager.Data{"data": string(message.Data)})
		return ""
	}

	return string(data)
}

func connectToSyslog(endpoint string) *syslog.Writer {
	syslog, err := syslog.Dial("tcp", endpoint, syslog.LOG_INFO, "nats-to-syslog")
	handleError(err, "connecting to syslog")
	logger.Info("connected-to-syslog", lager.Data{"endpoint": endpoint})
	return syslog
}

func connectToNATS(natsUri string) *nats.Conn {
	natsClient, err := nats.Connect(natsUri)
	handleError(err, "connecting to nats")
	logger.Info("connected-to-nats", lager.Data{"uri": natsUri})
	return natsClient
}

func sendToSyslog(message *nats.Msg, syslog *syslog.Writer) {
	logEntry := buildLogEntry(message)
	logger.Debug("message-sent-to-syslog", lager.Data{"message": logEntry})
	err := syslog.Info(logEntry)
	if err != nil {
		logger.Error("logging-to-syslog-failed", err)
		stop <- true
	}
}

func setupLogger(debug bool) {
	if debug {
		logger.RegisterSink(lager.NewWriterSink(os.Stdout, lager.DEBUG))
	} else {
		logger.RegisterSink(lager.NewWriterSink(os.Stdout, lager.INFO))
	}
}

func trapSignals() {
	signals := make(chan os.Signal, 1)
	signal.Notify(signals, syscall.SIGINT)
	signal.Notify(signals, syscall.SIGKILL)
	signal.Notify(signals, syscall.SIGTERM)

	go func() {
		for signal := range signals {
			logger.Info("signal-caught", lager.Data{"signal": signal})
			stop <- true
		}
	}()
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/hex"
	"encoding/json"
	"expvar"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"log/syslog"
	"net/http"
	_ "net/http/pprof"
	"net/url"
	"os"
	"runtime"
	"sync"

	pickle "github.com/kisielk/og-rek"
)

var Debug int

var Config = struct {
	Backends []string
	MaxProcs int
	Port     int

	mu          sync.RWMutex
	metricPaths map[string][]string
}{
	MaxProcs:    1,
	Port:        8080,
	metricPaths: make(map[string][]string),
}

var Metrics = struct {
	Requests *expvar.Int
	Errors   *expvar.Int
}{
	Requests: expvar.NewInt("requests"),
	Errors:   expvar.NewInt("errors"),
}

var logger multilog

type serverResponse struct {
	server   string
	response []byte
}

func multiGet(servers []string, uri string) []serverResponse {

	if Debug > 0 {
		logger.Logln("querying servers=", servers, "uri=", uri)
	}

	ch := make(chan serverResponse)

	for _, server := range servers {
		go func(server string, ch chan<- serverResponse) {

			u, err := url.Parse(server + uri)
			if err != nil {
				logger.Logln("error parsing uri: ", server+uri, ":", err)
				ch <- serverResponse{server, nil}
				return
			}
			req := http.Request{
				URL:    u,
				Header: make(http.Header),
			}

			resp, err := http.DefaultClient.Do(&req)
			if err != nil {
				logger.Logln("error querying ", server, "/", uri, ":", err)
				ch <- serverResponse{server, nil}
				return
			}
			defer resp.Body.Close()

			if resp.StatusCode != 200 {
				logger.Logln("bad response code ", server, "/", uri, ":", resp.StatusCode)
				ch <- serverResponse{server, nil}
				return
			}

			body, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				logger.Logln("error reading body: ", server, "/", uri, ":", err)
				ch <- serverResponse{server, nil}
				return
			}

			ch <- serverResponse{server, body}
		}(server, ch)
	}

	var response []serverResponse

	for i := 0; i < len(servers); i++ {
		r := <-ch
		if r.response != nil {
			response = append(response, r)
		}
	}

	return response
}

func findHandler(w http.ResponseWriter, req *http.Request) {

	if Debug > 0 {
		logger.Logln("request: ", req.URL.RequestURI())
	}

	Metrics.Requests.Add(1)

	responses := multiGet(Config.Backends, req.URL.RequestURI())

	if responses == nil || len(responses) == 0 {
		logger.Logln("error querying backends for: ", req.URL.RequestURI())
		http.Error(w, "error querying backends", http.StatusInternalServerError)
		return
	}

	// metric -> [server1, ... ]
	paths := make(map[string][]string)

	var metrics []map[interface{}]interface{}
	for _, r := range responses {
		d := pickle.NewDecoder(bytes.NewReader(r.response))
		metric, err := d.Decode()
		if err != nil {
			logger.Logf("error decoding response from server:%s: req:%s: err=%s", r.server, req.URL.RequestURI(), err)
			if Debug > 1 {
				logger.Logln("\n" + hex.Dump(r.response))
			}
			Metrics.Errors.Add(1)
			continue
		}

		marray, ok := metric.([]interface{})
		if !ok {
			logger.Logf("bad type for metric:%t from server:%s: req:%s", metric, r.server, req.URL.RequestURI())
			http.Error(w, fmt.Sprintf("bad type for metric: %t", metric), http.StatusInternalServerError)
			Metrics.Errors.Add(1)
			return
		}

		for i, m := range marray {
			mm, ok := m.(map[interface{}]interface{})
			if !ok {
				logger.Logf("bad type for metric[%d]:%t from server:%s: req:%s", i, m, r.server, req.URL.RequestURI())
				http.Error(w, fmt.Sprintf("bad type for metric[%d]:%t", i, m), http.StatusInternalServerError)
				Metrics.Errors.Add(1)
				return
			}
			name, ok := mm["metric_path"].(string)
			if !ok {
				logger.Logf("bad type for metric_path:%t from server:%s: req:%s", mm["metric_path"], r.server, req.URL.RequestURI())
				http.Error(w, fmt.Sprintf("bad type for metric_path: %t", mm["metric_path"]), http.StatusInternalServerError)
				Metrics.Errors.Add(1)
				return
			}
			p, ok := paths[name]
			if !ok {
				// we haven't seen this name yet
				// add the metric to the list of metrics to return
				metrics = append(metrics, mm)
			}
			// add the server to the list of servers that know about this metric
			p = append(p, r.server)
			paths[name] = p
		}
	}

	// update our cache of which servers have which metrics
	Config.mu.Lock()
	for k, v := range paths {
		Config.metricPaths[k] = v
	}
	Config.mu.Unlock()

	w.Header().Set("Content-Type", "application/pickle")

	pEnc := pickle.NewEncoder(w)
	pEnc.Encode(metrics)
}

func renderHandler(w http.ResponseWriter, req *http.Request) {

	if Debug > 0 {
		logger.Logln("request: ", req.URL.RequestURI())
	}

	Metrics.Requests.Add(1)

	req.ParseForm()
	target := req.FormValue("target")

	if target == "" {
		http.Error(w, "empty target", http.StatusBadRequest)
		return
	}

	var serverList []string
	var ok bool

	Config.mu.RLock()
	// lookup the server list for this metric, or use all the servers if it's unknown
	if serverList, ok = Config.metricPaths[target]; !ok || serverList == nil || len(serverList) == 0 {
		serverList = Config.Backends
	}
	Config.mu.RUnlock()

	responses := multiGet(serverList, req.URL.RequestURI())

	if responses == nil || len(responses) == 0 {
		logger.Logln("error querying backends for: ", req.URL.RequestURI())
		http.Error(w, "error querying backends", http.StatusInternalServerError)
		Metrics.Errors.Add(1)
		return
	}

	// nothing to merge
	if len(responses) == 1 {
		w.Header().Set("Content-Type", "application/pickle")
		w.Write(responses[0].response)
	}

	// decode everything
	var decoded [][]interface{}
	for _, r := range responses {
		d := pickle.NewDecoder(bytes.NewReader(r.response))
		metric, err := d.Decode()
		if err != nil {
			logger.Logf("error decoding response from server:%s: req:%s: err=%s", r.server, req.URL.RequestURI(), err)
			if Debug > 1 {
				logger.Logln("\n" + hex.Dump(r.response))
			}
			Metrics.Errors.Add(1)
			continue
		}

		marray, ok := metric.([]interface{})
		if !ok {
			err := fmt.Sprintf("bad type for metric:%d from server:%s req:%s", metric, r.server, req.URL.RequestURI())
			logger.Logln(err)
			http.Error(w, err, http.StatusInternalServerError)
			Metrics.Errors.Add(1)
			return
		}
		if len(marray) == 0 {
			continue
		}
		decoded = append(decoded, marray)
	}

	if Debug > 2 {
		logger.Logf("request: %s: %v", req.URL.RequestURI(), decoded)
	}

	if len(decoded) == 0 {
		logger.Logf("no decoded responses to merge for req:%s", req.URL.RequestURI())
		w.Header().Set("Content-Type", "application/pickle")
		w.Write(responses[0].response)
		return
	}

	if len(decoded) == 1 {
		logger.Logf("only one decoded responses to merge for req:%s", req.URL.RequestURI())
		w.Header().Set("Content-Type", "application/pickle")
		// send back whatever data we have
		e := pickle.NewEncoder(w)
		e.Encode(decoded[0])
		return
	}

	if len(decoded[0]) != 1 {
		err := fmt.Sprintf("bad length for decoded[]:%d from req:%s", len(decoded[0]), req.URL.RequestURI())
		logger.Logln(err)
		http.Error(w, err, http.StatusInternalServerError)
		Metrics.Errors.Add(1)
		return
	}

	base, ok := decoded[0][0].(map[interface{}]interface{})
	if !ok {
		err := fmt.Sprintf("bad type for decoded:%t from req:%s", decoded[0][0], req.URL.RequestURI())
		logger.Logln(err)
		http.Error(w, err, http.StatusInternalServerError)
		Metrics.Errors.Add(1)
		return
	}

	values, ok := base["values"].([]interface{})
	if !ok {
		err := fmt.Sprintf("bad type for values:%t from req:%s", base["values"], req.URL.RequestURI())
		logger.Logln(err)
		http.Error(w, err, http.StatusInternalServerError)
		Metrics.Errors.Add(1)
		return
	}

fixValues:
	for i := 0; i < len(values); i++ {
		if _, ok := values[i].(pickle.None); ok {
			// find one in the other values arrays
			for other := 1; other < len(decoded); other++ {
				m, ok := decoded[other][0].(map[interface{}]interface{})
				if !ok {
					logger.Logln(fmt.Sprintf("bad type for decoded[%d][0]: %t", other, decoded[other][0]))
					Metrics.Errors.Add(1)
					break fixValues
				}

				ovalues, ok := m["values"].([]interface{})
				if !ok {
					logger.Logf("bad type for ovalues:%t from req:%s (skipping)", m["values"], req.URL.RequestURI())
					Metrics.Errors.Add(1)
					break fixValues
				}

				if len(ovalues) != len(values) {
					logger.Logf("request: %s: unable to merge ovalues: len(values)=%d but len(ovalues)=%d", req.URL.RequestURI(), len(values), len(ovalues))
					Metrics.Errors.Add(1)
					break fixValues
				}

				if _, ok := ovalues[i].(pickle.None); !ok {
					values[i] = ovalues[i]
					break
				}
			}
		}
	}

	// the first response is where we've been filling in our data, so we're ok just to serialize it as our response
	w.Header().Set("Content-Type", "application/pickle")
	e := pickle.NewEncoder(w)
	e.Encode(decoded[0])
}

func stripCommentHeader(cfg []byte) []byte {

	// strip out the comment header block that begins with '#' characters
	// as soon as we see a line that starts with something _other_ than '#', we're done

	idx := 0
	for cfg[0] == '#' {
		idx = bytes.Index(cfg, []byte("\n"))
		if idx == -1 || idx+1 == len(cfg) {
			return nil
		}
		cfg = cfg[idx+1:]
	}

	return cfg
}

func main() {

	configFile := flag.String("c", "", "config file (json)")
	port := flag.Int("p", 0, "port to listen on")
	maxprocs := flag.Int("maxprocs", 0, "GOMAXPROCS")
	flag.IntVar(&Debug, "d", 0, "enable debug logging")
	logStdout := flag.Bool("stdout", false, "write logging output also to stdout (default: only syslog)")

	flag.Parse()

	if *configFile == "" {
		log.Fatal("missing config file")
	}

	cfgjs, err := ioutil.ReadFile(*configFile)
	if err != nil {
		log.Fatal("unable to load config file:", err)
	}

	cfgjs = stripCommentHeader(cfgjs)

	if cfgjs == nil {
		log.Fatal("error removing header comment from ", *configFile)
	}

	err = json.Unmarshal(cfgjs, &Config)
	if err != nil {
		log.Fatal("error parsing config file: ", err)
	}

	if len(Config.Backends) == 0 {
		log.Fatal("no Backends loaded -- exiting")
	}

	// command line overrides config file

	if *port != 0 {
		Config.Port = *port
	}

	if *maxprocs != 0 {
		Config.MaxProcs = *maxprocs
	}

	// set up our logging
	slog, err := syslog.New(syslog.LOG_DAEMON, "carbonzipper")
	if err != nil {
		log.Fatal("can't obtain a syslog connection", err)
	}
	logger = append(logger, &sysLogger{w: slog})

	if *logStdout {
		logger = append(logger, &stdoutLogger{log.New(os.Stdout, "", log.LstdFlags)})
	}

	logger.Logln("setting GOMAXPROCS=", Config.MaxProcs)
	runtime.GOMAXPROCS(Config.MaxProcs)

	http.HandleFunc("/metrics/find/", findHandler)
	http.HandleFunc("/render/", renderHandler)

	portStr := fmt.Sprintf(":%d", Config.Port)
	logger.Logln("listening on", portStr)
	log.Fatal(http.ListenAndServe(portStr, nil))
}

// trivial logging classes

type Logger interface {
	Log(string)
}

type stdoutLogger struct{ logger *log.Logger }

func (l *stdoutLogger) Log(s string) { l.logger.Print(s) }

type sysLogger struct{ w *syslog.Writer }

func (l *sysLogger) Log(s string) { l.w.Info(s) }

type multilog []Logger

func (ml multilog) Logln(a ...interface{}) {
	s := fmt.Sprintln(a...)
	for _, l := range ml {
		l.Log(s)
	}
}

func (ml multilog) Logf(format string, a ...interface{}) {
	s := fmt.Sprintf(format, a...)
	for _, l := range ml {
		l.Log(s)
	}
}
<commit_msg>publish numbers on in-flight http requests<commit_after>package main

import (
	"bytes"
	"encoding/hex"
	"encoding/json"
	"expvar"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"log/syslog"
	"net/http"
	_ "net/http/pprof"
	"net/url"
	"os"
	"runtime"
	"sync"
	"time"

	pickle "github.com/kisielk/og-rek"
)

var Debug int

var Config = struct {
	Backends []string
	MaxProcs int
	Port     int

	mu          sync.RWMutex
	metricPaths map[string][]string
}{
	MaxProcs:    1,
	Port:        8080,
	metricPaths: make(map[string][]string),
}

var Metrics = struct {
	Requests *expvar.Int
	Errors   *expvar.Int
}{
	Requests: expvar.NewInt("requests"),
	Errors:   expvar.NewInt("errors"),
}

var logger multilog

type serverResponse struct {
	server   string
	response []byte
}

func multiGet(servers []string, uri string) []serverResponse {

	if Debug > 0 {
		logger.Logln("querying servers=", servers, "uri=", uri)
	}

	ch := make(chan serverResponse)

	for _, server := range servers {
		go func(server string, ch chan<- serverResponse) {

			u, err := url.Parse(server + uri)
			if err != nil {
				logger.Logln("error parsing uri: ", server+uri, ":", err)
				ch <- serverResponse{server, nil}
				return
			}
			req := http.Request{
				URL:    u,
				Header: make(http.Header),
			}

			resp, err := http.DefaultClient.Do(&req)
			if err != nil {
				logger.Logln("error querying ", server, "/", uri, ":", err)
				ch <- serverResponse{server, nil}
				return
			}
			defer resp.Body.Close()

			if resp.StatusCode != 200 {
				logger.Logln("bad response code ", server, "/", uri, ":", resp.StatusCode)
				ch <- serverResponse{server, nil}
				return
			}

			body, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				logger.Logln("error reading body: ", server, "/", uri, ":", err)
				ch <- serverResponse{server, nil}
				return
			}

			ch <- serverResponse{server, body}
		}(server, ch)
	}

	var response []serverResponse

	for i := 0; i < len(servers); i++ {
		r := <-ch
		if r.response != nil {
			response = append(response, r)
		}
	}

	return response
}

func findHandler(w http.ResponseWriter, req *http.Request) {

	if Debug > 0 {
		logger.Logln("request: ", req.URL.RequestURI())
	}

	Metrics.Requests.Add(1)

	responses := multiGet(Config.Backends, req.URL.RequestURI())

	if responses == nil || len(responses) == 0 {
		logger.Logln("error querying backends for: ", req.URL.RequestURI())
		http.Error(w, "error querying backends", http.StatusInternalServerError)
		return
	}

	// metric -> [server1, ... ]
	paths := make(map[string][]string)

	var metrics []map[interface{}]interface{}
	for _, r := range responses {
		d := pickle.NewDecoder(bytes.NewReader(r.response))
		metric, err := d.Decode()
		if err != nil {
			logger.Logf("error decoding response from server:%s: req:%s: err=%s", r.server, req.URL.RequestURI(), err)
			if Debug > 1 {
				logger.Logln("\n" + hex.Dump(r.response))
			}
			Metrics.Errors.Add(1)
			continue
		}

		marray, ok := metric.([]interface{})
		if !ok {
			logger.Logf("bad type for metric:%t from server:%s: req:%s", metric, r.server, req.URL.RequestURI())
			http.Error(w, fmt.Sprintf("bad type for metric: %t", metric), http.StatusInternalServerError)
			Metrics.Errors.Add(1)
			return
		}

		for i, m := range marray {
			mm, ok := m.(map[interface{}]interface{})
			if !ok {
				logger.Logf("bad type for metric[%d]:%t from server:%s: req:%s", i, m, r.server, req.URL.RequestURI())
				http.Error(w, fmt.Sprintf("bad type for metric[%d]:%t", i, m), http.StatusInternalServerError)
				Metrics.Errors.Add(1)
				return
			}
			name, ok := mm["metric_path"].(string)
			if !ok {
				logger.Logf("bad type for metric_path:%t from server:%s: req:%s", mm["metric_path"], r.server, req.URL.RequestURI())
				http.Error(w, fmt.Sprintf("bad type for metric_path: %t", mm["metric_path"]), http.StatusInternalServerError)
				Metrics.Errors.Add(1)
				return
			}
			p, ok := paths[name]
			if !ok {
				// we haven't seen this name yet
				// add the metric to the list of metrics to return
				metrics = append(metrics, mm)
			}
			// add the server to the list of servers that know about this metric
			p = append(p, r.server)
			paths[name] = p
		}
	}

	// update our cache of which servers have which metrics
	Config.mu.Lock()
	for k, v := range paths {
		Config.metricPaths[k] = v
	}
	Config.mu.Unlock()

	w.Header().Set("Content-Type", "application/pickle")

	pEnc := pickle.NewEncoder(w)
	pEnc.Encode(metrics)
}

func renderHandler(w http.ResponseWriter, req *http.Request) {

	if Debug > 0 {
		logger.Logln("request: ", req.URL.RequestURI())
	}

	Metrics.Requests.Add(1)

	req.ParseForm()
	target := req.FormValue("target")

	if target == "" {
		http.Error(w, "empty target", http.StatusBadRequest)
		return
	}

	var serverList []string
	var ok bool

	Config.mu.RLock()
	// lookup the server list for this metric, or use all the servers if it's unknown
	if serverList, ok = Config.metricPaths[target]; !ok || serverList == nil || len(serverList) == 0 {
		serverList = Config.Backends
	}
	Config.mu.RUnlock()

	responses := multiGet(serverList, req.URL.RequestURI())

	if responses == nil || len(responses) == 0 {
		logger.Logln("error querying backends for: ", req.URL.RequestURI())
		http.Error(w, "error querying backends", http.StatusInternalServerError)
		Metrics.Errors.Add(1)
		return
	}

	// nothing to merge
	if len(responses) == 1 {
		w.Header().Set("Content-Type", "application/pickle")
		w.Write(responses[0].response)
	}

	// decode everything
	var decoded [][]interface{}
	for _, r := range responses {
		d := pickle.NewDecoder(bytes.NewReader(r.response))
		metric, err := d.Decode()
		if err != nil {
			logger.Logf("error decoding response from server:%s: req:%s: err=%s", r.server, req.URL.RequestURI(), err)
			if Debug > 1 {
				logger.Logln("\n" + hex.Dump(r.response))
			}
			Metrics.Errors.Add(1)
			continue
		}

		marray, ok := metric.([]interface{})
		if !ok {
			err := fmt.Sprintf("bad type for metric:%d from server:%s req:%s", metric, r.server, req.URL.RequestURI())
			logger.Logln(err)
			http.Error(w, err, http.StatusInternalServerError)
			Metrics.Errors.Add(1)
			return
		}
		if len(marray) == 0 {
			continue
		}
		decoded = append(decoded, marray)
	}

	if Debug > 2 {
		logger.Logf("request: %s: %v", req.URL.RequestURI(), decoded)
	}

	if len(decoded) == 0 {
		logger.Logf("no decoded responses to merge for req:%s", req.URL.RequestURI())
		w.Header().Set("Content-Type", "application/pickle")
		w.Write(responses[0].response)
		return
	}

	if len(decoded) == 1 {
		logger.Logf("only one decoded responses to merge for req:%s", req.URL.RequestURI())
		w.Header().Set("Content-Type", "application/pickle")
		// send back whatever data we have
		e := pickle.NewEncoder(w)
		e.Encode(decoded[0])
		return
	}

	if len(decoded[0]) != 1 {
		err := fmt.Sprintf("bad length for decoded[]:%d from req:%s", len(decoded[0]), req.URL.RequestURI())
		logger.Logln(err)
		http.Error(w, err, http.StatusInternalServerError)
		Metrics.Errors.Add(1)
		return
	}

	base, ok := decoded[0][0].(map[interface{}]interface{})
	if !ok {
		err := fmt.Sprintf("bad type for decoded:%t from req:%s", decoded[0][0], req.URL.RequestURI())
		logger.Logln(err)
		http.Error(w, err, http.StatusInternalServerError)
		Metrics.Errors.Add(1)
		return
	}

	values, ok := base["values"].([]interface{})
	if !ok {
		err := fmt.Sprintf("bad type for values:%t from req:%s", base["values"], req.URL.RequestURI())
		logger.Logln(err)
		http.Error(w, err, http.StatusInternalServerError)
		Metrics.Errors.Add(1)
		return
	}

fixValues:
	for i := 0; i < len(values); i++ {
		if _, ok := values[i].(pickle.None); ok {
			// find one in the other values arrays
			for other := 1; other < len(decoded); other++ {
				m, ok := decoded[other][0].(map[interface{}]interface{})
				if !ok {
					logger.Logln(fmt.Sprintf("bad type for decoded[%d][0]: %t", other, decoded[other][0]))
					Metrics.Errors.Add(1)
					break fixValues
				}

				ovalues, ok := m["values"].([]interface{})
				if !ok {
					logger.Logf("bad type for ovalues:%t from req:%s (skipping)", m["values"], req.URL.RequestURI())
					Metrics.Errors.Add(1)
					break fixValues
				}

				if len(ovalues) != len(values) {
					logger.Logf("request: %s: unable to merge ovalues: len(values)=%d but len(ovalues)=%d", req.URL.RequestURI(), len(values), len(ovalues))
					Metrics.Errors.Add(1)
					break fixValues
				}

				if _, ok := ovalues[i].(pickle.None); !ok {
					values[i] = ovalues[i]
					break
				}
			}
		}
	}

	// the first response is where we've been filling in our data, so we're ok just to serialize it as our response
	w.Header().Set("Content-Type", "application/pickle")
	e := pickle.NewEncoder(w)
	e.Encode(decoded[0])
}

func stripCommentHeader(cfg []byte) []byte {

	// strip out the comment header block that begins with '#' characters
	// as soon as we see a line that starts with something _other_ than '#', we're done

	idx := 0
	for cfg[0] == '#' {
		idx = bytes.Index(cfg, []byte("\n"))
		if idx == -1 || idx+1 == len(cfg) {
			return nil
		}
		cfg = cfg[idx+1:]
	}

	return cfg
}

func main() {

	configFile := flag.String("c", "", "config file (json)")
	port := flag.Int("p", 0, "port to listen on")
	maxprocs := flag.Int("maxprocs", 0, "GOMAXPROCS")
	flag.IntVar(&Debug, "d", 0, "enable debug logging")
	logStdout := flag.Bool("stdout", false, "write logging output also to stdout (default: only syslog)")

	flag.Parse()

	if *configFile == "" {
		log.Fatal("missing config file")
	}

	cfgjs, err := ioutil.ReadFile(*configFile)
	if err != nil {
		log.Fatal("unable to load config file:", err)
	}

	cfgjs = stripCommentHeader(cfgjs)

	if cfgjs == nil {
		log.Fatal("error removing header comment from ", *configFile)
	}

	err = json.Unmarshal(cfgjs, &Config)
	if err != nil {
		log.Fatal("error parsing config file: ", err)
	}

	if len(Config.Backends) == 0 {
		log.Fatal("no Backends loaded -- exiting")
	}

	// command line overrides config file

	if *port != 0 {
		Config.Port = *port
	}

	if *maxprocs != 0 {
		Config.MaxProcs = *maxprocs
	}

	// set up our logging
	slog, err := syslog.New(syslog.LOG_DAEMON, "carbonzipper")
	if err != nil {
		log.Fatal("can't obtain a syslog connection", err)
	}
	logger = append(logger, &sysLogger{w: slog})

	if *logStdout {
		logger = append(logger, &stdoutLogger{log.New(os.Stdout, "", log.LstdFlags)})
	}

	logger.Logln("setting GOMAXPROCS=", Config.MaxProcs)
	runtime.GOMAXPROCS(Config.MaxProcs)

	expvar.Publish("httptrack", expvar.Func(trackedConnections))

	http.HandleFunc("/metrics/find/", trackConnections(findHandler))
	http.HandleFunc("/render/", trackConnections(renderHandler))

	portStr := fmt.Sprintf(":%d", Config.Port)
	logger.Logln("listening on", portStr)
	log.Fatal(http.ListenAndServe(portStr, nil))
}

func trackedConnections() interface{} {

	connectionsLock.Lock()
	defer connectionsLock.Unlock()

	m := make(map[string][]string)

	for k, v := range connections {
		u := k.URL.String()
		s := m[u]
		s = append(s, time.Since(v).String())
		m[u] = s
	}

	return m
}

var connections = make(map[*http.Request]time.Time)
var connectionsLock sync.Mutex

func trackConnections(fn http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, req *http.Request) {
		connectionsLock.Lock()
		connections[req] = time.Now()
		connectionsLock.Unlock()

		fn(w, req)

		connectionsLock.Lock()
		delete(connections, req)
		connectionsLock.Unlock()
	}
}

// trivial logging classes

type Logger interface {
	Log(string)
}

type stdoutLogger struct{ logger *log.Logger }

func (l *stdoutLogger) Log(s string) { l.logger.Print(s) }

type sysLogger struct{ w *syslog.Writer }

func (l *sysLogger) Log(s string) { l.w.Info(s) }

type multilog []Logger

func (ml multilog) Logln(a ...interface{}) {
	s := fmt.Sprintln(a...)
	for _, l := range ml {
		l.Log(s)
	}
}

func (ml multilog) Logf(format string, a ...interface{}) {
	s := fmt.Sprintf(format, a...)
	for _, l := range ml {
		l.Log(s)
	}
}
<|endoftext|>