<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
)

type BasicAuthItem struct {
	UserName string `json:"username"`
	Password string `json:"password"`
	Url      string `json:"url"`
}

type DnspodItem struct {
	UserName  string `json:"username"`
	Password  string `json:"password"`
	Domain    string `json:"domain"`
	SubDomain string `json:"sub_domain"`
}

type Setting struct {
	BasicAuthItems []BasicAuthItem `json:"basic"`
	DnspodItems    []DnspodItem    `json:"dnspod"`
}

var (
	currentExternalIP string
)

func getCurrentExternalIP() string {
	return ""
}

func basicAuthorizeHttpRequest(user string, password string, requestUrl string) {
	client := &http.Client{}
	req, err := http.NewRequest("GET", requestUrl, nil)
	req.SetBasicAuth(user, password)
	resp, err := client.Do(req)
	if err != nil {
		fmt.Errorf("request %s failed", requestUrl)
		return
	}

	_, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		fmt.Errorf("reading response failed")
		return
	}
}

func dnspodRequest(user string, password string, domain string, sub_domain string) {
	// get domainn id first
	// if the sub domain doesn't exist, add one
	// otherwise just update it
}

func main() {
	fmt.Println("Dynamic DNS client")

	appConf, err := os.Open("app.conf")
	if err != nil {
		fmt.Println("opening app.conf failed:", err)
		return
	}

	defer func() {
		appConf.Close()
	}()

	b, err := ioutil.ReadAll(appConf)
	if err != nil {
		fmt.Println("reading app.conf failed:", err)
		return
	}
	setting := &Setting{}
	err = json.Unmarshal(b, &setting)
	if err != nil {
		fmt.Println("unmarshalling app.conf failed:", err)
		return
	}

	currentExternalIP = getCurrentExternalIP()
	for _, v := range setting.BasicAuthItems {
		basicAuthorizeHttpRequest(v.UserName, v.Password, v.Url)
	}
}
<commit_msg>(*)finished update dnspod record<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"time"
)

type BasicAuthItem struct {
	UserName string `json:"username"`
	Password string `json:"password"`
	Url      string `json:"url"`
}

type DnspodItem struct {
	UserName  string `json:"username"`
	Password  string `json:"password"`
	Domain    string `json:"domain"`
	SubDomain string `json:"sub_domain"`
}

type Setting struct {
	BasicAuthItems []BasicAuthItem `json:"basic"`
	DnspodItems    []DnspodItem    `json:"dnspod"`
}

type DomainItem struct {
	Id   int    `json:"id"`
	Name string `json:"name"`
}

type DomainList struct {
	Domains []DomainItem `json:"domains"`
}

type RecordItem struct {
	Id   string `json:"id"`
	Name string `json:"name"`
}

type RecordList struct {
	Records []RecordItem `json:"records"`
}

var (
	currentExternalIP string
	lastExternalIP    string
	domainList        = &DomainList{}
)

func getCurrentExternalIP() string {
	return ""
}

func basicAuthorizeHttpRequest(user string, password string, requestUrl string) {
	client := &http.Client{}
	req, err := http.NewRequest("GET", requestUrl, nil)
	req.SetBasicAuth(user, password)
	resp, err := client.Do(req)
	if err != nil {
		fmt.Printf("request %s failed", requestUrl)
		return
	}

	_, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		fmt.Printf("reading response failed")
		return
	}
}

func dnspodRequest(user string, password string, domain string, sub_domain string) {
	needDomainList := false
	if len(domainList.Domains) == 0 {
		needDomainList = true
	}
	var domainId int = 0
	if needDomainList == false {
		needDomainList = true
		for _, v := range domainList.Domains {
			if v.Name == domain {
				needDomainList = false
				domainId = v.Id
				break
			}
		}
	}

	client := &http.Client{}
	if needDomainList {
		// get domainn id first
		domainListUrl := "https://dnsapi.cn/Domain.List"
		resp, err := client.PostForm(domainListUrl, url.Values{
			"login_email":    {user},
			"login_password": {password},
			"format":         {"json"},
		})
		if err != nil {
			fmt.Printf("request domain list failed")
			return
		}

		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			fmt.Printf("reading domain list failed")
			return
		}

		if err = json.Unmarshal(body, &domainList); err != nil {
			fmt.Printf("unmarshalling domain list %s failed", string(body))
			return
		}
	}
	foundDomain := false
	for _, v := range domainList.Domains {
		if v.Name == domain {
			foundDomain = true
			domainId = v.Id
			break
		}
	}

	if foundDomain == false {
		fmt.Printf("domain %s doesn't exists", domain)
		return
	}

	// check record list
	recordListUrl := "https://dnsapi.cn/Record.List"
	resp, err := client.PostForm(recordListUrl, url.Values{
		"login_email":    {user},
		"login_password": {password},
		"format":         {"json"},
		"domain_id":      {strconv.Itoa(domainId)},
	})
	if err != nil {
		fmt.Printf("request record list failed")
		return
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		fmt.Printf("reading record list failed")
		return
	}

	recordList := new(RecordList)
	if err = json.Unmarshal(body, recordList); err != nil {
		fmt.Printf("unmarshalling record list %s failed", string(body))
		fmt.Println(err)
		return
	}
	foundRecord := false
	var recordId string
	for _, v := range recordList.Records {
		if v.Name == sub_domain {
			foundRecord = true
			recordId = v.Id
			break
		}
	}
	fmt.Println("4")
	if foundRecord == false {
		// if the sub domain doesn't exist, add one
		addRecordUrl := "https://dnsapi.cn/Record.Create"
		resp, err := client.PostForm(addRecordUrl, url.Values{
			"login_email":    {user},
			"login_password": {password},
			"format":         {"json"},
			"domain_id":      {strconv.Itoa(domainId)},
			"sub_domain":     {sub_domain},
			"record_type":    {"A"},
			"record_line":    {"默认"},
			"value":          {currentExternalIP},
		})
		if err != nil {
			fmt.Printf("request record insert failed")
			return
		}

		if _, err = ioutil.ReadAll(resp.Body); err != nil {
			fmt.Printf("reading record insert response failed")
			return
		}

		fmt.Printf("A record inserted: %s.%s => %s", sub_domain, domain, getCurrentExternalIP())
	} else {
		// otherwise just update it
		modifyRecordUrl := "https://dnsapi.cn/Record.Modify"
		resp, err := client.PostForm(modifyRecordUrl, url.Values{
			"login_email":    {user},
			"login_password": {password},
			"format":         {"json"},
			"record_id":      {recordId},
			"domain_id":      {strconv.Itoa(domainId)},
			"sub_domain":     {sub_domain},
			"record_type":    {"A"},
			"record_line":    {"默认"},
			"value":          {currentExternalIP},
		})
		if err != nil {
			fmt.Printf("request record modify failed")
			return
		}

		if _, err = ioutil.ReadAll(resp.Body); err != nil {
			fmt.Printf("reading record modify response failed")
			return
		}
		fmt.Printf("A record updated: %s.%s => %s", sub_domain, domain, getCurrentExternalIP())
	}
}

func main() {
	fmt.Println("Dynamic DNS client")

	appConf, err := os.Open("app.conf")
	if err != nil {
		fmt.Println("opening app.conf failed:", err)
		return
	}

	defer func() {
		appConf.Close()
	}()

	b, err := ioutil.ReadAll(appConf)
	if err != nil {
		fmt.Println("reading app.conf failed:", err)
		return
	}
	setting := &Setting{}
	err = json.Unmarshal(b, &setting)
	if err != nil {
		fmt.Println("unmarshalling app.conf failed:", err)
		return
	}

	timer := time.NewTicker(time.Duration(1) * time.Minute) // every 1 minute
	for {
		select {
		case <-timer.C:
			currentExternalIP = getCurrentExternalIP()
			if lastExternalIP != currentExternalIP {
				for _, v := range setting.BasicAuthItems {
					basicAuthorizeHttpRequest(v.UserName, v.Password, v.Url)
				}

				for _, v := range setting.DnspodItems {
					dnspodRequest(v.UserName, v.Password, v.Domain, v.SubDomain)
				}
				lastExternalIP = currentExternalIP
			}
		}
	}
	timer.Stop()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"html/template"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"strings"

	"github.com/bjwbell/renfish/auth"
	"github.com/bjwbell/renfish/conf"
	"github.com/bjwbell/renfish/db"
	"github.com/bjwbell/renfish/submit"
	"github.com/bjwbell/renroll/src/renroll"
)

func indexHandler(w http.ResponseWriter, r *http.Request) {
	log.Print("indexhandler - start")
	index := struct{ Conf conf.Configuration }{conf.Config()}
	t, e := template.ParseFiles("idx.html", "templates/header.html", "templates/topbar.html", "templates/bottombar.html")
	if e != nil {
		panic(e)
	}
	log.Print("indexhandler - execute")
	if e = t.Execute(w, index); e != nil {
		panic(e)
	}
}

func robotsHandler(w http.ResponseWriter, r *http.Request) {
	log.Print("robothandler - start")
	index := struct{ Conf conf.Configuration }{conf.Config()}
	t, e := template.ParseFiles("robots.txt")
	if e != nil {
		panic(e)
	}
	log.Print("robothandler - execute")
	if e = t.Execute(w, index); e != nil {
		panic(e)
	}
}

func googleAdwordsVerifyHandler(w http.ResponseWriter, r *http.Request) {
	log.Print("adwordsVerifyHandler - start")
	index := struct{ Conf conf.Configuration }{conf.Config()}
	t, e := template.ParseFiles("google41fd03a6c9348593.html")
	if e != nil {
		panic(e)
	}
	log.Print("adwordsVerifyHandler - execute")
	if e = t.Execute(w, index); e != nil {
		panic(e)
	}
}

func aboutHandler(w http.ResponseWriter, r *http.Request) {
	about := struct{ Conf conf.Configuration }{conf.Config()}
	t, _ := template.ParseFiles(
		"about.html",
		"templates/header.html",
		"templates/topbar.html",
		"templates/bottombar.html")
	t.Execute(w, about)
}

func unreleasedHandler(w http.ResponseWriter, r *http.Request) {
	conf := struct{ Conf renroll.Configuration }{renroll.Config()}
	conf.Conf.GPlusSigninCallback = "gSettings"
	conf.Conf.FacebookSigninCallback = "fbSettings"
	t, _ := template.ParseFiles(
		"unreleased.html",
		"templates/header.html",
		"templates/topbar.html",
		"templates/bottombar.html")
	t.Execute(w, conf)
}

func getNextIP() string {
	ips := db.DbGetIPs(db.DbName)
	return db.DbGetNextAvailableIP(ips)
}

func createSite(emailAddress, siteName string) {
	domain := siteName + "." + "renfish.com"
	// Add nginx conf file
	nginxConf := `server {
     listen 443 ssl;
    listen [::]:443 ssl;
    server_name  <site-name>;
    ssl_certificate     /etc/letsencrypt/live/<site-name>/cert.pem;
    ssl_certificate_key /etc/letsencrypt/live/<site-name>/privkey.pem;
    ssl_protocols       TLSv1 TLSv1.1 TLSv1.2;
    ssl_ciphers         HIGH:!aNULL:!MD5;
    location / {
             proxy_pass http://<ip-address>:8080;
            proxy_set_header Host $host;
    }
}
server {
    listen              80;
    server_name  <site-name>;
    location / {
            proxy_pass http://<ip-address>;
             proxy_set_header Host $host;
    }
}
`
	ipAddr := getNextIP()
	nginxConf = strings.Replace(nginxConf, "<site-name>", domain, -1)
	nginxConf = strings.Replace(nginxConf, "<ip-address>", ipAddr, -1)
	fileName := "/etc/nginx/sites-available/" + siteName + "." + "renfish.com"
	if err := ioutil.WriteFile(fileName, []byte(nginxConf), 0644); err != nil {
		auth.LogError(fmt.Sprintf("ERROR WRITING NGINX CONF FILE, sitename: %v, filename: %v, err: %v", siteName, fileName, err))
		return
	}

	// create certificate
	out, err := exec.Command("certbot", "certonly", "-n", "-q", "--standalone", "--pre-hook", "service nginx stop", "--post-hook", "service nginx start", "-d", domain).CombinedOutput()
	if err != nil {
		auth.LogError(fmt.Sprintf("CERTBOT ERROR, err: %v, stdout: %v", err, string(out)))
		log.Fatal(err)
	} else {
		fmt.Println("CREATED CERTBOT CERTIFICATE")
	}

	// Link nginx conf file to sites-enabled/
	symlink := "/etc/nginx/sites-enabled/" + siteName + "." + "renfish.com"
	if err := os.Symlink(fileName, symlink); err != nil {
		auth.LogError(fmt.Sprintf("ERROR CREATING NGINX CONF FILE SYMLINK, sitename: %v, filename: %v, symlink: %v, err: %v", siteName, fileName, symlink, err))
		return
	} else {
		fmt.Println("CREATED NGINX CONF FILE")
	}

	// Reload nginx conf
	out, err = exec.Command("nginx", "-s", "reload").CombinedOutput()
	if err != nil {
		auth.LogError(fmt.Sprintf("ERROR RELOADING NGINX CONF, err: %v, stdout: %v", err, string(out)))
		log.Fatal(err)
	} else {
		fmt.Println("RELOADED NGINX CONF")
	}

	// start Gophish container
	out, err = exec.Command("docker", "run", "--net", "gophish", "--ip", ipAddr, "bjwbell/gophish-container", "/gophish/gophish").CombinedOutput()
	if err != nil {
		auth.LogError(fmt.Sprintf("ERROR STARTING GOPHISH CONTAINER, err: %v, stdout: %v", err, string(out)))
		log.Fatal(err)
	} else {
		fmt.Println("STARTED GOPHISH CONTAINER")
	}

	// Save details to database
	if _, success := db.SaveSite(emailAddress, siteName, ipAddr); !success {
		auth.LogError(fmt.Sprintf("ERROR SAVING SITE TO DB email (%s), sitename (%s), ip (%s)",
			emailAddress, siteName, ipAddr))
		log.Fatal(nil)
	} else {
		fmt.Println(fmt.Sprintf("SAVED SITE TO DB email (%s), sitename (%s), ip (%s)", emailAddress, siteName, ipAddr))
	}
	return
}

func createsiteHandler(w http.ResponseWriter, r *http.Request) {
	conf := struct {
		Conf     renroll.Configuration
		Email    string
		SiteName string
	}{renroll.Config(), "", ""}
	conf.Conf.GPlusSigninCallback = "gSettings"
	conf.Conf.FacebookSigninCallback = "fbSettings"
	if err := r.ParseForm(); err != nil {
		auth.LogError(fmt.Sprintf("ERROR PARSEFORM, ERR: %v", err))
		t, _ := template.ParseFiles(
			"setuperror.html",
			"templates/header.html",
			"templates/topbar.html",
			"templates/bottombar.html")
		if err := t.Execute(w, conf); err != nil {
			auth.LogError(fmt.Sprintf("ERROR t.EXECUTE, ERR: %v", err))
		}
	}
	email := r.Form.Get("email")
	siteName := r.Form.Get("sitename")
	conf.Email = email
	conf.SiteName = "https://" + siteName + "." + r.Host
	if email == "" || siteName == "" {
		auth.LogError(fmt.Sprintf("MiSSING EMAIL or SITENAME, email: %v, sitename: %v", email, siteName))
		t, _ := template.ParseFiles(
			"setuperror.html",
			"templates/header.html",
			"templates/topbar.html",
			"templates/bottombar.html")
		if err := t.Execute(w, conf); err != nil {
			auth.LogError(fmt.Sprintf("ERROR t.EXECUTE, ERR: %v", err))
		}
	} else {
		createSite(email, siteName)
		t, _ := template.ParseFiles(
			"setup.html",
			"templates/header.html",
			"templates/topbar.html",
			"templates/bottombar.html")
		if err := t.Execute(w, conf); err != nil {
			auth.LogError(fmt.Sprintf("ERROR t.EXECUTE, ERR: %v", err))
		}
	}
}

func settingsHandler(w http.ResponseWriter, r *http.Request) {
	conf := struct{ Conf conf.Configuration }{conf.Config()}
	conf.Conf.GPlusSigninCallback = "gSettings"
	conf.Conf.FacebookSigninCallback = "fbSettings"
	t, _ := template.ParseFiles(
		"settings.html",
		"templates/header.html",
		"templates/topbar.html",
		"templates/bottombar.html")
	if err := t.Execute(w, conf); err != nil {
		auth.LogError(fmt.Sprintf("ERROR t.EXECUTE, ERR: %v", err))
	}
}

func redir(w http.ResponseWriter, req *http.Request) {
	host := req.Host
	httpsPort := "443"
	if strings.Index(host, ":8080") != -1 {
		httpsPort = "8443"
	}
	host = strings.TrimSuffix(host, ":8080")
	host = strings.TrimSuffix(host, ":80")
	if httpsPort == "443" {
		http.Redirect(w, req, "https://"+host+req.RequestURI, http.StatusMovedPermanently)
	} else {
		http.Redirect(w, req, "https://"+host+":"+httpsPort+req.RequestURI, http.StatusMovedPermanently)
	}
}

func main() {
	http.HandleFunc("/about", aboutHandler)
	http.HandleFunc("/auth/getemail", auth.GetGPlusEmailHandler)
	http.HandleFunc("/createaccount", auth.CreateAccountHandler)
	http.HandleFunc("/index", indexHandler)
	http.HandleFunc("/logerror", auth.LogErrorHandler)
	http.HandleFunc("/oauth2callback", auth.Oauth2callback)
	http.HandleFunc("/settings", settingsHandler)
	http.HandleFunc("/signinform", auth.SigninFormHandler)
	http.HandleFunc("/submit", submit.SubmitHandler)
	http.HandleFunc("/unreleased", unreleasedHandler)
	http.HandleFunc("/createsite", createsiteHandler)

	http.HandleFunc("/index.html", indexHandler)
	http.HandleFunc("/robots.txt", robotsHandler)
	http.HandleFunc("/google41fd03a6c9348593.html", googleAdwordsVerifyHandler)

	http.Handle("/css/", http.StripPrefix("/css/", http.FileServer(http.Dir("./css"))))
	http.Handle("/font-awesome-4.7.0/", http.StripPrefix("/font-awesome-4.7.0/", http.FileServer(http.Dir("./font-awesome-4.7.0"))))
	http.Handle("/fonts/", http.StripPrefix("/fonts/", http.FileServer(http.Dir("./fonts"))))
	http.Handle("/images/", http.StripPrefix("/images/", http.FileServer(http.Dir("./images"))))
	http.Handle("/js/", http.StripPrefix("/js/", http.FileServer(http.Dir("./js"))))
	http.Handle("/screenshots/", http.StripPrefix("/screenshots/", http.FileServer(http.Dir("./screenshots"))))
	http.HandleFunc("/", indexHandler)

	// HTTP to HTTPS redirection
	// go func() {
	// 	err := http.ListenAndServe(":80", http.HandlerFunc(redir))
	// 	if err != nil {
	// 		log.Print("HTTP ListenAndServe :8080", err)
	// 		log.Print("Trying HTTP ListenAndServe :8080.")
	// 		panic(http.ListenAndServe(":8080", http.HandlerFunc(redir)))

	// 	}
	// }()

	if !db.Exists(db.DbName) {
		db.Create(db.DbName)
	}

	go func() {
		err := http.ListenAndServe(":80", nil)
		if err != nil {
			log.Print("HTTP ListenAndServe :80, ", err)
			log.Print("Trying HTTP ListenAndServe :8080.")
			if err != nil {
				panic(http.ListenAndServe(":8080", nil))
			}

		}
	}()

	cert := "/etc/letsencrypt/live/renfish.com/cert.pem"
	privkey := "/etc/letsencrypt/live/renfish.com/privkey.pem"
	if _, err := os.Stat(cert); os.IsNotExist(err) {
		log.Print("cert: ", err)
		cert = "./generate_cert/cert.pem"
	}
	if _, err := os.Stat(privkey); os.IsNotExist(err) {
		log.Print("cert: ", err)
		privkey = "./generate_cert/key.pem"
	}
	err := http.ListenAndServeTLS(":443", cert, privkey, nil)
	if err != nil {
		log.Print("HTTPS ListenAndServe :8443")
		err = http.ListenAndServeTLS(":8443", cert, privkey, nil)
		if err != nil {
			log.Print("HTTPS ListenAndServe: ", err)
		}
	}
}
<commit_msg>User Docker SDK and remove renroll.Config<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"html/template"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"strings"

	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/container"
	"github.com/docker/docker/api/types/network"
	"github.com/docker/docker/client"
	"golang.org/x/net/context"

	"github.com/bjwbell/renfish/auth"
	"github.com/bjwbell/renfish/conf"
	"github.com/bjwbell/renfish/db"
	"github.com/bjwbell/renfish/submit"
)

type Configuration struct {
	GmailAddress           string
	GmailPassword          string
	GoogleClientId         string
	GoogleClientSecret     string
	GooglePlusScopes       string
	GPlusSigninCallback    string
	GoogleAnalyticsId      string
	FacebookScopes         string
	FacebookAppId          string
	FacebookSigninCallback string
}

func Config() Configuration {
	file, _ := os.Open("conf.json")
	decoder := json.NewDecoder(file)
	configuration := Configuration{}
	err := decoder.Decode(&configuration)
	if err != nil {
		log.Fatal(err)
	}
	return configuration
}

func indexHandler(w http.ResponseWriter, r *http.Request) {
	log.Print("indexhandler - start")
	index := struct{ Conf conf.Configuration }{conf.Config()}
	t, e := template.ParseFiles("idx.html", "templates/header.html", "templates/topbar.html", "templates/bottombar.html")
	if e != nil {
		panic(e)
	}
	log.Print("indexhandler - execute")
	if e = t.Execute(w, index); e != nil {
		panic(e)
	}
}

func robotsHandler(w http.ResponseWriter, r *http.Request) {
	log.Print("robothandler - start")
	index := struct{ Conf conf.Configuration }{conf.Config()}
	t, e := template.ParseFiles("robots.txt")
	if e != nil {
		panic(e)
	}
	log.Print("robothandler - execute")
	if e = t.Execute(w, index); e != nil {
		panic(e)
	}
}

func googleAdwordsVerifyHandler(w http.ResponseWriter, r *http.Request) {
	log.Print("adwordsVerifyHandler - start")
	index := struct{ Conf conf.Configuration }{conf.Config()}
	t, e := template.ParseFiles("google41fd03a6c9348593.html")
	if e != nil {
		panic(e)
	}
	log.Print("adwordsVerifyHandler - execute")
	if e = t.Execute(w, index); e != nil {
		panic(e)
	}
}

func aboutHandler(w http.ResponseWriter, r *http.Request) {
	about := struct{ Conf conf.Configuration }{conf.Config()}
	t, _ := template.ParseFiles(
		"about.html",
		"templates/header.html",
		"templates/topbar.html",
		"templates/bottombar.html")
	t.Execute(w, about)
}

func unreleasedHandler(w http.ResponseWriter, r *http.Request) {
	conf := struct{ Conf Configuration }{Config()}
	conf.Conf.GPlusSigninCallback = "gSettings"
	conf.Conf.FacebookSigninCallback = "fbSettings"
	t, _ := template.ParseFiles(
		"unreleased.html",
		"templates/header.html",
		"templates/topbar.html",
		"templates/bottombar.html")
	t.Execute(w, conf)
}

func getNextIP() string {
	ips := db.DbGetIPs(db.DbName)
	return db.DbGetNextAvailableIP(ips)
}

func createSite(emailAddress, siteName string) {
	domain := siteName + "." + "renfish.com"
	// Add nginx conf file
	nginxConf := `server {
     listen 443 ssl;
    listen [::]:443 ssl;
    server_name  <site-name>;
    ssl_certificate     /etc/letsencrypt/live/<site-name>/cert.pem;
    ssl_certificate_key /etc/letsencrypt/live/<site-name>/privkey.pem;
    ssl_protocols       TLSv1 TLSv1.1 TLSv1.2;
    ssl_ciphers         HIGH:!aNULL:!MD5;
    location / {
             proxy_pass http://<ip-address>:8080;
            proxy_set_header Host $host;
    }
}
server {
    listen              80;
    server_name  <site-name>;
    location / {
            proxy_pass http://<ip-address>;
             proxy_set_header Host $host;
    }
}
`
	ipAddr := getNextIP()
	nginxConf = strings.Replace(nginxConf, "<site-name>", domain, -1)
	nginxConf = strings.Replace(nginxConf, "<ip-address>", ipAddr, -1)
	fileName := "/etc/nginx/sites-available/" + siteName + "." + "renfish.com"
	if err := ioutil.WriteFile(fileName, []byte(nginxConf), 0644); err != nil {
		auth.LogError(fmt.Sprintf("ERROR WRITING NGINX CONF FILE, sitename: %v, filename: %v, err: %v", siteName, fileName, err))
		return
	}

	// create certificate
	out, err := exec.Command("certbot", "certonly", "-n", "-q", "--standalone", "--pre-hook", "service nginx stop", "--post-hook", "service nginx start", "-d", domain).CombinedOutput()
	if err != nil {
		auth.LogError(fmt.Sprintf("CERTBOT ERROR, err: %v, stdout: %v", err, string(out)))
		log.Fatal(err)
	} else {
		fmt.Println("CREATED CERTBOT CERTIFICATE")
	}

	// Link nginx conf file to sites-enabled/
	symlink := "/etc/nginx/sites-enabled/" + siteName + "." + "renfish.com"
	if err := os.Symlink(fileName, symlink); err != nil {
		auth.LogError(fmt.Sprintf("ERROR CREATING NGINX CONF FILE SYMLINK, sitename: %v, filename: %v, symlink: %v, err: %v", siteName, fileName, symlink, err))
		return
	} else {
		fmt.Println("CREATED NGINX CONF FILE")
	}

	// Reload nginx conf
	out, err = exec.Command("nginx", "-s", "reload").CombinedOutput()
	if err != nil {
		auth.LogError(fmt.Sprintf("ERROR RELOADING NGINX CONF, err: %v, stdout: %v", err, string(out)))
		log.Fatal(err)
	} else {
		fmt.Println("RELOADED NGINX CONF")
	}

	// start Gophish container
	ctx := context.Background()
	cli, err := client.NewEnvClient()
	if err != nil {
		panic(err)
	}

	imageName := "bjwbell/gophish-container"
	out3, err3 := cli.ImagePull(ctx, imageName, types.ImagePullOptions{})
	if err3 != nil {
		panic(err3)
	}
	io.Copy(os.Stdout, out3)

	// createConfig := types.ContainerCreateConfig{
	// 	Name:             "",
	// 	Config:           nil,  //*container.Config
	// 	HostConfig:       nil,  // *container.HostConfig
	// 	NetworkingConfig: nil,  // *network.NetworkingConfig
	// 	AdjustCPUShares:  true, //bool
	// }

	var nsconfig map[string]*network.EndpointSettings
	nsconfig["gophish"] = nil
	networkConfig := network.NetworkingConfig{EndpointsConfig: nsconfig}
	resp, err3 := cli.ContainerCreate(ctx, &container.Config{
		Image: imageName,
	}, nil, &networkConfig, "")
	if err3 != nil {
		panic(err3)
	}

	if err3 := cli.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{}); err3 != nil {
		panic(err3)
	}

	fmt.Println(resp.ID)

	// out, err = exec.Command("docker", "run", "--net", "gophish", "--ip", ipAddr, "bjwbell/gophish-container", "/gophish/gophish").CombinedOutput()
	// if err != nil {
	// 	auth.LogError(fmt.Sprintf("ERROR STARTING GOPHISH CONTAINER, err: %v, stdout: %v", err, string(out)))
	// 	log.Fatal(err)
	// } else {
	// 	fmt.Println("STARTED GOPHISH CONTAINER")
	// }

	// Save details to database
	if _, success := db.SaveSite(emailAddress, siteName, ipAddr); !success {
		auth.LogError(fmt.Sprintf("ERROR SAVING SITE TO DB email (%s), sitename (%s), ip (%s)",
			emailAddress, siteName, ipAddr))
		log.Fatal(nil)
	} else {
		fmt.Println(fmt.Sprintf("SAVED SITE TO DB email (%s), sitename (%s), ip (%s)", emailAddress, siteName, ipAddr))
	}
	return
}

func createsiteHandler(w http.ResponseWriter, r *http.Request) {
	conf := struct {
		Conf     Configuration
		Email    string
		SiteName string
	}{Config(), "", ""}
	conf.Conf.GPlusSigninCallback = "gSettings"
	conf.Conf.FacebookSigninCallback = "fbSettings"
	if err := r.ParseForm(); err != nil {
		auth.LogError(fmt.Sprintf("ERROR PARSEFORM, ERR: %v", err))
		t, _ := template.ParseFiles(
			"setuperror.html",
			"templates/header.html",
			"templates/topbar.html",
			"templates/bottombar.html")
		if err := t.Execute(w, conf); err != nil {
			auth.LogError(fmt.Sprintf("ERROR t.EXECUTE, ERR: %v", err))
		}
	}
	email := r.Form.Get("email")
	siteName := r.Form.Get("sitename")
	conf.Email = email
	conf.SiteName = "https://" + siteName + "." + r.Host
	if email == "" || siteName == "" {
		auth.LogError(fmt.Sprintf("MiSSING EMAIL or SITENAME, email: %v, sitename: %v", email, siteName))
		t, _ := template.ParseFiles(
			"setuperror.html",
			"templates/header.html",
			"templates/topbar.html",
			"templates/bottombar.html")
		if err := t.Execute(w, conf); err != nil {
			auth.LogError(fmt.Sprintf("ERROR t.EXECUTE, ERR: %v", err))
		}
	} else {
		createSite(email, siteName)
		t, _ := template.ParseFiles(
			"setup.html",
			"templates/header.html",
			"templates/topbar.html",
			"templates/bottombar.html")
		if err := t.Execute(w, conf); err != nil {
			auth.LogError(fmt.Sprintf("ERROR t.EXECUTE, ERR: %v", err))
		}
	}
}

func settingsHandler(w http.ResponseWriter, r *http.Request) {
	conf := struct{ Conf conf.Configuration }{conf.Config()}
	conf.Conf.GPlusSigninCallback = "gSettings"
	conf.Conf.FacebookSigninCallback = "fbSettings"
	t, _ := template.ParseFiles(
		"settings.html",
		"templates/header.html",
		"templates/topbar.html",
		"templates/bottombar.html")
	if err := t.Execute(w, conf); err != nil {
		auth.LogError(fmt.Sprintf("ERROR t.EXECUTE, ERR: %v", err))
	}
}

func redir(w http.ResponseWriter, req *http.Request) {
	host := req.Host
	httpsPort := "443"
	if strings.Index(host, ":8080") != -1 {
		httpsPort = "8443"
	}
	host = strings.TrimSuffix(host, ":8080")
	host = strings.TrimSuffix(host, ":80")
	if httpsPort == "443" {
		http.Redirect(w, req, "https://"+host+req.RequestURI, http.StatusMovedPermanently)
	} else {
		http.Redirect(w, req, "https://"+host+":"+httpsPort+req.RequestURI, http.StatusMovedPermanently)
	}
}

func main() {
	http.HandleFunc("/about", aboutHandler)
	http.HandleFunc("/auth/getemail", auth.GetGPlusEmailHandler)
	http.HandleFunc("/createaccount", auth.CreateAccountHandler)
	http.HandleFunc("/index", indexHandler)
	http.HandleFunc("/logerror", auth.LogErrorHandler)
	http.HandleFunc("/oauth2callback", auth.Oauth2callback)
	http.HandleFunc("/settings", settingsHandler)
	http.HandleFunc("/signinform", auth.SigninFormHandler)
	http.HandleFunc("/submit", submit.SubmitHandler)
	http.HandleFunc("/unreleased", unreleasedHandler)
	http.HandleFunc("/createsite", createsiteHandler)

	http.HandleFunc("/index.html", indexHandler)
	http.HandleFunc("/robots.txt", robotsHandler)
	http.HandleFunc("/google41fd03a6c9348593.html", googleAdwordsVerifyHandler)

	http.Handle("/css/", http.StripPrefix("/css/", http.FileServer(http.Dir("./css"))))
	http.Handle("/font-awesome-4.7.0/", http.StripPrefix("/font-awesome-4.7.0/", http.FileServer(http.Dir("./font-awesome-4.7.0"))))
	http.Handle("/fonts/", http.StripPrefix("/fonts/", http.FileServer(http.Dir("./fonts"))))
	http.Handle("/images/", http.StripPrefix("/images/", http.FileServer(http.Dir("./images"))))
	http.Handle("/js/", http.StripPrefix("/js/", http.FileServer(http.Dir("./js"))))
	http.Handle("/screenshots/", http.StripPrefix("/screenshots/", http.FileServer(http.Dir("./screenshots"))))
	http.HandleFunc("/", indexHandler)

	// HTTP to HTTPS redirection
	// go func() {
	// 	err := http.ListenAndServe(":80", http.HandlerFunc(redir))
	// 	if err != nil {
	// 		log.Print("HTTP ListenAndServe :8080", err)
	// 		log.Print("Trying HTTP ListenAndServe :8080.")
	// 		panic(http.ListenAndServe(":8080", http.HandlerFunc(redir)))

	// 	}
	// }()

	if !db.Exists(db.DbName) {
		db.Create(db.DbName)
	}

	go func() {
		err := http.ListenAndServe(":80", nil)
		if err != nil {
			log.Print("HTTP ListenAndServe :80, ", err)
			log.Print("Trying HTTP ListenAndServe :8080.")
			if err != nil {
				panic(http.ListenAndServe(":8080", nil))
			}

		}
	}()

	cert := "/etc/letsencrypt/live/renfish.com/cert.pem"
	privkey := "/etc/letsencrypt/live/renfish.com/privkey.pem"
	if _, err := os.Stat(cert); os.IsNotExist(err) {
		log.Print("cert: ", err)
		cert = "./generate_cert/cert.pem"
	}
	if _, err := os.Stat(privkey); os.IsNotExist(err) {
		log.Print("cert: ", err)
		privkey = "./generate_cert/key.pem"
	}
	err := http.ListenAndServeTLS(":443", cert, privkey, nil)
	if err != nil {
		log.Print("HTTPS ListenAndServe :8443")
		err = http.ListenAndServeTLS(":8443", cert, privkey, nil)
		if err != nil {
			log.Print("HTTPS ListenAndServe: ", err)
		}
	}
}
<|endoftext|>


<commit_before>package river

import (
	rss "github.com/jteeuwen/go-pkg-rss"
	"time"
	"log"
)

type River interface {
	Latest() <-chan Feed
}

type poller struct {
	uri    string
	feed   *rss.Feed
	in     chan Feed
}

func newPoller(uri string) River {
	p := &poller{
	  uri: uri,
	  feed: nil,
	  in: make(chan Feed),
	}

	p.feed = rss.New(5, true, p.chanHandler, p.itemHandler)
	go p.poll()
	return p
}

func (w *poller) poll() {
	w.fetch()

	for {
		select {
		// case <-time.After(time.Duration(w.feed.SecondsTillUpdate()) * time.Second):
		case <-time.After(5 * time.Second):
			w.fetch()
		}
	}
}

func (w *poller) fetch() {
	if err := w.feed.Fetch(w.uri, nil); err != nil {
		log.Println("error fetching", w.uri + ":", err)
	}
}

func (w *poller) chanHandler(feed *rss.Feed, newchannels []*rss.Channel) {
	// ignore
}

func (w *poller) itemHandler(feed *rss.Feed, ch *rss.Channel, newitems []*rss.Item) {
	items := []Item{}
	for _, item := range newitems {
		converted := convertItem(item)

		if converted == nil { continue }

		items = append(items, *converted)
	}

	log.Println(len(items), "new item(s) in", feed.Url)
	if len(items) == 0 { return }

	feedUrl := feed.Url
	websiteUrl := ""
	for _, link := range ch.Links {
		if feedUrl != "" && websiteUrl != "" { break }

		if link.Rel == "self" {
			feedUrl = link.Href
		} else {
			websiteUrl = link.Href
		}
	}

	toSend := Feed{
  	FeedUrl: feedUrl,
    WebsiteUrl: websiteUrl,
	  FeedTitle: ch.Title,
  	FeedDescription: ch.Description,
	  WhenLastUpdate: RssTime{time.Now()},
	  Items: items,
	}

	w.in <- toSend
}

func (w *poller) Latest() <-chan Feed {
	c := make(chan Feed)
	go func() {
		for {
			in := <-w.in
			c <- in
		}
	}()
	return c
}
<commit_msg>remove unnecessary go routine<commit_after>package river

import (
	rss "github.com/jteeuwen/go-pkg-rss"
	"time"
	"log"
)

type River interface {
	Latest() <-chan Feed
}

type poller struct {
	uri    string
	feed   *rss.Feed
	in     chan Feed
}

func newPoller(uri string) River {
	p := &poller{}
	p.uri = uri
	p.in = make(chan Feed)
	p.feed = rss.New(5, true, p.chanHandler, p.itemHandler)

	go p.poll()
	return p
}

func (w *poller) poll() {
	w.fetch()

	for {
		select {
		case <-time.After(time.Duration(w.feed.SecondsTillUpdate()) * time.Second):
			w.fetch()
		}
	}
}

func (w *poller) fetch() {
	if err := w.feed.Fetch(w.uri, nil); err != nil {
		log.Println("error fetching", w.uri + ":", err)
	}
}

func (w *poller) chanHandler(feed *rss.Feed, newchannels []*rss.Channel) {}

func (w *poller) itemHandler(feed *rss.Feed, ch *rss.Channel, newitems []*rss.Item) {
	items := []Item{}
	for _, item := range newitems {
		converted := convertItem(item)

		if converted != nil {
			items = append(items, *converted)
		}
	}

	log.Println(len(items), "new item(s) in", feed.Url)
	if len(items) == 0 { return }

	feedUrl := feed.Url
	websiteUrl := ""
	for _, link := range ch.Links {
		if feedUrl != "" && websiteUrl != "" { break }

		if link.Rel == "self" {
			feedUrl = link.Href
		} else {
			websiteUrl = link.Href
		}
	}

	w.in <- Feed{
  	FeedUrl: feedUrl,
    WebsiteUrl: websiteUrl,
	  FeedTitle: ch.Title,
  	FeedDescription: ch.Description,
	  WhenLastUpdate: RssTime{time.Now()},
	  Items: items,
	}
}

func (w *poller) Latest() <-chan Feed {
	return w.in
}
<|endoftext|>


<commit_before>// Copyright 2016 The go-vgo Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// https://github.com/go-vgo/robotgo/blob/master/LICENSE
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

// +build darwin windows

package robotgo

import (
	"testing"

	"github.com/vcaesar/tt"
)

func TestColor(t *testing.T) {
	s := GetPixelColor(10, 10)
	tt.IsType(t, "string", s)
	tt.NotEmpty(t, s)

	c := GetPxColor(10, 10)
	s1 := PadHex(c)
	tt.Equal(t, s, s1)
}

func TestSize(t *testing.T) {
	x, y := GetScreenSize()
	tt.NotZero(t, x)
	tt.NotZero(t, y)

	x, y = GetScaleSize()
	tt.NotZero(t, x)
	tt.NotZero(t, x)
}

func TestMoveMouse(t *testing.T) {
	MoveMouse(20, 20)
	MilliSleep(10)
	x, y := GetMousePos()

	tt.Equal(t, 20, x)
	tt.Equal(t, 20, y)
}

func TestMoveMouseSmooth(t *testing.T) {
	MoveMouseSmooth(100, 100)
	MilliSleep(10)
	x, y := GetMousePos()

	tt.Equal(t, 100, x)
	tt.Equal(t, 100, y)
}

func TestDragMouse(t *testing.T) {
	DragMouse(500, 500)
	MilliSleep(10)
	x, y := GetMousePos()

	tt.Equal(t, 500, x)
	tt.Equal(t, 500, y)
}

func TestScrollMouse(t *testing.T) {
	ScrollMouse(120, "up")
	MilliSleep(100)

	Scroll(210, 210)
}

func TestMoveRelative(t *testing.T) {
	Move(200, 200)
	MilliSleep(10)

	MoveRelative(10, -10)
	MilliSleep(10)

	x, y := GetMousePos()
	tt.Equal(t, 210, x)
	tt.Equal(t, 190, y)
}

func TestMoveSmoothRelative(t *testing.T) {
	Move(200, 200)
	MilliSleep(10)

	MoveSmoothRelative(10, -10)
	MilliSleep(10)

	x, y := GetMousePos()
	tt.Equal(t, 210, x)
	tt.Equal(t, 190, y)
}

func TestKey(t *testing.T) {
	e := KeyTap("v", "cmd")
	tt.Empty(t, e)

	e = KeyToggle("v", "up")
	tt.Empty(t, e)
}

func TestTypeStr(t *testing.T) {
	c := CharCodeAt("s", 0)
	tt.Equal(t, 115, c)
}

func TestBitmap(t *testing.T) {
	bit := CaptureScreen()
	tt.NotNil(t, bit)
	e := SaveBitmap(bit, "robot_test.png")
	tt.Empty(t, e)

	bit1 := OpenBitmap("robot_test.png")
	b := tt.TypeOf(bit, bit1)
	tt.True(t, b)
	tt.NotNil(t, bit1)
}
<commit_msg>update move mouse smooth test code<commit_after>// Copyright 2016 The go-vgo Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// https://github.com/go-vgo/robotgo/blob/master/LICENSE
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

// +build darwin windows

package robotgo

import (
	"testing"

	"github.com/vcaesar/tt"
)

func TestColor(t *testing.T) {
	s := GetPixelColor(10, 10)
	tt.IsType(t, "string", s)
	tt.NotEmpty(t, s)

	c := GetPxColor(10, 10)
	s1 := PadHex(c)
	tt.Equal(t, s, s1)
}

func TestSize(t *testing.T) {
	x, y := GetScreenSize()
	tt.NotZero(t, x)
	tt.NotZero(t, y)

	x, y = GetScaleSize()
	tt.NotZero(t, x)
	tt.NotZero(t, x)
}

func TestMoveMouse(t *testing.T) {
	MoveMouse(20, 20)
	MilliSleep(10)
	x, y := GetMousePos()

	tt.Equal(t, 20, x)
	tt.Equal(t, 20, y)
}

func TestMoveMouseSmooth(t *testing.T) {
	b := MoveMouseSmooth(100, 100)
	MilliSleep(10)
	x, y := GetMousePos()

	tt.True(t, b)
	tt.Equal(t, 100, x)
	tt.Equal(t, 100, y)
}

func TestDragMouse(t *testing.T) {
	DragMouse(500, 500)
	MilliSleep(10)
	x, y := GetMousePos()

	tt.Equal(t, 500, x)
	tt.Equal(t, 500, y)
}

func TestScrollMouse(t *testing.T) {
	ScrollMouse(120, "up")
	MilliSleep(100)

	Scroll(210, 210)
}

func TestMoveRelative(t *testing.T) {
	Move(200, 200)
	MilliSleep(10)

	MoveRelative(10, -10)
	MilliSleep(10)

	x, y := GetMousePos()
	tt.Equal(t, 210, x)
	tt.Equal(t, 190, y)
}

func TestMoveSmoothRelative(t *testing.T) {
	Move(200, 200)
	MilliSleep(10)

	MoveSmoothRelative(10, -10)
	MilliSleep(10)

	x, y := GetMousePos()
	tt.Equal(t, 210, x)
	tt.Equal(t, 190, y)
}

func TestKey(t *testing.T) {
	e := KeyTap("v", "cmd")
	tt.Empty(t, e)

	e = KeyToggle("v", "up")
	tt.Empty(t, e)
}

func TestTypeStr(t *testing.T) {
	c := CharCodeAt("s", 0)
	tt.Equal(t, 115, c)
}

func TestBitmap(t *testing.T) {
	bit := CaptureScreen()
	tt.NotNil(t, bit)
	e := SaveBitmap(bit, "robot_test.png")
	tt.Empty(t, e)

	bit1 := OpenBitmap("robot_test.png")
	b := tt.TypeOf(bit, bit1)
	tt.True(t, b)
	tt.NotNil(t, bit1)
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
	"fmt"
	"image"
	"image/draw"
	"image/jpeg"
	"image/png"
	"io"
	"log"
	"os"
	"path"
	"runtime"
	"strings"
)

type Ratio struct {
	w int
	h int
}

var dir string
var back image.Image
var ratios []Ratio

// box function creates to complementary rectangle for the given image size `xx` and `yy`
func box(xx int, yy int) (rectx int, recty int, err error) {
	rectx, recty = 0, 0
	surf := -1

	for _, ratio := range ratios {
		w, h := ratio.w, ratio.h
		x, y := xx, yy

		// img & ratio orientation fit
		if (x-y)*(w-h) < 0 {
			w, h = h, w
		}

		if x*h == y*w {
			err = errors.New("Perfect fit, doing nothing")
			continue
		}
		// complement on the right side
		if x*h > y*w {
			y = x * h / w
		} else {
			x = y * w / h
		}

		// select the best available ratio
		if x*y/surf < 1 {
			rectx, recty = x, y
			surf = x * y
		}
	}

	return rectx, recty, err
}

// resize function concatenates the given image with its complementary "bleed"
func resize(filename string, runningjobs chan int, donejobs chan string) {
	in, err := os.Open(filename)
	if err != nil {
		log.Fatal(err)
	}
	defer in.Close()

	src, format, err := image.Decode(in)
	if err != nil {
		log.Fatal(err)
	}

	outfile := path.Join(dir, path.Base(filename))
	out, err := os.Create(outfile)
	if err != nil {
		log.Fatal(err)
	}
	defer out.Close()

	rect := image.ZR
	rectx, recty, errb := box(src.Bounds().Dx(), src.Bounds().Dy())
	if errb != nil {
		io.Copy(in, out)
	} else {
		rect = image.Rect(0, 0, rectx, recty)
		dst := image.NewRGBA(rect)
		draw.Draw(dst, dst.Bounds(), back, image.ZP, draw.Src)
		draw.Draw(dst, src.Bounds(), src, image.ZP, draw.Src)

		switch format {
		case "png":
			png.Encode(out, dst)
		case "jpeg":
			jpeg.Encode(out, dst, nil)
		default:
			log.Fatal("Unknown format ", format)
		}
	}
	fmt.Println("•• Done with ", donejob)
	<-runningjobs
	donejobs <- filename
}

// main triggers and waits for resizing goroutines
func main() {
	var r string
	var c string
	var p int

	flag.StringVar(&dir, "dir", "resized", "Put the resized images in this directory.")
	flag.StringVar(&r, "ratio", "4:3,3:2,5:4", "Use the best ratio from this list.")
	flag.StringVar(&c, "color", "white", "Use this color for padding (white, black or transparent.")
	flag.IntVar(&p, "parallel", runtime.NumCPU(), "Handle images in parallel, defaults to the actuel number of CPU available.")
	flag.Parse()
	args := flag.Args()
	if len(args) == 0 {
		log.Fatal("No images to resize.\n")
	}

	runtime.GOMAXPROCS(p)
	var runningjobs = make(chan int, p)
	var donejobs = make(chan string, len(args))
	var donejob string

	switch c {
	case "white":
		back = image.White
	case "black":
		back = image.Black
	case "transparent":
		back = image.Transparent
	default:
		log.Fatal("Unknown color ", c)
	}
	parts := strings.Split(r, ",")
	ratios = make([]Ratio, len(parts))
	for i, part := range parts {
		fmt.Sscanf(part, "%d:%d", &ratios[i].w, &ratios[i].h)
	}
	os.MkdirAll(dir, 0755)

	go func() {
		for job, filename := range args {
			runningjobs <- job
			fmt.Println("Bleeding ", filename)
			go resize(filename, runningjobs, donejobs)
		}
	}()

	for _, _ = range args {
		donejob = <-donejobs
	}
}
<commit_msg>now without sync package and without stupid typo<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"image"
	"image/draw"
	"image/jpeg"
	"image/png"
	"io"
	"log"
	"os"
	"path"
	"runtime"
	"strings"
)

type Ratio struct {
	w int
	h int
}

var dir string
var back image.Image
var ratios []Ratio

// box function creates to complementary rectangle for the given image size `xx` and `yy`
func box(xx int, yy int) (rectx int, recty int, err error) {
	rectx, recty = 0, 0
	surf := -1

	for _, ratio := range ratios {
		w, h := ratio.w, ratio.h
		x, y := xx, yy

		// img & ratio orientation fit
		if (x-y)*(w-h) < 0 {
			w, h = h, w
		}

		if x*h == y*w {
			err = errors.New("Perfect fit, doing nothing")
			continue
		}
		// complement on the right side
		if x*h > y*w {
			y = x * h / w
		} else {
			x = y * w / h
		}

		// select the best available ratio
		if x*y/surf < 1 {
			rectx, recty = x, y
			surf = x * y
		}
	}

	return rectx, recty, err
}

// resize function concatenates the given image with its complementary "bleed"
func resize(filename string, runningjobs chan int, donejobs chan string) {
	in, err := os.Open(filename)
	if err != nil {
		log.Fatal(err)
	}
	defer in.Close()

	src, format, err := image.Decode(in)
	if err != nil {
		log.Fatal(err)
	}

	outfile := path.Join(dir, path.Base(filename))
	out, err := os.Create(outfile)
	if err != nil {
		log.Fatal(err)
	}
	defer out.Close()

	rect := image.ZR
	rectx, recty, errb := box(src.Bounds().Dx(), src.Bounds().Dy())
	if errb != nil {
		io.Copy(in, out)
	} else {
		rect = image.Rect(0, 0, rectx, recty)
		dst := image.NewRGBA(rect)
		draw.Draw(dst, dst.Bounds(), back, image.ZP, draw.Src)
		draw.Draw(dst, src.Bounds(), src, image.ZP, draw.Src)

		switch format {
		case "png":
			png.Encode(out, dst)
		case "jpeg":
			jpeg.Encode(out, dst, nil)
		default:
			log.Fatal("Unknown format ", format)
		}
	}
	fmt.Println("•• Done with ", filename)
	<-runningjobs
	donejobs <- filename
}

// main triggers and waits for resizing goroutines
func main() {
	var r string
	var c string
	var p int

	flag.StringVar(&dir, "dir", "resized", "Put the resized images in this directory.")
	flag.StringVar(&r, "ratio", "4:3,3:2,5:4", "Use the best ratio from this list.")
	flag.StringVar(&c, "color", "white", "Use this color for padding (white, black or transparent.")
	flag.IntVar(&p, "parallel", runtime.NumCPU(), "Handle images in parallel, defaults to the actuel number of CPU available.")
	flag.Parse()
	args := flag.Args()
	if len(args) == 0 {
		log.Fatal("No images to resize.\n")
	}

	runtime.GOMAXPROCS(p)
	var runningjobs = make(chan int, p)
	var donejobs = make(chan string, len(args))

	switch c {
	case "white":
		back = image.White
	case "black":
		back = image.Black
	case "transparent":
		back = image.Transparent
	default:
		log.Fatal("Unknown color ", c)
	}
	parts := strings.Split(r, ",")
	ratios = make([]Ratio, len(parts))
	for i, part := range parts {
		fmt.Sscanf(part, "%d:%d", &ratios[i].w, &ratios[i].h)
	}
	os.MkdirAll(dir, 0755)

	go func() {
		for job, filename := range args {
			runningjobs <- job
			fmt.Println("Bleeding ", filename)
			go resize(filename, runningjobs, donejobs)
		}
	}()

	for _, _ = range args {
		<-donejobs
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License. See the AUTHORS file
// for names of contributors.
//
// Author: Matt Tracy

package stop_test

import (
	"testing"

	"github.com/cockroachdb/cockroach/util/leaktest"
)

//go:generate ../util/leaktest/add-leaktest.sh *_test.go

func TestMain(m *testing.M) {
	leaktest.TestMainWithLeakCheck(m)
}
<commit_msg>util/stop: use correct `add-leaktest.sh` path<commit_after>// Copyright 2015 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License. See the AUTHORS file
// for names of contributors.
//
// Author: Matt Tracy

package stop_test

import (
	"testing"

	"github.com/cockroachdb/cockroach/util/leaktest"
)

//go:generate ../leaktest/add-leaktest.sh *_test.go

func TestMain(m *testing.M) {
	leaktest.TestMainWithLeakCheck(m)
}
<|endoftext|>


<commit_before>// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package types

import (
	"fmt"
	"math"

	"github.com/juju/errors"
)

// AddUint64 adds uint64 a and b if no overflow, else returns error.
func AddUint64(a uint64, b uint64) (uint64, error) {
	if math.MaxUint64-a < b {
		return 0, errors.Trace(ErrOverflow.GenByArgs("BIGINT UNSIGNED", fmt.Sprintf("(%d, %d)", a, b)))
	}
	return a + b, nil
}

// AddInt64 adds int64 a and b if no overflow, otherwise returns error.
func AddInt64(a int64, b int64) (int64, error) {
	if (a > 0 && b > 0 && math.MaxInt64-a < b) ||
		(a < 0 && b < 0 && math.MinInt64-a > b) {
		return 0, errors.Trace(ErrOverflow.GenByArgs("BIGINT", fmt.Sprintf("(%d, %d)", a, b)))
	}

	return a + b, nil
}

// AddInteger adds uint64 a and int64 b and returns uint64 if no overflow error.
func AddInteger(a uint64, b int64) (uint64, error) {
	if b >= 0 {
		return AddUint64(a, uint64(b))
	}

	if uint64(-b) > a {
		return 0, errors.Trace(ErrOverflow.GenByArgs("BIGINT UNSIGNED", fmt.Sprintf("(%d, %d)", a, b)))
	}
	return a - uint64(-b), nil
}

// SubUint64 subtracts uint64 a with b and returns uint64 if no overflow error.
func SubUint64(a uint64, b uint64) (uint64, error) {
	if a < b {
		return 0, errors.Trace(ErrOverflow.GenByArgs("BIGINT UNSIGNED", fmt.Sprintf("(%d, %d)", a, b)))
	}
	return a - b, nil
}

// SubInt64 subtracts int64 a with b and returns int64 if no overflow error.
func SubInt64(a int64, b int64) (int64, error) {
	if (a > 0 && b < 0 && math.MaxInt64-a < -b) ||
		(a < 0 && b > 0 && math.MinInt64-a > -b) ||
		(a == 0 && b == math.MinInt64) {
		return 0, errors.Trace(ErrOverflow.GenByArgs("BIGINT", fmt.Sprintf("(%d, %d)", a, b)))
	}
	return a - b, nil
}

// SubUintWithInt subtracts uint64 a with int64 b and returns uint64 if no overflow error.
func SubUintWithInt(a uint64, b int64) (uint64, error) {
	if b < 0 {
		return AddUint64(a, uint64(-b))
	}
	return SubUint64(a, uint64(b))
}

// SubIntWithUint subtracts int64 a with uint64 b and returns uint64 if no overflow error.
func SubIntWithUint(a int64, b uint64) (uint64, error) {
	if a < 0 || uint64(a) < b {
		return 0, errors.Trace(ErrOverflow.GenByArgs("BIGINT UNSIGNED", fmt.Sprintf("(%d, %d)", a, b)))
	}
	return uint64(a) - b, nil
}

// MulUint64 multiplies uint64 a and b and returns uint64 if no overflow error.
func MulUint64(a uint64, b uint64) (uint64, error) {
	if b > 0 && a > math.MaxUint64/b {
		return 0, errors.Trace(ErrOverflow.GenByArgs("BIGINT UNSIGNED", fmt.Sprintf("(%d, %d)", a, b)))
	}
	return a * b, nil
}

// MulInt64 multiplies int64 a and b and returns int64 if no overflow error.
func MulInt64(a int64, b int64) (int64, error) {
	if a == 0 || b == 0 {
		return 0, nil
	}

	var (
		res      uint64
		err      error
		negative = false
	)

	if a > 0 && b > 0 {
		res, err = MulUint64(uint64(a), uint64(b))
	} else if a < 0 && b < 0 {
		res, err = MulUint64(uint64(-a), uint64(-b))
	} else if a < 0 && b > 0 {
		negative = true
		res, err = MulUint64(uint64(-a), uint64(b))
	} else {
		negative = true
		res, err = MulUint64(uint64(a), uint64(-b))
	}

	if err != nil {
		return 0, errors.Trace(err)
	}

	if negative {
		// negative result
		if res > math.MaxInt64+1 {
			return 0, errors.Trace(ErrOverflow.GenByArgs("BIGINT", fmt.Sprintf("(%d, %d)", a, b)))
		}

		return -int64(res), nil
	}

	// positive result
	if res > math.MaxInt64 {
		return 0, errors.Trace(ErrOverflow.GenByArgs("BIGINT", fmt.Sprintf("(%d, %d)", a, b)))
	}

	return int64(res), nil
}

// MulInteger multiplies uint64 a and int64 b, and returns uint64 if no overflow error.
func MulInteger(a uint64, b int64) (uint64, error) {
	if a == 0 || b == 0 {
		return 0, nil
	}

	if b < 0 {
		return 0, errors.Trace(ErrOverflow.GenByArgs("BIGINT UNSIGNED", fmt.Sprintf("(%d, %d)", a, b)))
	}

	return MulUint64(a, uint64(b))
}

// DivInt64 divides int64 a with b, returns int64 if no overflow error.
// It just checks overflow, if b is zero, a "divide by zero" panic throws.
func DivInt64(a int64, b int64) (int64, error) {
	if a == math.MinInt64 && b == -1 {
		return 0, errors.Trace(ErrOverflow.GenByArgs("BIGINT", fmt.Sprintf("(%d, %d)", a, b)))
	}

	return a / b, nil
}

// DivUintWithInt divides uint64 a with int64 b, returns uint64 if no overflow error.
// It just checks overflow, if b is zero, a "divide by zero" panic throws.
func DivUintWithInt(a uint64, b int64) (uint64, error) {
	if b < 0 {
		if a != 0 && uint64(-b) <= a {
			return 0, errors.Trace(ErrOverflow.GenByArgs("BIGINT UNSIGNED", fmt.Sprintf("(%d, %d)", a, b)))
		}

		return 0, nil
	}

	return a / uint64(b), nil
}

// DivIntWithUint divides int64 a with uint64 b, returns uint64 if no overflow error.
// It just checks overflow, if b is zero, a "divide by zero" panic throws.
func DivIntWithUint(a int64, b uint64) (uint64, error) {
	if a < 0 {
		if uint64(-a) >= b {
			return 0, errors.Trace(ErrOverflow.GenByArgs("BIGINT", fmt.Sprintf("(%d, %d)", a, b)))
		}

		return 0, nil
	}

	return uint64(a) / b, nil
}
<commit_msg>util/types: cleanup error.Trace (#3113)<commit_after>// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package types

import (
	"fmt"
	"math"

	"github.com/juju/errors"
)

// AddUint64 adds uint64 a and b if no overflow, else returns error.
func AddUint64(a uint64, b uint64) (uint64, error) {
	if math.MaxUint64-a < b {
		return 0, ErrOverflow.GenByArgs("BIGINT UNSIGNED", fmt.Sprintf("(%d, %d)", a, b))
	}
	return a + b, nil
}

// AddInt64 adds int64 a and b if no overflow, otherwise returns error.
func AddInt64(a int64, b int64) (int64, error) {
	if (a > 0 && b > 0 && math.MaxInt64-a < b) ||
		(a < 0 && b < 0 && math.MinInt64-a > b) {
		return 0, ErrOverflow.GenByArgs("BIGINT", fmt.Sprintf("(%d, %d)", a, b))
	}

	return a + b, nil
}

// AddInteger adds uint64 a and int64 b and returns uint64 if no overflow error.
func AddInteger(a uint64, b int64) (uint64, error) {
	if b >= 0 {
		return AddUint64(a, uint64(b))
	}

	if uint64(-b) > a {
		return 0, ErrOverflow.GenByArgs("BIGINT UNSIGNED", fmt.Sprintf("(%d, %d)", a, b))
	}
	return a - uint64(-b), nil
}

// SubUint64 subtracts uint64 a with b and returns uint64 if no overflow error.
func SubUint64(a uint64, b uint64) (uint64, error) {
	if a < b {
		return 0, ErrOverflow.GenByArgs("BIGINT UNSIGNED", fmt.Sprintf("(%d, %d)", a, b))
	}
	return a - b, nil
}

// SubInt64 subtracts int64 a with b and returns int64 if no overflow error.
func SubInt64(a int64, b int64) (int64, error) {
	if (a > 0 && b < 0 && math.MaxInt64-a < -b) ||
		(a < 0 && b > 0 && math.MinInt64-a > -b) ||
		(a == 0 && b == math.MinInt64) {
		return 0, ErrOverflow.GenByArgs("BIGINT", fmt.Sprintf("(%d, %d)", a, b))
	}
	return a - b, nil
}

// SubUintWithInt subtracts uint64 a with int64 b and returns uint64 if no overflow error.
func SubUintWithInt(a uint64, b int64) (uint64, error) {
	if b < 0 {
		return AddUint64(a, uint64(-b))
	}
	return SubUint64(a, uint64(b))
}

// SubIntWithUint subtracts int64 a with uint64 b and returns uint64 if no overflow error.
func SubIntWithUint(a int64, b uint64) (uint64, error) {
	if a < 0 || uint64(a) < b {
		return 0, ErrOverflow.GenByArgs("BIGINT UNSIGNED", fmt.Sprintf("(%d, %d)", a, b))
	}
	return uint64(a) - b, nil
}

// MulUint64 multiplies uint64 a and b and returns uint64 if no overflow error.
func MulUint64(a uint64, b uint64) (uint64, error) {
	if b > 0 && a > math.MaxUint64/b {
		return 0, ErrOverflow.GenByArgs("BIGINT UNSIGNED", fmt.Sprintf("(%d, %d)", a, b))
	}
	return a * b, nil
}

// MulInt64 multiplies int64 a and b and returns int64 if no overflow error.
func MulInt64(a int64, b int64) (int64, error) {
	if a == 0 || b == 0 {
		return 0, nil
	}

	var (
		res      uint64
		err      error
		negative = false
	)

	if a > 0 && b > 0 {
		res, err = MulUint64(uint64(a), uint64(b))
	} else if a < 0 && b < 0 {
		res, err = MulUint64(uint64(-a), uint64(-b))
	} else if a < 0 && b > 0 {
		negative = true
		res, err = MulUint64(uint64(-a), uint64(b))
	} else {
		negative = true
		res, err = MulUint64(uint64(a), uint64(-b))
	}

	if err != nil {
		return 0, errors.Trace(err)
	}

	if negative {
		// negative result
		if res > math.MaxInt64+1 {
			return 0, ErrOverflow.GenByArgs("BIGINT", fmt.Sprintf("(%d, %d)", a, b))
		}

		return -int64(res), nil
	}

	// positive result
	if res > math.MaxInt64 {
		return 0, ErrOverflow.GenByArgs("BIGINT", fmt.Sprintf("(%d, %d)", a, b))
	}

	return int64(res), nil
}

// MulInteger multiplies uint64 a and int64 b, and returns uint64 if no overflow error.
func MulInteger(a uint64, b int64) (uint64, error) {
	if a == 0 || b == 0 {
		return 0, nil
	}

	if b < 0 {
		return 0, ErrOverflow.GenByArgs("BIGINT UNSIGNED", fmt.Sprintf("(%d, %d)", a, b))
	}

	return MulUint64(a, uint64(b))
}

// DivInt64 divides int64 a with b, returns int64 if no overflow error.
// It just checks overflow, if b is zero, a "divide by zero" panic throws.
func DivInt64(a int64, b int64) (int64, error) {
	if a == math.MinInt64 && b == -1 {
		return 0, ErrOverflow.GenByArgs("BIGINT", fmt.Sprintf("(%d, %d)", a, b))
	}

	return a / b, nil
}

// DivUintWithInt divides uint64 a with int64 b, returns uint64 if no overflow error.
// It just checks overflow, if b is zero, a "divide by zero" panic throws.
func DivUintWithInt(a uint64, b int64) (uint64, error) {
	if b < 0 {
		if a != 0 && uint64(-b) <= a {
			return 0, ErrOverflow.GenByArgs("BIGINT UNSIGNED", fmt.Sprintf("(%d, %d)", a, b))
		}

		return 0, nil
	}

	return a / uint64(b), nil
}

// DivIntWithUint divides int64 a with uint64 b, returns uint64 if no overflow error.
// It just checks overflow, if b is zero, a "divide by zero" panic throws.
func DivIntWithUint(a int64, b uint64) (uint64, error) {
	if a < 0 {
		if uint64(-a) >= b {
			return 0, ErrOverflow.GenByArgs("BIGINT", fmt.Sprintf("(%d, %d)", a, b))
		}

		return 0, nil
	}

	return uint64(a) / b, nil
}
<|endoftext|>


<commit_before>// Package camoproxy provides an HTTP proxy server with content type
// restrictions as well as regex host allow and deny list support.
package camoproxy

import (
	"code.google.com/p/gorilla/mux"
	"errors"
	"io"
	"net"
	"net/http"
	"net/url"
	"regexp"
	"strings"
	"time"
)

// Config holds configuration data used when creating a
// Proxy with New.
type Config struct {
	// HmacKey is a string to be used as the hmac key
	HmacKey         string
	// AllowList is a list of string represenstations of regex (not compiled
	// regex) that are used as a whitelist filter. If an AllowList is present,
	// then anything not matching is dropped. If no AllowList is present,
	// no Allow filtering is done.
	AllowList       []string
	// DenyList is a list of string represenstations of regex (not compiled
	// regex). The deny filter check occurs after the allow filter check
	// (if any).
	DenyList        []string
	// MaxSize is the maximum valid image size response (in bytes).
	MaxSize         int64
	// FollowRedirects is a boolean that specifies whether upstream redirects
	// are followed (10 depth) or not.
	FollowRedirects bool
	// Request timeout is a timeout for fetching upstream data.
	RequestTimeout  time.Duration
}

// Interface for Proxy to use for stats/metrics
// This must be goroutine safe, as AddBytes and AddServed will be called from
// many goroutines.
type ProxyMetrics interface {
	AddBytes(bc int64)
	AddServed()
	//GetStats() (b uint64, c uint64)
}

// A Proxy is a Camo like HTTP proxy, that provides content type
// restrictions as well as regex host allow and deny list support
type Proxy struct {
	client    *http.Client
	hmacKey   []byte
	allowList []*regexp.Regexp
	denyList  []*regexp.Regexp
	maxSize   int64
	metrics   ProxyMetrics
}

// ServerHTTP handles the client request, validates the request is validly
// HMAC signed, filters based on the Allow/Deny list, and then proxies
// valid requests to the desired endpoint. Responses are filtered for
// proper image content types.
func (p *Proxy) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	Logger.Debugln("Request:", req.URL)
	if p.metrics != nil {
		go p.metrics.AddServed()
	}

	w.Header().Set("Server", ServerNameVer)

	vars := mux.Vars(req)
	surl, ok := DecodeUrl(&p.hmacKey, vars["sigHash"], vars["encodedUrl"])
	if !ok {
		http.Error(w, "Bad Signature", http.StatusForbidden)
		return
	}
	Logger.Debugln("URL:", surl)

	u, err := url.Parse(surl)
	if err != nil {
		Logger.Debugln(err)
		http.Error(w, "Bad url", http.StatusBadRequest)
		return
	}

	if u.Host == "" {
		http.Error(w, "Bad url", http.StatusNotFound)
		return
	}

	// if allowList is set, require match
	matchFound := true
	if len(p.allowList) > 0 {
		matchFound = false
		for _, rgx := range p.allowList {
			if rgx.MatchString(u.Host) {
				matchFound = true
			}
		}
	}
	if !matchFound {
		http.Error(w, "Allowlist host failure", http.StatusNotFound)
		return
	}

	// filter out denyList urls based on regexes. Do this second
	// as denyList takes precedence
	for _, rgx := range p.denyList {
		if rgx.MatchString(u.Host) {
			http.Error(w, "Denylist host failure", http.StatusNotFound)
			return
		}
	}

	nreq, err := http.NewRequest("GET", surl, nil)
	if err != nil {
		Logger.Debugln("Could not create NewRequest", err)
		http.Error(w, "Error Fetching Resource", http.StatusBadGateway)
		return
	}

	// filter headers
	p.copyHeader(&nreq.Header, &req.Header, &ValidReqHeaders)
	if req.Header.Get("X-Forwarded-For") == "" {
		host, _, err := net.SplitHostPort(req.RemoteAddr)
		if err == nil && !addr1918match.MatchString(host) {
			nreq.Header.Add("X-Forwarded-For", host)
		}
	}
	nreq.Header.Add("connection", "close")
	nreq.Header.Add("user-agent", ServerNameVer)

	resp, err := p.client.Do(nreq)
	if err != nil {
		Logger.Debugln("Could not connect to endpoint", err)
		if strings.Contains(err.Error(), "timeout") {
			http.Error(w, "Error Fetching Resource", http.StatusBadGateway)
		} else {
			http.Error(w, "Error Fetching Resource", http.StatusNotFound)
		}
		return
	}
	defer resp.Body.Close()

	// check for too large a response
	if resp.ContentLength > p.maxSize {
		Logger.Debugln("Content length exceeded", surl)
		http.Error(w, "Content length exceeded", http.StatusNotFound)
		return
	}

	switch resp.StatusCode {
	case 200:
		// check content type
		ct, ok := resp.Header[http.CanonicalHeaderKey("content-type")]
		if !ok || ct[0][:6] != "image/" {
			Logger.Debugln("Non-Image content-type returned", u)
			http.Error(w, "Non-Image content-type returned",
				http.StatusBadRequest)
			return
		}
	case 300:
		Logger.Debugln("Multiple choices not supported")
		http.Error(w, "Multiple choices not supported", http.StatusNotFound)
		return
	case 301, 302, 303, 307:
		// if we get a redirect here, we either disabled following,
		// or followed until max depth and still got one (redirect loop)
		http.Error(w, "Not Found", http.StatusNotFound)
		return
	case 304:
		h := w.Header()
		p.copyHeader(&h, &resp.Header, &ValidRespHeaders)
		h.Set("X-Content-Type-Options", "nosniff")
		w.WriteHeader(304)
		return
	case 404:
		http.Error(w, "Not Found", http.StatusNotFound)
		return
	case 500, 502, 503, 504:
		// upstream errors should probably just 502. client can try later.
		http.Error(w, "Error Fetching Resource", http.StatusBadGateway)
		return
	default:
		http.Error(w, "Not Found", http.StatusNotFound)
		return
	}

	h := w.Header()
	p.copyHeader(&h, &resp.Header, &ValidRespHeaders)
	h.Set("X-Content-Type-Options", "nosniff")
	w.WriteHeader(resp.StatusCode)

	// since this uses io.Copy from the respBody, it is streaming
	// from the request to the response. This means it will nearly
	// always end up with a chunked response.
	// Change to the following to send whole body at once, and
	// read whole body at once too:
	//    body, err := ioutil.ReadAll(resp.Body)
	//    if err != nil {
	//        Logger.Println("Error writing response:", err)
	//    }
	//    w.Write(body)
	// Might use quite a bit of memory though. Untested.
	bW, err := io.Copy(w, resp.Body)
	if err != nil {
		Logger.Println("Error writing response:", err)
		return
	}

	if p.metrics != nil {
		go p.metrics.AddBytes(bW)
	}
	Logger.Debugln(req, resp.StatusCode)
}

// copy headers from src into dst
// empty filter map will result in no filtering being done
func (p *Proxy) copyHeader(dst, src *http.Header, filter *map[string]bool) {
	f := *filter
	filtering := false
	if len(f) > 0 {
		filtering = true
	}

	for k, vv := range *src {
		if x, ok := f[k]; filtering && (!ok || !x) {
			continue
		}
		for _, v := range vv {
			dst.Add(k, v)
		}
	}
}

// sets a proxy metrics (ProxyMetrics interface) for the proxy
func (p *Proxy) SetMetricsCollector(pm ProxyMetrics) {
	p.metrics = pm
}

// Returns a new Proxy. An error is returned if there was a failure
// to parse the regex from the passed Config.
func New(pc Config) (*Proxy, error) {
	tr := &http.Transport{
		Dial: func(netw, addr string) (net.Conn, error) {
			c, err := net.DialTimeout(netw, addr, pc.RequestTimeout)
			if err != nil {
				return nil, err
			}
			// also set time limit on reading
			c.SetDeadline(time.Now().Add(pc.RequestTimeout))
			return c, nil
		}}

	// spawn an idle conn trimmer
	go func() {
		time.Sleep(5 * time.Minute)
		tr.CloseIdleConnections()
	}()

	// build/compile regex
	client := &http.Client{Transport: tr}
	if !pc.FollowRedirects {
		client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
			return errors.New("Not following redirect")
		}
	}

	allow := make([]*regexp.Regexp, 0)
	deny := make([]*regexp.Regexp, 0)

	var c *regexp.Regexp
	var err error
	for _, v := range pc.DenyList {
		c, err = regexp.Compile(v)
		if err != nil {
			return nil, err
		}
		deny = append(deny, c)
	}
	for _, v := range pc.AllowList {
		c, err = regexp.Compile(v)
		if err != nil {
			return nil, err
		}
		allow = append(allow, c)
	}

	return &Proxy{
		client:    client,
		hmacKey:   []byte(pc.HmacKey),
		allowList: allow,
		denyList:  deny,
		maxSize:   pc.MaxSize}, nil
}
<commit_msg>reformat some comments<commit_after>// Package camoproxy provides an HTTP proxy server with content type
// restrictions as well as regex host allow and deny list support.
package camoproxy

import (
	"code.google.com/p/gorilla/mux"
	"errors"
	"io"
	"net"
	"net/http"
	"net/url"
	"regexp"
	"strings"
	"time"
)

// Config holds configuration data used when creating a Proxy with New.
type Config struct {
	// HmacKey is a string to be used as the hmac key
	HmacKey         string
	// AllowList is a list of string represenstations of regex (not compiled
	// regex) that are used as a whitelist filter. If an AllowList is present,
	// then anything not matching is dropped. If no AllowList is present,
	// no Allow filtering is done.
	AllowList       []string
	// DenyList is a list of string represenstations of regex (not compiled
	// regex). The deny filter check occurs after the allow filter check
	// (if any).
	DenyList        []string
	// MaxSize is the maximum valid image size response (in bytes).
	MaxSize         int64
	// FollowRedirects is a boolean that specifies whether upstream redirects
	// are followed (10 depth) or not.
	FollowRedirects bool
	// Request timeout is a timeout for fetching upstream data.
	RequestTimeout  time.Duration
}

// Interface for Proxy to use for stats/metrics.
// This must be goroutine safe, as AddBytes and AddServed will be called from
// many goroutines.
type ProxyMetrics interface {
	AddBytes(bc int64)
	AddServed()
	//GetStats() (b uint64, c uint64)
}

// A Proxy is a Camo like HTTP proxy, that provides content type
// restrictions as well as regex host allow and deny list support.
type Proxy struct {
	client    *http.Client
	hmacKey   []byte
	allowList []*regexp.Regexp
	denyList  []*regexp.Regexp
	maxSize   int64
	metrics   ProxyMetrics
}

// ServerHTTP handles the client request, validates the request is validly
// HMAC signed, filters based on the Allow/Deny list, and then proxies
// valid requests to the desired endpoint. Responses are filtered for
// proper image content types.
func (p *Proxy) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	Logger.Debugln("Request:", req.URL)
	if p.metrics != nil {
		go p.metrics.AddServed()
	}

	w.Header().Set("Server", ServerNameVer)

	vars := mux.Vars(req)
	surl, ok := DecodeUrl(&p.hmacKey, vars["sigHash"], vars["encodedUrl"])
	if !ok {
		http.Error(w, "Bad Signature", http.StatusForbidden)
		return
	}
	Logger.Debugln("URL:", surl)

	u, err := url.Parse(surl)
	if err != nil {
		Logger.Debugln(err)
		http.Error(w, "Bad url", http.StatusBadRequest)
		return
	}

	if u.Host == "" {
		http.Error(w, "Bad url", http.StatusNotFound)
		return
	}

	// if allowList is set, require match
	matchFound := true
	if len(p.allowList) > 0 {
		matchFound = false
		for _, rgx := range p.allowList {
			if rgx.MatchString(u.Host) {
				matchFound = true
			}
		}
	}
	if !matchFound {
		http.Error(w, "Allowlist host failure", http.StatusNotFound)
		return
	}

	// filter out denyList urls based on regexes. Do this second
	// as denyList takes precedence
	for _, rgx := range p.denyList {
		if rgx.MatchString(u.Host) {
			http.Error(w, "Denylist host failure", http.StatusNotFound)
			return
		}
	}

	nreq, err := http.NewRequest("GET", surl, nil)
	if err != nil {
		Logger.Debugln("Could not create NewRequest", err)
		http.Error(w, "Error Fetching Resource", http.StatusBadGateway)
		return
	}

	// filter headers
	p.copyHeader(&nreq.Header, &req.Header, &ValidReqHeaders)
	if req.Header.Get("X-Forwarded-For") == "" {
		host, _, err := net.SplitHostPort(req.RemoteAddr)
		if err == nil && !addr1918match.MatchString(host) {
			nreq.Header.Add("X-Forwarded-For", host)
		}
	}
	nreq.Header.Add("connection", "close")
	nreq.Header.Add("user-agent", ServerNameVer)

	resp, err := p.client.Do(nreq)
	if err != nil {
		Logger.Debugln("Could not connect to endpoint", err)
		if strings.Contains(err.Error(), "timeout") {
			http.Error(w, "Error Fetching Resource", http.StatusBadGateway)
		} else {
			http.Error(w, "Error Fetching Resource", http.StatusNotFound)
		}
		return
	}
	defer resp.Body.Close()

	// check for too large a response
	if resp.ContentLength > p.maxSize {
		Logger.Debugln("Content length exceeded", surl)
		http.Error(w, "Content length exceeded", http.StatusNotFound)
		return
	}

	switch resp.StatusCode {
	case 200:
		// check content type
		ct, ok := resp.Header[http.CanonicalHeaderKey("content-type")]
		if !ok || ct[0][:6] != "image/" {
			Logger.Debugln("Non-Image content-type returned", u)
			http.Error(w, "Non-Image content-type returned",
				http.StatusBadRequest)
			return
		}
	case 300:
		Logger.Debugln("Multiple choices not supported")
		http.Error(w, "Multiple choices not supported", http.StatusNotFound)
		return
	case 301, 302, 303, 307:
		// if we get a redirect here, we either disabled following,
		// or followed until max depth and still got one (redirect loop)
		http.Error(w, "Not Found", http.StatusNotFound)
		return
	case 304:
		h := w.Header()
		p.copyHeader(&h, &resp.Header, &ValidRespHeaders)
		h.Set("X-Content-Type-Options", "nosniff")
		w.WriteHeader(304)
		return
	case 404:
		http.Error(w, "Not Found", http.StatusNotFound)
		return
	case 500, 502, 503, 504:
		// upstream errors should probably just 502. client can try later.
		http.Error(w, "Error Fetching Resource", http.StatusBadGateway)
		return
	default:
		http.Error(w, "Not Found", http.StatusNotFound)
		return
	}

	h := w.Header()
	p.copyHeader(&h, &resp.Header, &ValidRespHeaders)
	h.Set("X-Content-Type-Options", "nosniff")
	w.WriteHeader(resp.StatusCode)

	// since this uses io.Copy from the respBody, it is streaming
	// from the request to the response. This means it will nearly
	// always end up with a chunked response.
	// Change to the following to send whole body at once, and
	// read whole body at once too:
	//    body, err := ioutil.ReadAll(resp.Body)
	//    if err != nil {
	//        Logger.Println("Error writing response:", err)
	//    }
	//    w.Write(body)
	// Might use quite a bit of memory though. Untested.
	bW, err := io.Copy(w, resp.Body)
	if err != nil {
		Logger.Println("Error writing response:", err)
		return
	}

	if p.metrics != nil {
		go p.metrics.AddBytes(bW)
	}
	Logger.Debugln(req, resp.StatusCode)
}

// copy headers from src into dst
// empty filter map will result in no filtering being done
func (p *Proxy) copyHeader(dst, src *http.Header, filter *map[string]bool) {
	f := *filter
	filtering := false
	if len(f) > 0 {
		filtering = true
	}

	for k, vv := range *src {
		if x, ok := f[k]; filtering && (!ok || !x) {
			continue
		}
		for _, v := range vv {
			dst.Add(k, v)
		}
	}
}

// sets a proxy metrics (ProxyMetrics interface) for the proxy
func (p *Proxy) SetMetricsCollector(pm ProxyMetrics) {
	p.metrics = pm
}

// Returns a new Proxy. An error is returned if there was a failure
// to parse the regex from the passed Config.
func New(pc Config) (*Proxy, error) {
	tr := &http.Transport{
		Dial: func(netw, addr string) (net.Conn, error) {
			c, err := net.DialTimeout(netw, addr, pc.RequestTimeout)
			if err != nil {
				return nil, err
			}
			// also set time limit on reading
			c.SetDeadline(time.Now().Add(pc.RequestTimeout))
			return c, nil
		}}

	// spawn an idle conn trimmer
	go func() {
		time.Sleep(5 * time.Minute)
		tr.CloseIdleConnections()
	}()

	// build/compile regex
	client := &http.Client{Transport: tr}
	if !pc.FollowRedirects {
		client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
			return errors.New("Not following redirect")
		}
	}

	allow := make([]*regexp.Regexp, 0)
	deny := make([]*regexp.Regexp, 0)

	var c *regexp.Regexp
	var err error
	for _, v := range pc.DenyList {
		c, err = regexp.Compile(v)
		if err != nil {
			return nil, err
		}
		deny = append(deny, c)
	}
	for _, v := range pc.AllowList {
		c, err = regexp.Compile(v)
		if err != nil {
			return nil, err
		}
		allow = append(allow, c)
	}

	return &Proxy{
		client:    client,
		hmacKey:   []byte(pc.HmacKey),
		allowList: allow,
		denyList:  deny,
		maxSize:   pc.MaxSize}, nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"flag"
	"fmt"
	"container/list"
	"os"
	es "github.com/jensrantil/rewindd/eventstore"
	zmq "github.com/alecthomas/gozmq"
)

var (
	eventStorePath = flag.String("datadir", "data",
	"directory path where incoming events will be stored. Created"+
	" if non-existent.")
	commandSocketZPath = flag.String("commandsocket",
	"tcp://127.0.0.1:9002", "Command socket. Handles new eventsand"+
	" queries.")
	eventPublishZPath = flag.String("evpubsocket",
	"tcp:127.0.0.1:9003", "ZeroMQ event publishing socket.")
)

// Runs the server that distributes requests to workers.
// Panics on error since it is an essential piece of code required to
// run the application correctly.
func runServer(estore *es.EventStore) {
	context, err := zmq.NewContext()
	if err != nil {
		panic(err)
	}
	defer context.Close()

	commandsock, err := context.NewSocket(zmq.ROUTER)
	if err != nil {
		panic(err)
	}
	defer commandsock.Close()
	err = commandsock.Bind(*commandSocketZPath)
	if err != nil {
		panic(err)
	}

	evpubsock, err := context.NewSocket(zmq.PUB)
	if err != nil {
		panic(err)
	}
	defer evpubsock.Close()
	if binderr := evpubsock.Bind(*commandSocketZPath); binderr != nil {
		panic(binderr)
	}

	loopServer(estore, evpubsock, commandsock)
}

// The result of an asynchronous zmq.Poll call.
type zmqPollResult struct {
	nbrOfChanges int
	err error
}

// Polls a bunch of ZeroMQ sockets and notifies the result through a
// channel. This makes it possible to combine ZeroMQ polling with Go's
// own built-in channels.
func asyncPoll(notifier chan zmqPollResult, items zmq.PollItems) {
	a, b := zmq.Poll(items, -1)
	notifier <- zmqPollResult{a, b}
}

// The core ZeroMQ messaging loop. Handles requests and responses
// asynchronously using the router socket. Every request is delegated to
// a goroutine for maximum concurrency.
//
// `gozmq` does currently not support copy-free messages/frames. This means that
// every message passing through this function needs to be copied
// in-memory. If this becomes a bottleneck in the future, multiple
// router sockets can be hooked to this final router to scale message
// copying to multiple cores.
func loopServer(estore *es.EventStore, evpubsock, frontend zmq.Socket) {
	toPoll := zmq.PollItems{
		zmq.PollItem{Socket: frontend, zmq.Events: zmq.POLLIN},
	}

	pubchan := make(chan es.StoredEvent)
	estore.RegisterPublishedEventsChannel(pubchan)
	go publishAllSavedEvents(pubchan, evpubsock)

	pollchan := make(chan zmqPollResult)
	respchan := make(chan [][]byte)
	go asyncPoll(pollchan, toPoll)
	for {
		select {
		case <- pollchan:
			if toPoll[0].REvents&zmq.POLLIN != 0 {
				msg, _ := toPoll[0].Socket.RecvMultipart(0)
				go handleRequest(respchan, estore, msg)
			}
			go asyncPoll(pollchan, toPoll)
		case frames := <-respchan:
			if err := frontend.SendMultipart(frames, 0); err != nil {
				fmt.Println(err)
			}
		}
	}
}

// Publishes stored events to event listeners.
//
// Pops previously stored messages off a channel and published them to a
// ZeroMQ socket.
func publishAllSavedEvents(toPublish chan es.StoredEvent, evpub zmq.Socket) {
	msg := make([][]byte, 3)
	for {
		event := <-toPublish

		msg[0] = event.Stream
		msg[1] = event.Id
		msg[2] = event.Data

		if err := evpub.SendMultipart(msg, 0); err != nil {
			// TODO: Use logger
			fmt.Println(err)
		}
	}
}

// Handles a single ZeroMQ RES/REQ loop.
//
// The full request message stored in `msg` and the full ZeroMQ response
// is pushed to `respchan`. The function does not return any error
// because it is expected to be called asynchronously as a goroutine.
func handleRequest(respchan chan [][]byte, estore *es.EventStore, msg [][]byte) {

	// TODO: Rename to 'framelist'
	// TODO: Avoid this creation and move it into copyList.
	parts := list.New()
	for _, msgpart := range msg {
		parts.PushBack(msgpart)
	}

	// TODO: Possibly wrap ZeroMQ router frames into a Type before
	// calling this method. That would yield a nicer API without
	// nitty gritty ZeroMQ details.
	resptemplate := list.New()
	emptyFrame := []byte("")
	for true {
		resptemplate.PushBack(parts.Remove(parts.Front()))

		if bytes.Equal(parts.Front().Value.([]byte), emptyFrame) {
			break
		}
	}

	if parts.Len() == 0 {
		errstr := "Incoming command was empty. Ignoring it."
		// TODO: Migrate to logging system
		fmt.Println(errstr)
		response := copyList(resptemplate)
		response.PushBack([]byte("ERROR " + errstr))
		respchan <- listToFrames(response)
		return
	}

	command := string(parts.Front().Value.([]byte))
	switch command {
	case "PUBLISH":
		parts.Remove(parts.Front())
		if parts.Len() != 2 {
			errstr := "Wrong number of frames for PUBLISH."
			// TODO: Migrate to logging system
			fmt.Println(errstr)
			response := copyList(resptemplate)
			response.PushBack([]byte("ERROR " + errstr))
			respchan <- listToFrames(response)
		} else {
			estream := parts.Remove(parts.Front())
			data := parts.Remove(parts.Front())
			newevent := es.UnstoredEvent{
				Stream: estream.([]byte),
				Data: data.([]byte),
			}
			estore.Add(newevent)
			// TODO: Send response
		}
	case "QUERY":
		parts.Remove(parts.Front())
		if parts.Len() != 3 {
			errstr := "Wrong number of frames for QUERY."
			// TODO: Migrate to logging system
			fmt.Println(errstr)
			response := copyList(resptemplate)
			response.PushBack([]byte("ERROR " + errstr))
			respchan <- listToFrames(response)
		} else {
			estreamprefix := parts.Remove(parts.Front())
			fromid := parts.Remove(parts.Front())
			toid := parts.Remove(parts.Front())

			events := make(chan es.StoredEvent)
			req := es.QueryRequest{
				StreamPrefix: estreamprefix.([]byte),
				FromId: fromid.([]byte),
				ToId: toid.([]byte),
			}
			go estore.Query(req, events)
			for eventdata := range(events) {
				response := copyList(resptemplate)
				response.PushBack(eventdata.Stream)
				response.PushBack(eventdata.Id)
				response.PushBack(eventdata.Data)
				// TODO: Prepend the router
				// frames before!
				respchan <- listToFrames(response)
			}
			response := copyList(resptemplate)
			response.PushBack([]byte("END"))
			respchan <- listToFrames(response)
		}
	default:
		// TODO: Move these error strings out as constants of
		//       this package.

		// TODO: Move the chunk of code below into a separate
		// function and reuse for similar piece of code above.
		errstr := "Unknown request type."
		// TODO: Migrate to logging system
		fmt.Println(errstr)
		response := copyList(resptemplate)
		response.PushBack([]byte("ERROR " + errstr))
		respchan <- listToFrames(response)
	}
}

// Convert a doubly linked list of message frames to a slice of message
// frames.
func listToFrames(l *list.List) [][]byte {
	frames := make([][]byte, l.Len())
	i := 0
	for e := l.Front(); e != nil; e = e.Next() {
		frames[i] = e.Value.([]byte)
	}
	return frames
}

// Helper function for copying a doubly linked list.
func copyList(l *list.List) *list.List {
	replica := list.New()
	replica.PushBackList(l)
	return replica
}

// Main method. Will panic if things are so bad that the application
// will not start.
func main() {
	flag.Parse()

	desc, err := es.NewFileSystemDescriptor(*eventStorePath)
	if err != nil {
		fmt.Fprintln(os.Stderr, "could not create event store Desc")
		panic(err)
	}
	estore, err := es.NewEventStore(desc)
	if err != nil {
		fmt.Fprintln(os.Stderr, "could not create event store")
		panic(err)
	}
	defer estore.Close()

	runServer(estore)

	// TODO: Handle SIGINT correctly and smoothly.
}
<commit_msg>Correcting typo in ZeroMQ URL<commit_after>package main

import (
	"bytes"
	"flag"
	"fmt"
	"container/list"
	"os"
	es "github.com/jensrantil/rewindd/eventstore"
	zmq "github.com/alecthomas/gozmq"
)

var (
	eventStorePath = flag.String("datadir", "data",
	"directory path where incoming events will be stored. Created"+
	" if non-existent.")
	commandSocketZPath = flag.String("commandsocket",
	"tcp://127.0.0.1:9002", "Command socket. Handles new eventsand"+
	" queries.")
	eventPublishZPath = flag.String("evpubsocket",
	"tcp://127.0.0.1:9003", "ZeroMQ event publishing socket.")
)

// Runs the server that distributes requests to workers.
// Panics on error since it is an essential piece of code required to
// run the application correctly.
func runServer(estore *es.EventStore) {
	context, err := zmq.NewContext()
	if err != nil {
		panic(err)
	}
	defer context.Close()

	commandsock, err := context.NewSocket(zmq.ROUTER)
	if err != nil {
		panic(err)
	}
	defer commandsock.Close()
	err = commandsock.Bind(*commandSocketZPath)
	if err != nil {
		panic(err)
	}

	evpubsock, err := context.NewSocket(zmq.PUB)
	if err != nil {
		panic(err)
	}
	defer evpubsock.Close()
	if binderr := evpubsock.Bind(*commandSocketZPath); binderr != nil {
		panic(binderr)
	}

	loopServer(estore, evpubsock, commandsock)
}

// The result of an asynchronous zmq.Poll call.
type zmqPollResult struct {
	nbrOfChanges int
	err error
}

// Polls a bunch of ZeroMQ sockets and notifies the result through a
// channel. This makes it possible to combine ZeroMQ polling with Go's
// own built-in channels.
func asyncPoll(notifier chan zmqPollResult, items zmq.PollItems) {
	a, b := zmq.Poll(items, -1)
	notifier <- zmqPollResult{a, b}
}

// The core ZeroMQ messaging loop. Handles requests and responses
// asynchronously using the router socket. Every request is delegated to
// a goroutine for maximum concurrency.
//
// `gozmq` does currently not support copy-free messages/frames. This means that
// every message passing through this function needs to be copied
// in-memory. If this becomes a bottleneck in the future, multiple
// router sockets can be hooked to this final router to scale message
// copying to multiple cores.
func loopServer(estore *es.EventStore, evpubsock, frontend zmq.Socket) {
	toPoll := zmq.PollItems{
		zmq.PollItem{Socket: frontend, zmq.Events: zmq.POLLIN},
	}

	pubchan := make(chan es.StoredEvent)
	estore.RegisterPublishedEventsChannel(pubchan)
	go publishAllSavedEvents(pubchan, evpubsock)

	pollchan := make(chan zmqPollResult)
	respchan := make(chan [][]byte)
	go asyncPoll(pollchan, toPoll)
	for {
		select {
		case <- pollchan:
			if toPoll[0].REvents&zmq.POLLIN != 0 {
				msg, _ := toPoll[0].Socket.RecvMultipart(0)
				go handleRequest(respchan, estore, msg)
			}
			go asyncPoll(pollchan, toPoll)
		case frames := <-respchan:
			if err := frontend.SendMultipart(frames, 0); err != nil {
				fmt.Println(err)
			}
		}
	}
}

// Publishes stored events to event listeners.
//
// Pops previously stored messages off a channel and published them to a
// ZeroMQ socket.
func publishAllSavedEvents(toPublish chan es.StoredEvent, evpub zmq.Socket) {
	msg := make([][]byte, 3)
	for {
		event := <-toPublish

		msg[0] = event.Stream
		msg[1] = event.Id
		msg[2] = event.Data

		if err := evpub.SendMultipart(msg, 0); err != nil {
			// TODO: Use logger
			fmt.Println(err)
		}
	}
}

// Handles a single ZeroMQ RES/REQ loop.
//
// The full request message stored in `msg` and the full ZeroMQ response
// is pushed to `respchan`. The function does not return any error
// because it is expected to be called asynchronously as a goroutine.
func handleRequest(respchan chan [][]byte, estore *es.EventStore, msg [][]byte) {

	// TODO: Rename to 'framelist'
	// TODO: Avoid this creation and move it into copyList.
	parts := list.New()
	for _, msgpart := range msg {
		parts.PushBack(msgpart)
	}

	// TODO: Possibly wrap ZeroMQ router frames into a Type before
	// calling this method. That would yield a nicer API without
	// nitty gritty ZeroMQ details.
	resptemplate := list.New()
	emptyFrame := []byte("")
	for true {
		resptemplate.PushBack(parts.Remove(parts.Front()))

		if bytes.Equal(parts.Front().Value.([]byte), emptyFrame) {
			break
		}
	}

	if parts.Len() == 0 {
		errstr := "Incoming command was empty. Ignoring it."
		// TODO: Migrate to logging system
		fmt.Println(errstr)
		response := copyList(resptemplate)
		response.PushBack([]byte("ERROR " + errstr))
		respchan <- listToFrames(response)
		return
	}

	command := string(parts.Front().Value.([]byte))
	switch command {
	case "PUBLISH":
		parts.Remove(parts.Front())
		if parts.Len() != 2 {
			errstr := "Wrong number of frames for PUBLISH."
			// TODO: Migrate to logging system
			fmt.Println(errstr)
			response := copyList(resptemplate)
			response.PushBack([]byte("ERROR " + errstr))
			respchan <- listToFrames(response)
		} else {
			estream := parts.Remove(parts.Front())
			data := parts.Remove(parts.Front())
			newevent := es.UnstoredEvent{
				Stream: estream.([]byte),
				Data: data.([]byte),
			}
			estore.Add(newevent)
			// TODO: Send response
		}
	case "QUERY":
		parts.Remove(parts.Front())
		if parts.Len() != 3 {
			errstr := "Wrong number of frames for QUERY."
			// TODO: Migrate to logging system
			fmt.Println(errstr)
			response := copyList(resptemplate)
			response.PushBack([]byte("ERROR " + errstr))
			respchan <- listToFrames(response)
		} else {
			estreamprefix := parts.Remove(parts.Front())
			fromid := parts.Remove(parts.Front())
			toid := parts.Remove(parts.Front())

			events := make(chan es.StoredEvent)
			req := es.QueryRequest{
				StreamPrefix: estreamprefix.([]byte),
				FromId: fromid.([]byte),
				ToId: toid.([]byte),
			}
			go estore.Query(req, events)
			for eventdata := range(events) {
				response := copyList(resptemplate)
				response.PushBack(eventdata.Stream)
				response.PushBack(eventdata.Id)
				response.PushBack(eventdata.Data)
				// TODO: Prepend the router
				// frames before!
				respchan <- listToFrames(response)
			}
			response := copyList(resptemplate)
			response.PushBack([]byte("END"))
			respchan <- listToFrames(response)
		}
	default:
		// TODO: Move these error strings out as constants of
		//       this package.

		// TODO: Move the chunk of code below into a separate
		// function and reuse for similar piece of code above.
		errstr := "Unknown request type."
		// TODO: Migrate to logging system
		fmt.Println(errstr)
		response := copyList(resptemplate)
		response.PushBack([]byte("ERROR " + errstr))
		respchan <- listToFrames(response)
	}
}

// Convert a doubly linked list of message frames to a slice of message
// frames.
func listToFrames(l *list.List) [][]byte {
	frames := make([][]byte, l.Len())
	i := 0
	for e := l.Front(); e != nil; e = e.Next() {
		frames[i] = e.Value.([]byte)
	}
	return frames
}

// Helper function for copying a doubly linked list.
func copyList(l *list.List) *list.List {
	replica := list.New()
	replica.PushBackList(l)
	return replica
}

// Main method. Will panic if things are so bad that the application
// will not start.
func main() {
	flag.Parse()

	desc, err := es.NewFileSystemDescriptor(*eventStorePath)
	if err != nil {
		fmt.Fprintln(os.Stderr, "could not create event store Desc")
		panic(err)
	}
	estore, err := es.NewEventStore(desc)
	if err != nil {
		fmt.Fprintln(os.Stderr, "could not create event store")
		panic(err)
	}
	defer estore.Close()

	runServer(estore)

	// TODO: Handle SIGINT correctly and smoothly.
}
<|endoftext|>


<commit_before>// Package encodeio help read from/write to config file
package encodeio

import (
	"os"

	"github.com/cosiner/gohper/encoding"
	"github.com/cosiner/gohper/os2/file"
)

func Read(fname string, v interface{}, codec encoding.Codec) error {
	return file.Read(fname, func(fd *os.File) error {
		return codec.Decode(fd, v)
	})
}

func ReadJSON(fname string, v interface{}) error {
	return Read(fname, v, encoding.JSON{})
}

func Write(fname string, v interface{}, codec encoding.Codec) error {
	return file.Write(fname, func(fd *os.File) error {
		return codec.Encode(fd, v)
	})
}

func Trunc(fname string, v interface{}, codec encoding.Codec) error {
	return file.Trunc(fname, func(fd *os.File) error {
		return codec.Encode(fd, v)
	})
}
<commit_msg>utils: add ReadJSONWithComment<commit_after>// Package encodeio help read from/write to config file
package encodeio

import (
	"os"

	"bufio"
	"bytes"
	"io"

	"github.com/cosiner/gohper/encoding"
	"github.com/cosiner/gohper/os2/file"
)

var (
	commentPrefix = []byte("//")
)

func Read(fname string, v interface{}, codec encoding.Codec) error {
	return file.Read(fname, func(fd *os.File) error {
		return codec.Decode(fd, v)
	})
}

func ReadJSON(fname string, v interface{}) error {
	return Read(fname, v, encoding.JSON{})
}

func ReadJSONWithComment(fname string, v interface{}) error {
	return file.Read(fname, func(fd *os.File) error {
		buf := bytes.NewBuffer(make([]byte, 0, 1024))
		br := bufio.NewReader(fd)

		for {
			line, _, err := br.ReadLine()
			if err == io.EOF {
				if len(line) == 0 {
					break
				}
			} else if err != nil {
				return err
			}

			if !bytes.HasPrefix(bytes.TrimSpace(line), commentPrefix) {
				buf.Write(line)
			}
		}

		return encoding.JSON{}.Unmarshal(buf.Bytes(), v)
	})
}

func Write(fname string, v interface{}, codec encoding.Codec) error {
	return file.Write(fname, func(fd *os.File) error {
		return codec.Encode(fd, v)
	})
}

func Trunc(fname string, v interface{}, codec encoding.Codec) error {
	return file.Trunc(fname, func(fd *os.File) error {
		return codec.Encode(fd, v)
	})
}
<|endoftext|>


<commit_before>package expr

import (
	"fmt"

	"github.com/grafana/metrictank/api/models"
	"github.com/grafana/metrictank/batch"
	"github.com/grafana/metrictank/consolidation"
	"github.com/grafana/metrictank/util"
	"github.com/raintank/dur"
	"gopkg.in/raintank/schema.v1"
)

type FuncSummarize struct {
	in             GraphiteFunc
	intervalString string
	fn             string
	alignToFrom    bool
}

func NewSummarize() GraphiteFunc {
	return &FuncSummarize{fn: "sum", alignToFrom: false}
}

func (s *FuncSummarize) Signature() ([]Arg, []Arg) {
	return []Arg{
		ArgSeriesList{val: &s.in},
		ArgString{key: "interval", val: &s.intervalString, validator: []Validator{IsIntervalString}},
		ArgString{key: "func", opt: true, val: &s.fn, validator: []Validator{IsConsolFunc}},
		ArgBool{key: "alignToFrom", opt: true, val: &s.alignToFrom},
	}, []Arg{ArgSeriesList{}}
}

func (s *FuncSummarize) Context(context Context) Context {
	context.consol = 0
	return context
}

func (s *FuncSummarize) Exec(cache map[Req][]models.Series) ([]models.Series, error) {
	series, err := s.in.Exec(cache)
	if err != nil {
		return nil, err
	}

	interval, _ := dur.ParseDuration(s.intervalString)
	aggFunc := consolidation.GetAggFunc(consolidation.FromConsolidateBy(s.fn))

	var alignToFromTarget string
	if s.alignToFrom {
		alignToFromTarget = ", true"
	}
	newName := func(oldName string) string {
		return fmt.Sprintf("summarize(%s, \"%s\", \"%s\"%s)", oldName, s.intervalString, s.fn, alignToFromTarget)
	}

	var outputs []models.Series
	for _, serie := range series {
		var newStart, newEnd uint32 = serie.QueryFrom, serie.QueryTo
		if len(serie.Datapoints) > 0 {
			newStart = serie.Datapoints[0].Ts
			newEnd = serie.Datapoints[len(serie.Datapoints)-1].Ts + serie.Interval
		}
		if !s.alignToFrom {
			newStart = newStart - (newStart % interval)
			newEnd = newEnd - (newEnd % interval) + interval
		}

		out := summarizeValues(serie, aggFunc, interval, newStart, newEnd)

		output := models.Series{
			Target:     newName(serie.Target),
			QueryPatt:  newName(serie.QueryPatt),
			Tags:       serie.Tags,
			Datapoints: out,
			Interval:   interval,
		}
		outputs = append(outputs, output)
		cache[Req{}] = append(cache[Req{}], output)
	}
	return outputs, nil
}

func summarizeValues(serie models.Series, aggFunc batch.AggFunc, interval, start, end uint32) []schema.Point {
	out := pointSlicePool.Get().([]schema.Point)

	numPoints := int(util.Min(uint32(len(serie.Datapoints)), (start-end)/interval))

	for ts, i := start, 0; i < numPoints && ts < end; ts += interval {
		s := i
		for ; i < numPoints && serie.Datapoints[i].Ts < ts+interval; i++ {
			if serie.Datapoints[i].Ts <= ts {
				s = i
			}
		}

		out = append(out, schema.Point{Val: aggFunc(serie.Datapoints[s:i]), Ts: ts})
	}

	return out
}
<commit_msg>Remove key for intervalstring param<commit_after>package expr

import (
	"fmt"

	"github.com/grafana/metrictank/api/models"
	"github.com/grafana/metrictank/batch"
	"github.com/grafana/metrictank/consolidation"
	"github.com/grafana/metrictank/util"
	"github.com/raintank/dur"
	"gopkg.in/raintank/schema.v1"
)

type FuncSummarize struct {
	in             GraphiteFunc
	intervalString string
	fn             string
	alignToFrom    bool
}

func NewSummarize() GraphiteFunc {
	return &FuncSummarize{fn: "sum", alignToFrom: false}
}

func (s *FuncSummarize) Signature() ([]Arg, []Arg) {
	return []Arg{
		ArgSeriesList{val: &s.in},
		ArgString{val: &s.intervalString, validator: []Validator{IsIntervalString}},
		ArgString{key: "func", opt: true, val: &s.fn, validator: []Validator{IsConsolFunc}},
		ArgBool{key: "alignToFrom", opt: true, val: &s.alignToFrom},
	}, []Arg{ArgSeriesList{}}
}

func (s *FuncSummarize) Context(context Context) Context {
	context.consol = 0
	return context
}

func (s *FuncSummarize) Exec(cache map[Req][]models.Series) ([]models.Series, error) {
	series, err := s.in.Exec(cache)
	if err != nil {
		return nil, err
	}

	interval, _ := dur.ParseDuration(s.intervalString)
	aggFunc := consolidation.GetAggFunc(consolidation.FromConsolidateBy(s.fn))

	var alignToFromTarget string
	if s.alignToFrom {
		alignToFromTarget = ", true"
	}
	newName := func(oldName string) string {
		return fmt.Sprintf("summarize(%s, \"%s\", \"%s\"%s)", oldName, s.intervalString, s.fn, alignToFromTarget)
	}

	var outputs []models.Series
	for _, serie := range series {
		var newStart, newEnd uint32 = serie.QueryFrom, serie.QueryTo
		if len(serie.Datapoints) > 0 {
			newStart = serie.Datapoints[0].Ts
			newEnd = serie.Datapoints[len(serie.Datapoints)-1].Ts + serie.Interval
		}
		if !s.alignToFrom {
			newStart = newStart - (newStart % interval)
			newEnd = newEnd - (newEnd % interval) + interval
		}

		out := summarizeValues(serie, aggFunc, interval, newStart, newEnd)

		output := models.Series{
			Target:     newName(serie.Target),
			QueryPatt:  newName(serie.QueryPatt),
			Tags:       serie.Tags,
			Datapoints: out,
			Interval:   interval,
		}
		outputs = append(outputs, output)
		cache[Req{}] = append(cache[Req{}], output)
	}
	return outputs, nil
}

func summarizeValues(serie models.Series, aggFunc batch.AggFunc, interval, start, end uint32) []schema.Point {
	out := pointSlicePool.Get().([]schema.Point)

	numPoints := int(util.Min(uint32(len(serie.Datapoints)), (start-end)/interval))

	for ts, i := start, 0; i < numPoints && ts < end; ts += interval {
		s := i
		for ; i < numPoints && serie.Datapoints[i].Ts < ts+interval; i++ {
			if serie.Datapoints[i].Ts <= ts {
				s = i
			}
		}

		out = append(out, schema.Point{Val: aggFunc(serie.Datapoints[s:i]), Ts: ts})
	}

	return out
}
<|endoftext|>


<commit_before>// md5 supports MD5 hashes in various formats.
package md5

import (
	cryptomd5 "crypto/md5"
	"fmt"
	"math/big"

	"github.com/grokify/gotilla/type/stringsutil"
)

// Md5Base36Length is the length for a MD5 Base36 string
const (
	md5Base62Length int    = 22
	md5Base62Format string = `%022s`
	md5Base36Length int    = 25
	md5Base36Format string = `%025s`
	md5Base10Length int    = 39
	md5Base10Format string = `%039s`
)

// Md5Base10 returns a Base10 encoded MD5 hash of a string.
func Md5Base10(s string) string {
	i := new(big.Int)
	i.SetString(fmt.Sprintf("%x", cryptomd5.Sum([]byte(s))), 16)
	return fmt.Sprintf(md5Base10Format, i.String())
}

// Md5Base36 returns a Base36 encoded MD5 hash of a string.
func Md5Base36(s string) string {
	i := new(big.Int)
	i.SetString(fmt.Sprintf("%x", cryptomd5.Sum([]byte(s))), 16)
	return fmt.Sprintf(md5Base36Format, i.Text(36))
}

// Md5Base62 returns a Base62 encoded MD5 hash of a string.
// This uses the Golang alphabet [0-9a-zA-Z].
func Md5Base62(s string) string {
	i := new(big.Int)
	i.SetString(fmt.Sprintf("%x", cryptomd5.Sum([]byte(s))), 16)
	return fmt.Sprintf(md5Base62Format, i.Text(62))
}

// Md5Base62Upper returns a Base62 encoded MD5 hash of a string.
// Note Base62 encoding uses the GMP alphabet [0-9A-Za-z] instead
// of the Golang alphabet [0-9a-zA-Z] because the GMP alphabet
// may be more standard, e.g. used in GMP and follows ASCII
// table order.
func Md5Base62UpperFirst(s string) string {
	i := big.NewInt(0)
	i2, ok := i.SetString(fmt.Sprintf("%x", cryptomd5.Sum([]byte(s))), 16)
	if !ok {
		panic("E_CANNOT_CONVERT_HEX")
	}
	return fmt.Sprintf(md5Base62Format, stringsutil.ToOpposite(i2.Text(62)))
}
<commit_msg>streamline code<commit_after>// md5 supports MD5 hashes in various formats.
package md5

import (
	cryptomd5 "crypto/md5"
	"fmt"
	"math/big"

	"github.com/grokify/gotilla/type/stringsutil"
)

// Md5Base36Length is the length for a MD5 Base36 string
const (
	md5Base62Length int    = 22
	md5Base62Format string = `%022s`
	md5Base36Length int    = 25
	md5Base36Format string = `%025s`
	md5Base10Length int    = 39
	md5Base10Format string = `%039s`
)

// Md5Base10 returns a Base10 encoded MD5 hash of a string.
func Md5Base10(s string) string {
	i := new(big.Int)
	i.SetString(fmt.Sprintf("%x", cryptomd5.Sum([]byte(s))), 16)
	return fmt.Sprintf(md5Base10Format, i.String())
}

// Md5Base36 returns a Base36 encoded MD5 hash of a string.
func Md5Base36(s string) string {
	i := new(big.Int)
	i.SetString(fmt.Sprintf("%x", cryptomd5.Sum([]byte(s))), 16)
	return fmt.Sprintf(md5Base36Format, i.Text(36))
}

// Md5Base62 returns a Base62 encoded MD5 hash of a string.
// This uses the Golang alphabet [0-9a-zA-Z].
func Md5Base62(s string) string {
	i := new(big.Int)
	i.SetString(fmt.Sprintf("%x", cryptomd5.Sum([]byte(s))), 16)
	return fmt.Sprintf(md5Base62Format, i.Text(62))
}

// Md5Base62Upper returns a Base62 encoded MD5 hash of a string.
// Note Base62 encoding uses the GMP alphabet [0-9A-Za-z] instead
// of the Golang alphabet [0-9a-zA-Z] because the GMP alphabet
// may be more standard, e.g. used in GMP and follows ASCII
// table order.
func Md5Base62UpperFirst(s string) string {
	return stringsutil.ToOpposite(Md5Base62(s))
}
<|endoftext|>