<commit_before>// htwtxt – hosted twtxt server; see README for copyright and license info

package main

import "bufio"
import "errors"
import "flag"
import "github.com/gorilla/mux"
import "golang.org/x/crypto/bcrypt"
import "html/template"
import "io/ioutil"
import "log"
import "net/http"
import "os"
import "strconv"
import "strings"
import "time"

const loginsFile = "logins.txt"
const twtsDir = "feeds"
const portDefault = 8000

var templ *template.Template

func createFileIfNotExists(path string) {
	if _, err := os.Stat(path); err != nil {
		file, err := os.Create(path)
		if err != nil {
			log.Fatal("Can't create file: ", err)
		}
		file.Close()
	}
}

func appendToFile(path string, msg string) {
	fileWrite, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0600)
	defer fileWrite.Close()
	if err != nil {
		log.Fatal("Can't open file for appending", err)
	}
	if _, err = fileWrite.WriteString(msg); err != nil {
		log.Fatal("Can't write to file", err)
	}
}

func onlyLegalRunes(str string) bool {
	alphabet := "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" +
		"0123456789_"
	for _, ru := range str {
		if !(strings.ContainsRune(alphabet, ru)) {
			return false
		}
	}
	return true
}

func execTemplate(w http.ResponseWriter, file string, input string) {
	type data struct{ Msg string }
	err := templ.ExecuteTemplate(w, file, data{Msg: input})
	if err != nil {
		log.Fatal("Trouble executing template", err)
	}
}

func login(w http.ResponseWriter, r *http.Request) (string, error) {
	name := r.FormValue("name")
	pw := r.FormValue("password")
	loginValid := false
	file, err := os.Open(loginsFile)
	defer file.Close()
	if err != nil {
		log.Fatal("Can't open file for reading", err)
	}
	scanner := bufio.NewScanner(bufio.NewReader(file))
	for {
		if !scanner.Scan() {
			break
		}
		line := scanner.Text()
		tokens := strings.Split(line, " ")
		if len(tokens) == 3 {
			if 0 == strings.Compare(tokens[0], name) &&
				nil == bcrypt.CompareHashAndPassword(
					[]byte(tokens[1]), []byte(pw)) {
				loginValid = true

			}
		}
	}
	if !loginValid {
		execTemplate(w, "error.html", "Bad login.")
		return name, errors.New("")
	}
	return name, nil
}

func accountLine(w http.ResponseWriter, r *http.Request,
	checkDupl bool) (string, error) {
	name := r.FormValue("name")
	pw := r.FormValue("new_password")
	pw2 := r.FormValue("new_password2")
	mail := r.FormValue("mail")
	if 0 != strings.Compare(pw, pw2) || 0 == strings.Compare("name", "") ||
		0 == strings.Compare(pw, "") || !onlyLegalRunes(name) ||
		len(name) > 140 {
		execTemplate(w, "error.html", "Invalid values.")
		return "", errors.New("")
	}
	if checkDupl {
		fileRead, err := os.Open(loginsFile)
		defer fileRead.Close()
		if err != nil {
			log.Fatal("Can't open file for reading", err)
		}
		scanner := bufio.NewScanner(bufio.NewReader(fileRead))
		for {
			if !scanner.Scan() {
				break
			}
			line := scanner.Text()
			tokens := strings.Split(line, " ")
			if 0 == strings.Compare(name, tokens[0]) {
				execTemplate(w, "error.html", "Username taken.")
				return "", errors.New("")
			}
		}
	}
	hash, err := bcrypt.GenerateFromPassword([]byte(pw), bcrypt.DefaultCost)
	if err != nil {
		log.Fatal("Can't generate password hash", err)
	}
	return name + " " + string(hash) + " " + mail, nil
}

func indexHandler(w http.ResponseWriter, r *http.Request) {
	execTemplate(w, "index.html", "")
}

func signUpFormHandler(w http.ResponseWriter, r *http.Request) {
	execTemplate(w, "signupform.html", "")
}

func signUpHandler(w http.ResponseWriter, r *http.Request) {
	newLine, err := accountLine(w, r, true)
	if err != nil {
		return
	}
	appendToFile(loginsFile, newLine+"\n")
	execTemplate(w, "feedset.html", "")
}

func accountFormHandler(w http.ResponseWriter, r *http.Request) {
	execTemplate(w, "accountform.html", "")
}

func accountPostHandler(w http.ResponseWriter, r *http.Request) {
	name, err := login(w, r)
	if err != nil {
		return
	}
	newLine, err := accountLine(w, r, false)
	if err != nil {
		return
	}
	text, err := ioutil.ReadFile(loginsFile)
	if err != nil {
		log.Fatal("Can't read file", err)
	}
	lines := strings.Split(string(text), "\n")
	for i, line := range lines {
		tokens := strings.Split(line, " ")
		if 0 == strings.Compare(name, tokens[0]) {
			lines[i] = newLine
			break
		}
	}
	text = []byte(strings.Join(lines, "\n"))
	tmpFile := "tmp_" + loginsFile
	if err := ioutil.WriteFile(tmpFile, []byte(text), 0600); err != nil {
		log.Fatal("Trouble writing file", err)
	}
	if err := os.Rename(loginsFile, "_"+loginsFile); err != nil {
		log.Fatal("Trouble moving file", err)
	}
	if err := os.Rename(tmpFile, loginsFile); err != nil {
		log.Fatal("Trouble moving file", err)
	}
	if err := os.Remove("_" + loginsFile); err != nil {
		log.Fatal("Trouble removing file", err)
	}
	execTemplate(w, "feedset.html", "")
}

func listHandler(w http.ResponseWriter, r *http.Request) {
	file, err := os.Open(loginsFile)
	defer file.Close()
	if err != nil {
		log.Fatal("Can't open file for reading", err)
	}
	scanner := bufio.NewScanner(bufio.NewReader(file))
	var dir []string
	for {
		if !scanner.Scan() {
			break
		}
		line := scanner.Text()
		tokens := strings.Split(line, " ")
		if len(tokens) == 3 {
			dir = append(dir, tokens[0])
		}
	}
	type data struct{ Dir []string }
	err = templ.ExecuteTemplate(w, "list.html", data{Dir: dir})
	if err != nil {
		log.Fatal("Trouble executing template", err)
	}
}

func twtxtPostHandler(w http.ResponseWriter, r *http.Request) {
	name, err := login(w, r)
	if err != nil {
		return
	}
	text := r.FormValue("twt")
	twtsFile := twtsDir + "/" + name
	createFileIfNotExists(twtsFile)
	text = strings.Replace(text, "\n", " ", -1)
	appendToFile(twtsFile, time.Now().Format(time.RFC3339)+"\t"+text+"\n")
	http.Redirect(w, r, "/"+twtsFile, 302)
}

func twtxtHandler(w http.ResponseWriter, r *http.Request) {
	name := mux.Vars(r)["name"]
	if !onlyLegalRunes(name) {
		execTemplate(w, "error.html", "Bad path.")
		return
	}
	path := twtsDir + "/" + name
	if _, err := os.Stat(path); err != nil {
		execTemplate(w, "error.html", "Empty twtxt for user.")
		return
	}
	http.ServeFile(w, r, path)
}

func main() {
	var err error
	portPtr := flag.Int("port", 8000, "port to serve")
	keyPtr := flag.String("key", "", "SSL key file")
	certPtr := flag.String("cert", "", "SSL certificate file")
	flag.Parse()
	if ("" == *keyPtr && "" != *certPtr) ||
		("" != *keyPtr && "" == *certPtr) {
		log.Fatal("Expect either both key and certificate or none.")
	}
	if "" != *keyPtr {
		log.Println("using TLS")
		if _, err := os.Stat(*certPtr); err != nil {
			log.Fatal("No certificate file found.")
		}
		if _, err := os.Stat(*keyPtr); err != nil {
			log.Fatal("No server key file found.")
		}
	}
	createFileIfNotExists(loginsFile)
	// TODO: Handle err here.
	_ = os.Mkdir(twtsDir, 0700)
	templ, err = template.New("main").ParseGlob("./templates/*.html")
	if err != nil {
		log.Fatal("Can't set up new template: ", err)
	}
	router := mux.NewRouter()
	router.HandleFunc("/", indexHandler)
	router.HandleFunc("/feeds", listHandler).Methods("GET")
	router.HandleFunc("/feeds/", listHandler)
	router.HandleFunc("/account", accountFormHandler).Methods("GET")
	router.HandleFunc("/account", accountPostHandler).Methods("POST")
	router.HandleFunc("/signup", signUpFormHandler).Methods("GET")
	router.HandleFunc("/signup", signUpHandler).Methods("POST")
	router.HandleFunc("/feeds", twtxtPostHandler).Methods("POST")
	router.HandleFunc("/feeds/{name}", twtxtHandler)
	http.Handle("/", router)
	log.Println("serving at port", *portPtr)
	if "" != *keyPtr {
		err = http.ListenAndServeTLS(":"+strconv.Itoa(*portPtr),
			*certPtr, *keyPtr, nil)
	} else {
		err = http.ListenAndServe(":"+strconv.Itoa(*portPtr), nil)
	}
	if err != nil {
		log.Fatal("ListenAndServe: ", err)
	}
}
<commit_msg>Drop unused default port number constant.<commit_after>// htwtxt – hosted twtxt server; see README for copyright and license info

package main

import "bufio"
import "errors"
import "flag"
import "github.com/gorilla/mux"
import "golang.org/x/crypto/bcrypt"
import "html/template"
import "io/ioutil"
import "log"
import "net/http"
import "os"
import "strconv"
import "strings"
import "time"

const loginsFile = "logins.txt"
const twtsDir = "feeds"

var templ *template.Template

func createFileIfNotExists(path string) {
	if _, err := os.Stat(path); err != nil {
		file, err := os.Create(path)
		if err != nil {
			log.Fatal("Can't create file: ", err)
		}
		file.Close()
	}
}

func appendToFile(path string, msg string) {
	fileWrite, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0600)
	defer fileWrite.Close()
	if err != nil {
		log.Fatal("Can't open file for appending", err)
	}
	if _, err = fileWrite.WriteString(msg); err != nil {
		log.Fatal("Can't write to file", err)
	}
}

func onlyLegalRunes(str string) bool {
	alphabet := "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" +
		"0123456789_"
	for _, ru := range str {
		if !(strings.ContainsRune(alphabet, ru)) {
			return false
		}
	}
	return true
}

func execTemplate(w http.ResponseWriter, file string, input string) {
	type data struct{ Msg string }
	err := templ.ExecuteTemplate(w, file, data{Msg: input})
	if err != nil {
		log.Fatal("Trouble executing template", err)
	}
}

func login(w http.ResponseWriter, r *http.Request) (string, error) {
	name := r.FormValue("name")
	pw := r.FormValue("password")
	loginValid := false
	file, err := os.Open(loginsFile)
	defer file.Close()
	if err != nil {
		log.Fatal("Can't open file for reading", err)
	}
	scanner := bufio.NewScanner(bufio.NewReader(file))
	for {
		if !scanner.Scan() {
			break
		}
		line := scanner.Text()
		tokens := strings.Split(line, " ")
		if len(tokens) == 3 {
			if 0 == strings.Compare(tokens[0], name) &&
				nil == bcrypt.CompareHashAndPassword(
					[]byte(tokens[1]), []byte(pw)) {
				loginValid = true

			}
		}
	}
	if !loginValid {
		execTemplate(w, "error.html", "Bad login.")
		return name, errors.New("")
	}
	return name, nil
}

func accountLine(w http.ResponseWriter, r *http.Request,
	checkDupl bool) (string, error) {
	name := r.FormValue("name")
	pw := r.FormValue("new_password")
	pw2 := r.FormValue("new_password2")
	mail := r.FormValue("mail")
	if 0 != strings.Compare(pw, pw2) || 0 == strings.Compare("name", "") ||
		0 == strings.Compare(pw, "") || !onlyLegalRunes(name) ||
		len(name) > 140 {
		execTemplate(w, "error.html", "Invalid values.")
		return "", errors.New("")
	}
	if checkDupl {
		fileRead, err := os.Open(loginsFile)
		defer fileRead.Close()
		if err != nil {
			log.Fatal("Can't open file for reading", err)
		}
		scanner := bufio.NewScanner(bufio.NewReader(fileRead))
		for {
			if !scanner.Scan() {
				break
			}
			line := scanner.Text()
			tokens := strings.Split(line, " ")
			if 0 == strings.Compare(name, tokens[0]) {
				execTemplate(w, "error.html", "Username taken.")
				return "", errors.New("")
			}
		}
	}
	hash, err := bcrypt.GenerateFromPassword([]byte(pw), bcrypt.DefaultCost)
	if err != nil {
		log.Fatal("Can't generate password hash", err)
	}
	return name + " " + string(hash) + " " + mail, nil
}

func indexHandler(w http.ResponseWriter, r *http.Request) {
	execTemplate(w, "index.html", "")
}

func signUpFormHandler(w http.ResponseWriter, r *http.Request) {
	execTemplate(w, "signupform.html", "")
}

func signUpHandler(w http.ResponseWriter, r *http.Request) {
	newLine, err := accountLine(w, r, true)
	if err != nil {
		return
	}
	appendToFile(loginsFile, newLine+"\n")
	execTemplate(w, "feedset.html", "")
}

func accountFormHandler(w http.ResponseWriter, r *http.Request) {
	execTemplate(w, "accountform.html", "")
}

func accountPostHandler(w http.ResponseWriter, r *http.Request) {
	name, err := login(w, r)
	if err != nil {
		return
	}
	newLine, err := accountLine(w, r, false)
	if err != nil {
		return
	}
	text, err := ioutil.ReadFile(loginsFile)
	if err != nil {
		log.Fatal("Can't read file", err)
	}
	lines := strings.Split(string(text), "\n")
	for i, line := range lines {
		tokens := strings.Split(line, " ")
		if 0 == strings.Compare(name, tokens[0]) {
			lines[i] = newLine
			break
		}
	}
	text = []byte(strings.Join(lines, "\n"))
	tmpFile := "tmp_" + loginsFile
	if err := ioutil.WriteFile(tmpFile, []byte(text), 0600); err != nil {
		log.Fatal("Trouble writing file", err)
	}
	if err := os.Rename(loginsFile, "_"+loginsFile); err != nil {
		log.Fatal("Trouble moving file", err)
	}
	if err := os.Rename(tmpFile, loginsFile); err != nil {
		log.Fatal("Trouble moving file", err)
	}
	if err := os.Remove("_" + loginsFile); err != nil {
		log.Fatal("Trouble removing file", err)
	}
	execTemplate(w, "feedset.html", "")
}

func listHandler(w http.ResponseWriter, r *http.Request) {
	file, err := os.Open(loginsFile)
	defer file.Close()
	if err != nil {
		log.Fatal("Can't open file for reading", err)
	}
	scanner := bufio.NewScanner(bufio.NewReader(file))
	var dir []string
	for {
		if !scanner.Scan() {
			break
		}
		line := scanner.Text()
		tokens := strings.Split(line, " ")
		if len(tokens) == 3 {
			dir = append(dir, tokens[0])
		}
	}
	type data struct{ Dir []string }
	err = templ.ExecuteTemplate(w, "list.html", data{Dir: dir})
	if err != nil {
		log.Fatal("Trouble executing template", err)
	}
}

func twtxtPostHandler(w http.ResponseWriter, r *http.Request) {
	name, err := login(w, r)
	if err != nil {
		return
	}
	text := r.FormValue("twt")
	twtsFile := twtsDir + "/" + name
	createFileIfNotExists(twtsFile)
	text = strings.Replace(text, "\n", " ", -1)
	appendToFile(twtsFile, time.Now().Format(time.RFC3339)+"\t"+text+"\n")
	http.Redirect(w, r, "/"+twtsFile, 302)
}

func twtxtHandler(w http.ResponseWriter, r *http.Request) {
	name := mux.Vars(r)["name"]
	if !onlyLegalRunes(name) {
		execTemplate(w, "error.html", "Bad path.")
		return
	}
	path := twtsDir + "/" + name
	if _, err := os.Stat(path); err != nil {
		execTemplate(w, "error.html", "Empty twtxt for user.")
		return
	}
	http.ServeFile(w, r, path)
}

func main() {
	var err error
	portPtr := flag.Int("port", 8000, "port to serve")
	keyPtr := flag.String("key", "", "SSL key file")
	certPtr := flag.String("cert", "", "SSL certificate file")
	flag.Parse()
	if ("" == *keyPtr && "" != *certPtr) ||
		("" != *keyPtr && "" == *certPtr) {
		log.Fatal("Expect either both key and certificate or none.")
	}
	if "" != *keyPtr {
		log.Println("using TLS")
		if _, err := os.Stat(*certPtr); err != nil {
			log.Fatal("No certificate file found.")
		}
		if _, err := os.Stat(*keyPtr); err != nil {
			log.Fatal("No server key file found.")
		}
	}
	createFileIfNotExists(loginsFile)
	// TODO: Handle err here.
	_ = os.Mkdir(twtsDir, 0700)
	templ, err = template.New("main").ParseGlob("./templates/*.html")
	if err != nil {
		log.Fatal("Can't set up new template: ", err)
	}
	router := mux.NewRouter()
	router.HandleFunc("/", indexHandler)
	router.HandleFunc("/feeds", listHandler).Methods("GET")
	router.HandleFunc("/feeds/", listHandler)
	router.HandleFunc("/account", accountFormHandler).Methods("GET")
	router.HandleFunc("/account", accountPostHandler).Methods("POST")
	router.HandleFunc("/signup", signUpFormHandler).Methods("GET")
	router.HandleFunc("/signup", signUpHandler).Methods("POST")
	router.HandleFunc("/feeds", twtxtPostHandler).Methods("POST")
	router.HandleFunc("/feeds/{name}", twtxtHandler)
	http.Handle("/", router)
	log.Println("serving at port", *portPtr)
	if "" != *keyPtr {
		err = http.ListenAndServeTLS(":"+strconv.Itoa(*portPtr),
			*certPtr, *keyPtr, nil)
	} else {
		err = http.ListenAndServe(":"+strconv.Itoa(*portPtr), nil)
	}
	if err != nil {
		log.Fatal("ListenAndServe: ", err)
	}
}
<|endoftext|>


<commit_before>package main

/*
 * Written: Dave Bevan
 * Date   : 23 Oct '17
 *
 * Purpose: To monitor a local folder, watching for file events using iNotify.
 * Specifically, trigger when CLOSE events have been received.
 *
 * Configured via 3 environment variables:
 *
 *   WATCHFOLDER=/path/to/be/monitored - can ONLY be a local folder, not network shares (as inotify events are absent from anything other than local paths)
 *
 *   %%PATH%% will be replaced with the full pathname to the file concerned:
 *
 *   E.g. WORKFLOWURL=http://something.url/some/endpoint.php?path=%%PATH%%
 *        WORKFLOWURL=http://something.url/rest/endpoint/action/%%PATH%%
 *
 *   FILESUFFIX=.xml - only call WORKFLOWURL for CLOSE events on files ending in $FILESUFFIX
 *
 */
import (
	"github.com/betim/fsnotify"
	"github.com/yookoala/realpath"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"
)

func main() {
	log.SetFlags(log.LstdFlags | log.Lmicroseconds)

	watchfolder, ok := os.LookupEnv("WATCHFOLDER")
	if !ok {
		log.Fatal("Env WATCHFOLDER unset!")
	}

	watchfolder, err := realpath.Realpath(watchfolder)
	if err != nil {
		log.Fatal(error(err))
	}

	workflowurl, ok := os.LookupEnv("WORKFLOWURL")
	if !ok {
		log.Fatal("Env WORKFLOWURL unset!\n",
			"  E.g. WORKFLOWURL=http://something.url/some/endpoint.php?path=%%PATH%%\n",
			"       WORKFLOWURL=http://something.url/rest/endpoint/action/%%PATH%%")
	}

	filesuffix, ok := os.LookupEnv("FILESUFFIX")
	if !ok {
		log.Fatal("Env FILESUFFIX unset!")
	}

	startWatching(watchfolder, workflowurl, filesuffix)
}

func startWatching(watchfolder, workflowurl string, filesuffix string) {

	log.Println("WATCHFOLDER=" + watchfolder)
	log.Println("WORKFLOWURL=" + workflowurl)
	log.Println("FILESUFFIX=" + filesuffix)

	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		log.Fatal(err)
	}

	defer watcher.Close()

	done := make(chan bool)

	go func() {
		for {
			select {
			case event := <-watcher.Events:
				procEvent(event, workflowurl, filesuffix)
			case err := <-watcher.Errors:
				log.Println("error:", err)
			}
		}
	}()

	err = watcher.Add(watchfolder)
	if err != nil {
		log.Fatal(err)
	}

	<-done
}

func procEvent(event fsnotify.Event, workflowurl string, filesuffix string) {

	if event.Op&fsnotify.Close == fsnotify.Close {

		log.Println("CLOSE:", event.Name)

		if strings.HasSuffix(event.Name, filesuffix) {

			p := strings.LastIndex(event.Name, filesuffix)

			wfurl := strings.Replace(workflowurl, "%%PATH%%", url.PathEscape(event.Name[:p]), -1)

			if strings.Contains(wfurl, "?") {
				wfurl += "&FILESUFFIX=" + filesuffix
			} else {
				wfurl += "?FILESUFFIX=" + filesuffix
			}
			log.Println("WORKFLOW:", wfurl)

			var client = &http.Client{
				Timeout: time.Second * 5,
			}

			retval, err := client.Get(wfurl)
			if err != nil {
				log.Println("Error:", err)
			}

			defer retval.Body.Close()
			body, err := ioutil.ReadAll(retval.Body)
			if err != nil {
				log.Println("Error:", err)
			}

			log.Println("RESPONSE:", string(body))
		}
	}
}
<commit_msg>Handle invalid-host or other errors<commit_after>package main

/*
 * Written: Dave Bevan
 * Date   : 23 Oct '17
 *
 * Purpose: To monitor a local folder, watching for file events using iNotify.
 * Specifically, trigger when CLOSE events have been received.
 *
 * Configured via 3 environment variables:
 *
 *   WATCHFOLDER=/path/to/be/monitored - can ONLY be a local folder, not network shares (as inotify events are absent from anything other than local paths)
 *
 *   %%PATH%% will be replaced with the full pathname to the file concerned:
 *
 *   E.g. WORKFLOWURL=http://something.url/some/endpoint.php?path=%%PATH%%
 *        WORKFLOWURL=http://something.url/rest/endpoint/action/%%PATH%%
 *
 *   FILESUFFIX=.xml - only call WORKFLOWURL for CLOSE events on files ending in $FILESUFFIX
 *
 */
import (
	"github.com/betim/fsnotify"
	"github.com/yookoala/realpath"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"
)

func main() {
	log.SetFlags(log.LstdFlags | log.Lmicroseconds)

	watchfolder, ok := os.LookupEnv("WATCHFOLDER")
	if !ok {
		log.Fatal("Env WATCHFOLDER unset!")
	}

	watchfolder, err := realpath.Realpath(watchfolder)
	if err != nil {
		log.Fatal(error(err))
	}

	workflowurl, ok := os.LookupEnv("WORKFLOWURL")
	if !ok {
		log.Fatal("Env WORKFLOWURL unset!\n",
			"  E.g. WORKFLOWURL=http://something.url/some/endpoint.php?path=%%PATH%%\n",
			"       WORKFLOWURL=http://something.url/rest/endpoint/action/%%PATH%%")
	}

	filesuffix, ok := os.LookupEnv("FILESUFFIX")
	if !ok {
		log.Fatal("Env FILESUFFIX unset!")
	}

	startWatching(watchfolder, workflowurl, filesuffix)
}

func startWatching(watchfolder, workflowurl string, filesuffix string) {

	log.Println("WATCHFOLDER=" + watchfolder)
	log.Println("WORKFLOWURL=" + workflowurl)
	log.Println("FILESUFFIX=" + filesuffix)

	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		log.Fatal(err)
	}

	defer watcher.Close()

	done := make(chan bool)

	go func() {
		for {
			select {
			case event := <-watcher.Events:
				procEvent(event, workflowurl, filesuffix)
			case err := <-watcher.Errors:
				log.Println("error:", err)
			}
		}
	}()

	err = watcher.Add(watchfolder)
	if err != nil {
		log.Fatal(err)
	}

	<-done
}

func procEvent(event fsnotify.Event, workflowurl string, filesuffix string) {

	if event.Op&fsnotify.Close == fsnotify.Close {

		log.Println("CLOSE:", event.Name)

		if strings.HasSuffix(event.Name, filesuffix) {

			p := strings.LastIndex(event.Name, filesuffix)

			wfurl := strings.Replace(workflowurl, "%%PATH%%", url.PathEscape(event.Name[:p]), -1)

			if strings.Contains(wfurl, "?") {
				wfurl += "&FILESUFFIX=" + filesuffix
			} else {
				wfurl += "?FILESUFFIX=" + filesuffix
			}
			log.Println("WORKFLOW:", wfurl)

			var client = &http.Client{
				Timeout: time.Second * 5,
			}

			retval, err := client.Get(wfurl)
			if err != nil {

				log.Println("Error:", err)

			} else {

				defer retval.Body.Close()
				body, err := ioutil.ReadAll(retval.Body)
				if err != nil {
					log.Println("Error:", err)
				}

				log.Println("RESPONSE:", string(body))
			}
		}
	}
}
<|endoftext|>


<commit_before>package main

import (//"fmt"
       //"html"
       "log"
       "net/http"
       )

func main() {

	fs := http.FileServer(http.Dir("dashboard"))
  	http.Handle("/dashboard/", http.StripPrefix("/dashboard", fs))

  	log.Println("Listening...")
  	http.ListenAndServe(":8080", nil)

}

/**
package main

import (
    "fmt"
    "html"
    "log"
    "net/http"
)

func main() {
    http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
        fmt.Fprintf(w, "Hello, %q", html.EscapeString(r.URL.Path))
    })

    log.Fatal(http.ListenAndServe(":8080", nil))

}

*/<commit_msg>add current version<commit_after>package main

import (
	//"encoding/json"
	//"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	//"time"
	//"strings"
	"html"
	"net/url"
	"net"
    )

//initialize an array of strings
var results []string

//function to collect post requests
func parsePost (w http.ResponseWriter, r *http.Request){

	fmt.Println(r.Method)

	if r.Method == "POST" {

		//fmt.Printf("Req: %s %s\n", r.Host, r.URL.Path)
		//url := html.EscapeString(r.URL.Path)
		//fmt.Println(url)
		//s := strings.Split(url,"id")

		//before := s[0]
		//after := s[1]
		//fmt.Println(before,after)

		body, err := ioutil.ReadAll(r.Body)
		if err != nil{

			http.Error(w, "Error reading request body", http.StatusInternalServerError)
		}
		results = append(results, string(body))

		fmt.Fprint(w, "POST done")

	}else{

			http.Error(w, "Invalid request method", http.StatusMethodNotAllowed)
		}
}

func main() {

	//accept front end files
	fs := http.FileServer(http.Dir("dashboard"))
  	http.Handle("/dashboard/", http.StripPrefix("/dashboard", fs))

  	//create mux to identify post type
  	mux := http.NewServeMux()
  	mux.HandleFunc("/push", parsePost)

    /*s := "postgres://user:pass@host.com:5432/path?k=v#f"

    u, err := url.Parse(s)
    if err != nil {
        panic(err)
    }
    fmt.Println(u.Scheme)

    fmt.Println(u.User)
    fmt.Println(u.User.Username())
    p, _ := u.User.Password()
    fmt.Println(p)

    fmt.Println(u.Host)
    host, port, _ := net.SplitHostPort(u.Host)
    fmt.Println(host)
    fmt.Println(port)
    fmt.Println(u.Path)
    fmt.Println(u.Fragment)
    fmt.Println(u.RawQuery)
    m, _ := url.ParseQuery(u.RawQuery)
    fmt.Println(m)
    fmt.Println(m["k"][0])
	*/

  	log.Println("Listening...")
   	http.ListenAndServe(":8080", mux)
}

<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strconv"

	"github.com/bradleyfalzon/gopherci/internal/analyser"
	"github.com/bradleyfalzon/gopherci/internal/db"
	"github.com/bradleyfalzon/gopherci/internal/github"
	_ "github.com/go-sql-driver/mysql"
	"github.com/joho/godotenv"
	"github.com/pkg/errors"
	migrate "github.com/rubenv/sql-migrate"
)

func main() {
	err := godotenv.Load()
	if err != nil {
		log.Fatal("Error loading .env file:", err)
	}

	switch {
	case os.Getenv("GITHUB_ID") == "":
		log.Fatalln("GITHUB_ID is not set")
	case os.Getenv("GITHUB_PEM_FILE") == "":
		log.Fatalln("GITHUB_PEM_FILE is not set")
	}

	// Database
	log.Printf("Connecting to %q db name: %q, username: %q, host: %q, port: %q",
		os.Getenv("DB_DRIVER"), os.Getenv("DB_DATABASE"), os.Getenv("DB_USERNAME"), os.Getenv("DB_HOST"), os.Getenv("DB_PORT"),
	)

	dsn := fmt.Sprintf(`%s:%s@tcp(%s:%s)/%s?charset=utf8&collation=utf8_unicode_ci&timeout=6s&time_zone='%%2B00:00'&parseTime=true`,
		os.Getenv("DB_USERNAME"), os.Getenv("DB_PASSWORD"), os.Getenv("DB_HOST"), os.Getenv("DB_PORT"), os.Getenv("DB_DATABASE"),
	)

	sqlDB, err := sql.Open(os.Getenv("DB_DRIVER"), dsn)
	if err != nil {
		log.Fatal("Error setting up DB:", err)
	}

	// Do DB migrations
	migrations := &migrate.FileMigrationSource{Dir: "migrations"}
	migrate.SetTable("migrations")
	direction := migrate.Up
	if len(os.Args) > 1 && os.Args[1] == "down" {
		direction = migrate.Down
	}
	n, err := migrate.Exec(sqlDB, os.Getenv("DB_DRIVER"), migrations, direction)
	log.Printf("Applied %d migrations to database", n)
	if err != nil {
		log.Fatal(errors.Wrap(err, "could not execute all migrations"))
	}

	db, err := db.NewSQLDB(sqlDB, os.Getenv("DB_DRIVER"))
	if err != nil {
		log.Fatalln("could not initialise db:", err)
	}

	// Analyser
	log.Printf("Using analyser %q", os.Getenv("ANALYSER"))
	var analyse analyser.Analyser
	switch os.Getenv("ANALYSER") {
	case "filesystem":
		if os.Getenv("ANALYSER_FILESYSTEM_PATH") == "" {
			log.Fatalln("ANALYSER_FILESYSTEM_PATH is not set")
		}
		analyse, err = analyser.NewFileSystem(os.Getenv("ANALYSER_FILESYSTEM_PATH"))
		if err != nil {
			log.Fatalln("could not initialise file system analyser:", err)
		}
	case "docker":
		image := os.Getenv("ANALYSER_DOCKER_IMAGE")
		if image == "" {
			image = analyser.DockerDefaultImage
		}
		analyse, err = analyser.NewDocker(image)
		if err != nil {
			log.Fatalln("could not initialise Docker analyser:", err)
		}
	case "":
		log.Fatalln("ANALYSER is not set")
	default:
		log.Fatalf("Unknown ANALYSER option %q", os.Getenv("ANALYSER"))
	}

	// GitHub
	log.Printf("GitHub Integration ID: %q, GitHub Integration PEM File: %q", os.Getenv("GITHUB_ID"), os.Getenv("GITHUB_PEM_FILE"))
	integrationID, err := strconv.ParseInt(os.Getenv("GITHUB_ID"), 10, 64)
	if err != nil {
		log.Fatalf("could not parse integrationID %q", os.Getenv("GITHUB_ID"))
	}

	integrationKey, err := ioutil.ReadFile(os.Getenv("GITHUB_PEM_FILE"))
	if err != nil {
		log.Fatalf("could not read private key for GitHub integration: %s", err)
	}

	gh, err := github.New(analyse, db, int(integrationID), integrationKey)
	if err != nil {
		log.Fatalln("could not initialise GitHub:", err)
	}
	http.HandleFunc("/gh/webhook", gh.WebHookHandler)
	http.HandleFunc("/gh/callback", gh.CallBackHandler)

	// Listen
	log.Println("Listening on :3000")
	if err := http.ListenAndServe(":3000", nil); err != nil {
		log.Fatal(err)
	}
}
<commit_msg>Migrate down only one migration, not all<commit_after>package main

import (
	"database/sql"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strconv"

	"github.com/bradleyfalzon/gopherci/internal/analyser"
	"github.com/bradleyfalzon/gopherci/internal/db"
	"github.com/bradleyfalzon/gopherci/internal/github"
	_ "github.com/go-sql-driver/mysql"
	"github.com/joho/godotenv"
	"github.com/pkg/errors"
	migrate "github.com/rubenv/sql-migrate"
)

func main() {
	err := godotenv.Load()
	if err != nil {
		log.Fatal("Error loading .env file:", err)
	}

	switch {
	case os.Getenv("GITHUB_ID") == "":
		log.Fatalln("GITHUB_ID is not set")
	case os.Getenv("GITHUB_PEM_FILE") == "":
		log.Fatalln("GITHUB_PEM_FILE is not set")
	}

	// Database
	log.Printf("Connecting to %q db name: %q, username: %q, host: %q, port: %q",
		os.Getenv("DB_DRIVER"), os.Getenv("DB_DATABASE"), os.Getenv("DB_USERNAME"), os.Getenv("DB_HOST"), os.Getenv("DB_PORT"),
	)

	dsn := fmt.Sprintf(`%s:%s@tcp(%s:%s)/%s?charset=utf8&collation=utf8_unicode_ci&timeout=6s&time_zone='%%2B00:00'&parseTime=true`,
		os.Getenv("DB_USERNAME"), os.Getenv("DB_PASSWORD"), os.Getenv("DB_HOST"), os.Getenv("DB_PORT"), os.Getenv("DB_DATABASE"),
	)

	sqlDB, err := sql.Open(os.Getenv("DB_DRIVER"), dsn)
	if err != nil {
		log.Fatal("Error setting up DB:", err)
	}

	// Do DB migrations
	migrations := &migrate.FileMigrationSource{Dir: "migrations"}
	migrate.SetTable("migrations")
	direction := migrate.Up
	migrateMax := 0
	if len(os.Args) > 1 && os.Args[1] == "down" {
		direction = migrate.Down
		migrateMax = 1
	}
	n, err := migrate.ExecMax(sqlDB, os.Getenv("DB_DRIVER"), migrations, direction, migrateMax)
	log.Printf("Applied %d migrations to database", n)
	if err != nil {
		log.Fatal(errors.Wrap(err, "could not execute all migrations"))
	}

	db, err := db.NewSQLDB(sqlDB, os.Getenv("DB_DRIVER"))
	if err != nil {
		log.Fatalln("could not initialise db:", err)
	}

	// Analyser
	log.Printf("Using analyser %q", os.Getenv("ANALYSER"))
	var analyse analyser.Analyser
	switch os.Getenv("ANALYSER") {
	case "filesystem":
		if os.Getenv("ANALYSER_FILESYSTEM_PATH") == "" {
			log.Fatalln("ANALYSER_FILESYSTEM_PATH is not set")
		}
		analyse, err = analyser.NewFileSystem(os.Getenv("ANALYSER_FILESYSTEM_PATH"))
		if err != nil {
			log.Fatalln("could not initialise file system analyser:", err)
		}
	case "docker":
		image := os.Getenv("ANALYSER_DOCKER_IMAGE")
		if image == "" {
			image = analyser.DockerDefaultImage
		}
		analyse, err = analyser.NewDocker(image)
		if err != nil {
			log.Fatalln("could not initialise Docker analyser:", err)
		}
	case "":
		log.Fatalln("ANALYSER is not set")
	default:
		log.Fatalf("Unknown ANALYSER option %q", os.Getenv("ANALYSER"))
	}

	// GitHub
	log.Printf("GitHub Integration ID: %q, GitHub Integration PEM File: %q", os.Getenv("GITHUB_ID"), os.Getenv("GITHUB_PEM_FILE"))
	integrationID, err := strconv.ParseInt(os.Getenv("GITHUB_ID"), 10, 64)
	if err != nil {
		log.Fatalf("could not parse integrationID %q", os.Getenv("GITHUB_ID"))
	}

	integrationKey, err := ioutil.ReadFile(os.Getenv("GITHUB_PEM_FILE"))
	if err != nil {
		log.Fatalf("could not read private key for GitHub integration: %s", err)
	}

	gh, err := github.New(analyse, db, int(integrationID), integrationKey)
	if err != nil {
		log.Fatalln("could not initialise GitHub:", err)
	}
	http.HandleFunc("/gh/webhook", gh.WebHookHandler)
	http.HandleFunc("/gh/callback", gh.CallBackHandler)

	// Listen
	log.Println("Listening on :3000")
	if err := http.ListenAndServe(":3000", nil); err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"crypto/rand"
	"crypto/tls"
	"html/template"
	"log"
	"net"
	"net/http"
	"net/http/httputil"
	"net/url"
	"path"
	"path/filepath"
	"strings"
	"time"

	"github.com/daaku/go.httpgzip"
	"golang.org/x/crypto/acme/autocert"
)

//go:generate go-bindata ./static/... ./views
var templates = template.New("").Funcs(template.FuncMap{"add": func(a, b int) int { return a + b }})

var hostname_whitelist = []string{
	"www.sadbox.org", "sadbox.org",
	"www.sadbox.es", "sadbox.es",
	"www.geekwhack.org", "geekwhack.org",
}

type WebsiteName struct {
	Title, Brand string
}

type TemplateContext struct {
	Geekhack *Geekhack
	Webname  *WebsiteName
}

func NewContext(r *http.Request) *TemplateContext {
	title := "sadbox \u00B7 org"
	host := "sadbox.org"
	for _, v := range hostname_whitelist {
		if v == r.Host {
			title = strings.Replace(r.Host, ".", " \u00B7 ", -1)
			host = r.Host
			break
		}
	}
	return &TemplateContext{
		Webname: &WebsiteName{
			title,
			host,
		},
	}
}

func serveStatic(filename string) {
	relPath, err := filepath.Rel("static", filename)
	if err != nil {
		log.Fatal(err)
	}
	log.Println("Serving Static File:", relPath)

	handler := func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Cache-Control", "max-age=31536000")
		asset, err := Asset(filename)
		if err != nil {
			http.NotFound(w, r)
		}
		assetInfo, err := AssetInfo(filename)
		if err != nil {
			http.NotFound(w, r)
		}
		http.ServeContent(w, r, filename, assetInfo.ModTime(), bytes.NewReader(asset))
	}

	http.HandleFunc(path.Join("/", relPath), handler)
}

func CatchPanic(handler http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		defer func() {
			if r := recover(); r != nil {
				log.Printf("Recovered from panic: %v", r)
				http.Error(w, "Something went wrong!", http.StatusInternalServerError)
			}
		}()

		handler.ServeHTTP(w, r)
	})
}

func SendToHTTPS(w http.ResponseWriter, r *http.Request) {
	http.Redirect(w, r, "https://"+r.Host+r.RequestURI, http.StatusMovedPermanently)
}

func RedirectToHTTPS(handler http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		host, _, err := net.SplitHostPort(r.RemoteAddr)
		if err != nil {
			SendToHTTPS(w, r)
			return
		}

		ip := net.ParseIP(host)
		if ip == nil {
			SendToHTTPS(w, r)
			return
		}

		if !ip.IsLoopback() {
			SendToHTTPS(w, r)
			return
		}

		handler.ServeHTTP(w, r)
	})
}

func AddHeaders(handler http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Cache-Control", "max-age=120")
		w.Header().Set("Strict-Transport-Security", "max-age=31536000; includeSubDomains; preload")
		handler.ServeHTTP(w, r)
	})
}

func Log(handler http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		remoteHost := r.Header.Get("X-Forwarded-For")
		if remoteHost == "" {
			remoteHost = r.RemoteAddr
		}
		log.Printf("%s %s %s", remoteHost, r.Method, r.URL)
		handler.ServeHTTP(w, r)
	})
}

func getFiles(dir string) []string {
	files, err := AssetDir(dir)
	if err != nil {
		return []string{dir}
	}

	var outFiles []string
	for _, filename := range files {
		outFiles = append(outFiles, getFiles(path.Join(dir, filename))...)
	}
	return outFiles
}

type sessionKeys struct {
	keys [][32]byte
	buf  []byte
	tls  *tls.Config
}

func NewSessionKeys(tls *tls.Config) *sessionKeys {
	sk := &sessionKeys{
		buf: make([]byte, 32),
		tls: tls,
	}
	sk.addNewKey()
	return sk
}

func (sk *sessionKeys) addNewKey() {
	rand.Read(sk.buf)
	newKey := [32]byte{}
	copy(newKey[:], sk.buf)
	sk.keys = append([][32]byte{newKey}, sk.keys...)
	if len(sk.keys) > 4 {
		sk.keys = sk.keys[:5]
	}
	log.Println("Rotating Session Keys")
	sk.tls.SetSessionTicketKeys(sk.keys)
}

func (sk *sessionKeys) Spin() {
	for range time.Tick(24 * time.Hour) {
		sk.addNewKey()
	}
}

func main() {
	log.Println("Starting sadbox.org")

	for _, filename := range getFiles("views") {
		template.Must(templates.Parse(string(MustAsset(filename))))
	}

	// static files
	for _, filename := range getFiles("static") {
		serveStatic(filename)
	}

	geekhack, err := NewGeekhack()
	if err != nil {
		log.Fatal(err)
	}
	defer geekhack.db.Close()

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path != "/" {
			http.NotFound(w, r)
			return
		}

		ctx := NewContext(r)
		if err := templates.ExecuteTemplate(w, "main", ctx); err != nil {
			log.Println(err)
		}
	})

	// Geekhack stats! the geekhack struct will handle the routing to sub-things
	http.Handle("/geekhack/", geekhack)
	// Redirects to the right URL so I don't break old links
	http.Handle("/ghstats", http.RedirectHandler("/geekhack/", http.StatusMovedPermanently))
	http.Handle("/geekhack", http.RedirectHandler("/geekhack/", http.StatusMovedPermanently))

	localhost_znc, err := url.Parse("http://127.0.0.1:6698")
	if err != nil {
		log.Fatal(err)
	}
	http.Handle("/znc/", httputil.NewSingleHostReverseProxy(localhost_znc))
	http.Handle("/znc", http.RedirectHandler("/znc/", http.StatusMovedPermanently))

	servemux := httpgzip.NewHandler(
		CatchPanic(
			Log(
				AddHeaders(http.DefaultServeMux))))

	go func() {
		log.Fatal(http.ListenAndServe(":http", RedirectToHTTPS(servemux)))
	}()

	m := autocert.Manager{
		Prompt:     autocert.AcceptTOS,
		Cache:      autocert.DirCache("/home/sadbox-web/cert-cache"),
		HostPolicy: autocert.HostWhitelist(hostname_whitelist...),
		Email:      "blue6249@gmail.com",
		ForceRSA:   true,
	}

	var certs []tls.Certificate
	sadboxCert, err := tls.LoadX509KeyPair("/home/sadbox-web/cert-cache/sadbox.org",
		"/home/sadbox-web/cert-cache/sadbox.org")

	if err == nil {
		certs = append(certs, sadboxCert)
	}

	tlsconfig := &tls.Config{
		PreferServerCipherSuites: true,
		GetCertificate:           m.GetCertificate,
		Certificates:             certs,
	}

	go NewSessionKeys(tlsconfig).Spin()

	server := &http.Server{Addr: ":https", Handler: servemux, TLSConfig: tlsconfig}
	log.Fatal(server.ListenAndServeTLS("", ""))
}
<commit_msg>Set timeouts for http server<commit_after>package main

import (
	"bytes"
	"crypto/rand"
	"crypto/tls"
	"html/template"
	"log"
	"net"
	"net/http"
	"net/http/httputil"
	"net/url"
	"path"
	"path/filepath"
	"strings"
	"time"

	"github.com/daaku/go.httpgzip"
	"golang.org/x/crypto/acme/autocert"
)

//go:generate go-bindata ./static/... ./views
var templates = template.New("").Funcs(template.FuncMap{"add": func(a, b int) int { return a + b }})

var hostname_whitelist = []string{
	"www.sadbox.org", "sadbox.org",
	"www.sadbox.es", "sadbox.es",
	"www.geekwhack.org", "geekwhack.org",
}

type WebsiteName struct {
	Title, Brand string
}

type TemplateContext struct {
	Geekhack *Geekhack
	Webname  *WebsiteName
}

func NewContext(r *http.Request) *TemplateContext {
	title := "sadbox \u00B7 org"
	host := "sadbox.org"
	for _, v := range hostname_whitelist {
		if v == r.Host {
			title = strings.Replace(r.Host, ".", " \u00B7 ", -1)
			host = r.Host
			break
		}
	}
	return &TemplateContext{
		Webname: &WebsiteName{
			title,
			host,
		},
	}
}

func serveStatic(filename string) {
	relPath, err := filepath.Rel("static", filename)
	if err != nil {
		log.Fatal(err)
	}
	log.Println("Serving Static File:", relPath)

	handler := func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Cache-Control", "max-age=31536000")
		asset, err := Asset(filename)
		if err != nil {
			http.NotFound(w, r)
		}
		assetInfo, err := AssetInfo(filename)
		if err != nil {
			http.NotFound(w, r)
		}
		http.ServeContent(w, r, filename, assetInfo.ModTime(), bytes.NewReader(asset))
	}

	http.HandleFunc(path.Join("/", relPath), handler)
}

func CatchPanic(handler http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		defer func() {
			if r := recover(); r != nil {
				log.Printf("Recovered from panic: %v", r)
				http.Error(w, "Something went wrong!", http.StatusInternalServerError)
			}
		}()

		handler.ServeHTTP(w, r)
	})
}

func SendToHTTPS(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Connection", "close")
	http.Redirect(w, r, "https://"+r.Host+r.RequestURI, http.StatusMovedPermanently)
}

func RedirectToHTTPS(handler http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		host, _, err := net.SplitHostPort(r.RemoteAddr)
		if err != nil {
			SendToHTTPS(w, r)
			return
		}

		ip := net.ParseIP(host)
		if ip == nil {
			SendToHTTPS(w, r)
			return
		}

		if !ip.IsLoopback() {
			SendToHTTPS(w, r)
			return
		}

		handler.ServeHTTP(w, r)
	})
}

func AddHeaders(handler http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Cache-Control", "max-age=120")
		w.Header().Set("Strict-Transport-Security", "max-age=31536000; includeSubDomains; preload")
		handler.ServeHTTP(w, r)
	})
}

func Log(handler http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		remoteHost := r.Header.Get("X-Forwarded-For")
		if remoteHost == "" {
			remoteHost = r.RemoteAddr
		}
		log.Printf("%s %s %s", remoteHost, r.Method, r.URL)
		handler.ServeHTTP(w, r)
	})
}

func getFiles(dir string) []string {
	files, err := AssetDir(dir)
	if err != nil {
		return []string{dir}
	}

	var outFiles []string
	for _, filename := range files {
		outFiles = append(outFiles, getFiles(path.Join(dir, filename))...)
	}
	return outFiles
}

type sessionKeys struct {
	keys [][32]byte
	buf  []byte
	tls  *tls.Config
}

func NewSessionKeys(tls *tls.Config) *sessionKeys {
	sk := &sessionKeys{
		buf: make([]byte, 32),
		tls: tls,
	}
	sk.addNewKey()
	return sk
}

func (sk *sessionKeys) addNewKey() {
	rand.Read(sk.buf)
	newKey := [32]byte{}
	copy(newKey[:], sk.buf)
	sk.keys = append([][32]byte{newKey}, sk.keys...)
	if len(sk.keys) > 4 {
		sk.keys = sk.keys[:5]
	}
	log.Println("Rotating Session Keys")
	sk.tls.SetSessionTicketKeys(sk.keys)
}

func (sk *sessionKeys) Spin() {
	for range time.Tick(24 * time.Hour) {
		sk.addNewKey()
	}
}

func main() {
	log.Println("Starting sadbox.org")

	for _, filename := range getFiles("views") {
		template.Must(templates.Parse(string(MustAsset(filename))))
	}

	// static files
	for _, filename := range getFiles("static") {
		serveStatic(filename)
	}

	geekhack, err := NewGeekhack()
	if err != nil {
		log.Fatal(err)
	}
	defer geekhack.db.Close()

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path != "/" {
			http.NotFound(w, r)
			return
		}

		ctx := NewContext(r)
		if err := templates.ExecuteTemplate(w, "main", ctx); err != nil {
			log.Println(err)
		}
	})

	// Geekhack stats! the geekhack struct will handle the routing to sub-things
	http.Handle("/geekhack/", geekhack)
	// Redirects to the right URL so I don't break old links
	http.Handle("/ghstats", http.RedirectHandler("/geekhack/", http.StatusMovedPermanently))
	http.Handle("/geekhack", http.RedirectHandler("/geekhack/", http.StatusMovedPermanently))

	localhost_znc, err := url.Parse("http://127.0.0.1:6698")
	if err != nil {
		log.Fatal(err)
	}
	http.Handle("/znc/", httputil.NewSingleHostReverseProxy(localhost_znc))
	http.Handle("/znc", http.RedirectHandler("/znc/", http.StatusMovedPermanently))

	servemux := httpgzip.NewHandler(
		CatchPanic(
			Log(
				AddHeaders(http.DefaultServeMux))))

	httpSrv := &http.Server{
		ReadTimeout:  5 * time.Second,
		WriteTimeout: 5 * time.Second,
		Handler:      RedirectToHTTPS(servemux),
	}
	go func() { log.Fatal(httpSrv.ListenAndServe()) }()

	m := autocert.Manager{
		Prompt:     autocert.AcceptTOS,
		Cache:      autocert.DirCache("/home/sadbox-web/cert-cache"),
		HostPolicy: autocert.HostWhitelist(hostname_whitelist...),
		Email:      "blue6249@gmail.com",
		ForceRSA:   true,
	}

	var certs []tls.Certificate
	sadboxCert, err := tls.LoadX509KeyPair("/home/sadbox-web/cert-cache/sadbox.org",
		"/home/sadbox-web/cert-cache/sadbox.org")

	if err == nil {
		certs = append(certs, sadboxCert)
	}

	tlsconfig := &tls.Config{
		PreferServerCipherSuites: true,
		GetCertificate:           m.GetCertificate,
		Certificates:             certs,
	}

	go NewSessionKeys(tlsconfig).Spin()

	server := &http.Server{
		Addr:         ":https",
		Handler:      servemux,
		TLSConfig:    tlsconfig,
		ReadTimeout:  5 * time.Second,
		WriteTimeout: 5 * time.Second,
	}
	log.Fatal(server.ListenAndServeTLS("", ""))
}
<|endoftext|>


<commit_before>// Copyright 2016, RadiantBlue Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bytes"
	"crypto/rand"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"strconv"
	"strings"

	"github.com/venicegeo/pzsvc-exec/pzsvc"
)

type configType struct {
	CliCmd      string
	PzAddr      string
	SvcName     string
	URL         string
	Port        int
	Description	string
	ImageReqs	map[string]string
	Attributes	map[string]string
}

type outStruct struct {
	InFiles		map[string]string
	OutFiles	map[string]string
	ProgReturn	string
	Error		string
}

type pzCont struct {
	Type		string
	Content		string
	MimeType	string
}

type pzWrap struct {
	DataType	pzCont
	metadata	map[string]string
}

func main() {

	if len(os.Args) < 2 {
		fmt.Println("error: Insufficient parameters.  You must specify a config file.")
		return
	}

	// first argument after the base call should be the path to the config file.
	// ReadFile returns the contents of the file as a byte buffer.
	configBuf, err := ioutil.ReadFile(os.Args[1])
	if err != nil {
		fmt.Println("error:", err)
	}

	var configObj configType
	err = json.Unmarshal(configBuf, &configObj)
	if err != nil {
		fmt.Println("error:", err)
	}

	if configObj.Port <= 0 {
		configObj.Port = 8080
	}

	portStr := ":" + strconv.Itoa(configObj.Port)

	if configObj.SvcName != "" && configObj.PzAddr != "" {
		fmt.Println("About to manage registration.")
		err = pzsvc.ManageRegistration(	configObj.SvcName,
										configObj.Description,
										configObj.URL + "/execute",
										configObj.PzAddr,
										configObj.ImageReqs )
		if err != nil {
			fmt.Println("error:", err.Error())
		}
		fmt.Println("Registration managed.")

	}

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		r.ParseForm()
		switch r.URL.Path {
		case "/":
			fmt.Fprintf(w, "hello.")
		case "/execute":
			{

				var cmdParam string
				var inFileStr string
				var outTiffStr string
				var outTxtStr string
				var outGeoJStr string
				var usePz string

				// might be time to start looking into that "help" thing.

				if r.Method == "GET" {
					cmdParam = r.URL.Query().Get("cmd")
					inFileStr = r.URL.Query().Get("inFiles")
					outTiffStr = r.URL.Query().Get("outTiffs")
					outTxtStr = r.URL.Query().Get("outTxts")
					outGeoJStr = r.URL.Query().Get("outGeoJson")
					usePz = r.URL.Query().Get("pz")
				} else {
					cmdParam = r.FormValue("cmd")
					inFileStr = r.FormValue("inFiles")
					outTiffStr = r.FormValue("outTiffs")
					outTxtStr = r.FormValue("outTxts")
					outGeoJStr = r.FormValue("outGeoJson")
					usePz = r.FormValue("pz")
				}

				cmdConfigSlice := splitOrNil(configObj.CliCmd, " ")
				cmdParamSlice := splitOrNil(cmdParam, " ")
				cmdSlice := append(cmdConfigSlice, cmdParamSlice...)

				inFileSlice := splitOrNil(inFileStr, ",")
				outTiffSlice := splitOrNil(outTiffStr, ",")
				outTxtSlice := splitOrNil(outTxtStr, ",")
				outGeoJSlice := splitOrNil(outGeoJStr, ",")

				var output outStruct

				runID, err := psuUUID()
				if err != nil {
					addErr(&output.Error, err.Error())
				}

				err = os.Mkdir("./"+runID, 0777)
				if err != nil {
					addErr(&output.Error, err.Error())
				}
				defer os.RemoveAll("./" + runID)

				err = os.Chmod("./"+runID, 0777)
				if err != nil {
					addErr(&output.Error, err.Error())
				}

				if len(inFileSlice) > 0 {
					output.InFiles = make(map[string]string)
				}
				if len(outTiffSlice)+len(outTxtSlice)+len(outGeoJSlice) > 0 {
					output.OutFiles = make(map[string]string)
				}

				for i, inFile := range inFileSlice {

					fmt.Printf("Downloading file %s - %d of %d.\n", inFile, i, len(inFileSlice))
					fname, err := pzsvc.Download(inFile, runID, configObj.PzAddr)
					if err != nil {
						addErr(&output.Error, err.Error())
						fmt.Printf("Download failed.  %s", err.Error())
					} else {
						output.InFiles[inFile] = fname
						fmt.Printf("Successfully downloaded %s.", fname)
					}
				}

				if len(cmdSlice) == 0 {
					addErr(&output.Error, `No cmd specified in config file.  Please provide "cmd" param.`)
					break
				}

				fmt.Printf("Executing \"%s\".\n", configObj.CliCmd+" "+cmdParam)

				// we're calling this from inside a temporary subfolder.  If the
				// program called exists inside the initial pzsvc-exec folder, that's
				// probably where it's called from, and we need to acccess it directly.
				_, err = os.Stat(fmt.Sprintf("./%s", cmdSlice[0]))
				if err == nil || !(os.IsNotExist(err)){
					// ie, if there's a file in the start folder named the same thing
					// as the base command
					cmdSlice[0] = ("../" + cmdSlice[0])
				}

				clc := exec.Command(cmdSlice[0], cmdSlice[1:]...)
				clc.Dir = runID

				var b bytes.Buffer
				clc.Stdout = &b
				clc.Stderr = os.Stderr

				err = clc.Run()
				if err != nil {
					addErr(&output.Error, err.Error())
				}
				fmt.Printf("Program output: %s\n", b.String())
				output.ProgReturn = b.String()

				for i, outTiff := range outTiffSlice {
					fmt.Printf("Uploading Tiff %s - %d of %d.\n", outTiff, i, len(outTiffSlice))
					dataID, err := pzsvc.IngestTiff(outTiff, runID, configObj.PzAddr, cmdSlice[0])
					if err != nil {
						fmt.Fprintf(w, err.Error())
						fmt.Printf("Upload failed.  %s", err.Error())
					} else {
						output.OutFiles[outTiff] = dataID
					}
				}

				for i, outTxt := range outTxtSlice {
					fmt.Printf("Uploading Txt %s - %d of %d.\n", outTxt, i, len(outTxtSlice))
					dataID, err := pzsvc.IngestTxt(outTxt, runID, configObj.PzAddr, cmdSlice[0])
					if err != nil {
						addErr(&output.Error, err.Error())
						fmt.Printf("Upload failed.  %s", err.Error())
					} else {
						output.OutFiles[outTxt] = dataID
					}
				}

				for i, outGeoJ := range outGeoJSlice {
					fmt.Printf("Uploading GeoJson %s - %d of %d.\n", outGeoJ, i, len(outGeoJSlice))
					dataID, err := pzsvc.IngestGeoJson(outGeoJ, runID, configObj.PzAddr, cmdSlice[0])
					if err != nil {
						addErr(&output.Error, err.Error())
						fmt.Printf("Upload failed.  %s", err.Error())
					} else {
						output.OutFiles[outGeoJ] = dataID
					}
				}

				outBuf, err := json.Marshal(output)
				if err != nil {
					addErr(&output.Error, err.Error())
				}

				outStr := string(outBuf)

				// this isn't a thing now, but might again become a thing later.
				if usePz != "" {
					var cont pzCont
					var wrap pzWrap

					cont.Type = "text"
					cont.Content = outStr
					cont.MimeType = "text/plain"

					wrap.DataType = cont

					outBuf, err := json.Marshal(wrap)
					if err != nil {
						fmt.Fprintf(w, err.Error())
					}

					outStr = string(outBuf)
				}

				fmt.Fprintf(w, outStr)

			}
		case "/description":
			if configObj.Description == "" {
				fmt.Fprintf(w, "No description defined")
			} else {
				fmt.Fprintf(w, configObj.Description)
			}
		case "/attributes":
			if configObj.Attributes == nil {
				fmt.Fprintf(w, "{ }")
			} else {
				// convert attributes back into Json
				// this might require specifying the interface a bit better.
				//					fmt.Fprintf(w, configObj.Attributes)
			}
		case "/help":
			fmt.Fprintf(w, "We're sorry, help is not yet implemented.\n")
		default:
			fmt.Fprintf(w, "Command undefined.  Try help?\n")
		}
	})

	log.Fatal(http.ListenAndServe(portStr, nil))
}

func splitOrNil(inString, knife string) []string {
	if inString == "" {
		return nil
	}
	return strings.Split(inString, knife)
}

func psuUUID() (string, error) {

	b := make([]byte, 16)
	_, err := rand.Read(b)
	if err != nil {
		return "", err
	}

	return fmt.Sprintf("%X-%X-%X-%X-%X", b[0:4], b[4:6], b[6:8], b[8:10], b[10:]), nil
}

func addErr (thisErr *string, nextErr string) {
	if *thisErr == "" {
		*thisErr = nextErr
	} else {
		*thisErr = *thisErr + ";  " + nextErr
	}
}<commit_msg>adjusted the error output to be a sring slice, rather than a semicolon-separated list.<commit_after>// Copyright 2016, RadiantBlue Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bytes"
	"crypto/rand"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"strconv"
	"strings"

	"github.com/venicegeo/pzsvc-exec/pzsvc"
)

type configType struct {
	CliCmd      string
	PzAddr      string
	SvcName     string
	URL         string
	Port        int
	Description	string
	ImageReqs	map[string]string
	Attributes	map[string]string
}

type outStruct struct {
	InFiles		map[string]string
	OutFiles	map[string]string
	ProgReturn	string
	Errors		[]string
}

func main() {

	if len(os.Args) < 2 {
		fmt.Println("error: Insufficient parameters.  You must specify a config file.")
		return
	}

	// first argument after the base call should be the path to the config file.
	// ReadFile returns the contents of the file as a byte buffer.
	configBuf, err := ioutil.ReadFile(os.Args[1])
	if err != nil {
		fmt.Println("error:", err)
	}

	var configObj configType
	err = json.Unmarshal(configBuf, &configObj)
	if err != nil {
		fmt.Println("error:", err)
	}

	if configObj.Port <= 0 {
		configObj.Port = 8080
	}

	portStr := ":" + strconv.Itoa(configObj.Port)

	if configObj.SvcName != "" && configObj.PzAddr != "" {
		fmt.Println("About to manage registration.")
		err = pzsvc.ManageRegistration(	configObj.SvcName,
										configObj.Description,
										configObj.URL + "/execute",
										configObj.PzAddr,
										configObj.ImageReqs )
		if err != nil {
			fmt.Println("error:", err.Error())
		}
		fmt.Println("Registration managed.")

	}

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		r.ParseForm()
		switch r.URL.Path {
		case "/":
			fmt.Fprintf(w, "hello.")
		case "/execute":
			{

				var cmdParam string
				var inFileStr string
				var outTiffStr string
				var outTxtStr string
				var outGeoJStr string
				var usePz string

				// might be time to start looking into that "help" thing.

				if r.Method == "GET" {
					cmdParam = r.URL.Query().Get("cmd")
					inFileStr = r.URL.Query().Get("inFiles")
					outTiffStr = r.URL.Query().Get("outTiffs")
					outTxtStr = r.URL.Query().Get("outTxts")
					outGeoJStr = r.URL.Query().Get("outGeoJson")
					usePz = r.URL.Query().Get("pz")
				} else {
					cmdParam = r.FormValue("cmd")
					inFileStr = r.FormValue("inFiles")
					outTiffStr = r.FormValue("outTiffs")
					outTxtStr = r.FormValue("outTxts")
					outGeoJStr = r.FormValue("outGeoJson")
					usePz = r.FormValue("pz")
				}

				cmdConfigSlice := splitOrNil(configObj.CliCmd, " ")
				cmdParamSlice := splitOrNil(cmdParam, " ")
				cmdSlice := append(cmdConfigSlice, cmdParamSlice...)

				inFileSlice := splitOrNil(inFileStr, ",")
				outTiffSlice := splitOrNil(outTiffStr, ",")
				outTxtSlice := splitOrNil(outTxtStr, ",")
				outGeoJSlice := splitOrNil(outGeoJStr, ",")

				var output outStruct

				runID, err := psuUUID()
				if err != nil {
					output.Errors = append(output.Errors, err.Error())
				}

				err = os.Mkdir("./"+runID, 0777)
				if err != nil {
					output.Errors = append(output.Errors, err.Error())
				}
				defer os.RemoveAll("./" + runID)

				err = os.Chmod("./"+runID, 0777)
				if err != nil {
					output.Errors = append(output.Errors, err.Error())
				}

				if len(inFileSlice) > 0 {
					output.InFiles = make(map[string]string)
				}
				if len(outTiffSlice)+len(outTxtSlice)+len(outGeoJSlice) > 0 {
					output.OutFiles = make(map[string]string)
				}

				for i, inFile := range inFileSlice {

					fmt.Printf("Downloading file %s - %d of %d.\n", inFile, i, len(inFileSlice))
					fname, err := pzsvc.Download(inFile, runID, configObj.PzAddr)
					if err != nil {
						output.Errors = append(output.Errors,err.Error())
						fmt.Printf("Download failed.  %s", err.Error())
					} else {
						output.InFiles[inFile] = fname
						fmt.Printf("Successfully downloaded %s.", fname)
					}
				}

				if len(cmdSlice) == 0 {
					output.Errors = append(output.Errors, `No cmd specified in config file.  Please provide "cmd" param.`)
					break
				}

				fmt.Printf("Executing \"%s\".\n", configObj.CliCmd+" "+cmdParam)

				// we're calling this from inside a temporary subfolder.  If the
				// program called exists inside the initial pzsvc-exec folder, that's
				// probably where it's called from, and we need to acccess it directly.
				_, err = os.Stat(fmt.Sprintf("./%s", cmdSlice[0]))
				if err == nil || !(os.IsNotExist(err)){
					// ie, if there's a file in the start folder named the same thing
					// as the base command
					cmdSlice[0] = ("../" + cmdSlice[0])
				}

				clc := exec.Command(cmdSlice[0], cmdSlice[1:]...)
				clc.Dir = runID

				var b bytes.Buffer
				clc.Stdout = &b
				clc.Stderr = os.Stderr

				err = clc.Run()
				if err != nil {
					output.Errors = append(output.Errors, err.Error())
				}
				fmt.Printf("Program output: %s\n", b.String())
				output.ProgReturn = b.String()

				for i, outTiff := range outTiffSlice {
					fmt.Printf("Uploading Tiff %s - %d of %d.\n", outTiff, i, len(outTiffSlice))
					dataID, err := pzsvc.IngestTiff(outTiff, runID, configObj.PzAddr, cmdSlice[0])
					if err != nil {
						fmt.Fprintf(w, err.Error())
						fmt.Printf("Upload failed.  %s", err.Error())
					} else {
						output.OutFiles[outTiff] = dataID
					}
				}

				for i, outTxt := range outTxtSlice {
					fmt.Printf("Uploading Txt %s - %d of %d.\n", outTxt, i, len(outTxtSlice))
					dataID, err := pzsvc.IngestTxt(outTxt, runID, configObj.PzAddr, cmdSlice[0])
					if err != nil {
						output.Errors = append(output.Errors, err.Error())
						fmt.Printf("Upload failed.  %s", err.Error())
					} else {
						output.OutFiles[outTxt] = dataID
					}
				}

				for i, outGeoJ := range outGeoJSlice {
					fmt.Printf("Uploading GeoJson %s - %d of %d.\n", outGeoJ, i, len(outGeoJSlice))
					dataID, err := pzsvc.IngestGeoJson(outGeoJ, runID, configObj.PzAddr, cmdSlice[0])
					if err != nil {
						output.Errors = append(output.Errors, err.Error())
						fmt.Printf("Upload failed.  %s", err.Error())
					} else {
						output.OutFiles[outGeoJ] = dataID
					}
				}

				// this isn't a thing now, but might again become a thing later.
				if usePz != "" {

					type pzCont struct {
						Type		string
						Content		string
						MimeType	string
					}

					type pzWrap struct {
						DataType	pzCont
						metadata	map[string]string
					}

					var cont pzCont
					var wrap pzWrap
					outBuf, err := json.Marshal(output)
					if err != nil {
						output.Errors = append(output.Errors, err.Error())
					}

					cont.Type = "text"
					cont.Content = string(outBuf)
					cont.MimeType = "text/plain"

					wrap.DataType = cont

					printJson(wrap, w)
				} else {
					printJson(output, w)
				}

			}
		case "/description":
			if configObj.Description == "" {
				fmt.Fprintf(w, "No description defined")
			} else {
				fmt.Fprintf(w, configObj.Description)
			}
		case "/attributes":
			if configObj.Attributes == nil {
				fmt.Fprintf(w, "{ }")
			} else {
				// convert attributes back into Json
				// this might require specifying the interface a bit better.
				//					fmt.Fprintf(w, configObj.Attributes)
			}
		case "/help":
			fmt.Fprintf(w, "We're sorry, help is not yet implemented.\n")
		default:
			fmt.Fprintf(w, "Command undefined.  Try help?\n")
		}
	})

	log.Fatal(http.ListenAndServe(portStr, nil))
}

func splitOrNil(inString, knife string) []string {
	if inString == "" {
		return nil
	}
	return strings.Split(inString, knife)
}

func psuUUID() (string, error) {

	b := make([]byte, 16)
	_, err := rand.Read(b)
	if err != nil {
		return "", err
	}

	return fmt.Sprintf("%X-%X-%X-%X-%X", b[0:4], b[4:6], b[6:8], b[8:10], b[10:]), nil
}

func printJson (output interface{}, w http.ResponseWriter) {

				outBuf, err := json.Marshal(output)
				if err != nil {
					fmt.Fprintf(w, `{"Errors":"Json marshalling failure.  Data not reportable."}`)
				}

				outStr := string(outBuf)
				fmt.Fprintf(w, outStr)
}<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"html/template"
	"net/http"
	"os"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/anthonynsimon/parrot/api"
	"github.com/anthonynsimon/parrot/datastore"
	"github.com/anthonynsimon/parrot/logger"
	"github.com/anthonynsimon/parrot/render"
	"github.com/anthonynsimon/parrot/web"
	"github.com/joho/godotenv"
	"github.com/pressly/chi"
	"github.com/pressly/chi/middleware"
)

func init() {
	// Config log
	log.SetFormatter(&log.TextFormatter{})
}

func main() {
	// init environment variables
	err := godotenv.Load()
	if err != nil {
		log.Fatal(err)
	}

	// init and ping datastore
	dbName := os.Getenv("DB")
	dbURL := os.Getenv("DB_URL")

	ds, err := datastore.NewDatastore(dbName, dbURL)
	if err != nil {
		log.Fatal(err)
	}

	defer ds.Close()
	if err = ds.Ping(); err != nil {
		log.Fatal(err)
	}

	// init routers
	mainRouter := chi.NewRouter()
	mainRouter.Use(logger.Request)
	mainRouter.Use(middleware.StripSlashes)

	// mainRouter.Use(logger.Request) // TODO convert to http.Handler
	apiRouter := api.NewRouter(ds, []byte(os.Getenv("API_SIGNING_KEY")))
	mainRouter.Mount("/api", apiRouter)

	// init and ping api backend
	apiUrl := os.Getenv("API_URL")
	backend, err := datastore.NewDatastore("apiClient", apiUrl)
	if err != nil {
		log.Fatal(err)
	}
	defer ds.Close()
	if err = ds.Ping(); err != nil {
		log.Fatal(err)
	}

	web.Register(mainRouter, backend)

	// parse view templates
	render.Templates = template.Must(template.ParseGlob("./views/**/*.html"))

	// config server
	addr := "localhost:8080"

	// init server
	s := &http.Server{
		Addr:           addr,
		Handler:        mainRouter,
		ReadTimeout:    10 * time.Second,
		WriteTimeout:   10 * time.Second,
		MaxHeaderBytes: 1 << 20,
	}

	fmt.Println(fmt.Sprintf("Listening on %s", addr))

	log.Fatal(s.ListenAndServe())
}
<commit_msg>Conform var format<commit_after>package main

import (
	"fmt"
	"html/template"
	"net/http"
	"os"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/anthonynsimon/parrot/api"
	"github.com/anthonynsimon/parrot/datastore"
	"github.com/anthonynsimon/parrot/logger"
	"github.com/anthonynsimon/parrot/render"
	"github.com/anthonynsimon/parrot/web"
	"github.com/joho/godotenv"
	"github.com/pressly/chi"
	"github.com/pressly/chi/middleware"
)

func init() {
	// Config log
	log.SetFormatter(&log.TextFormatter{})
}

func main() {
	// init environment variables
	err := godotenv.Load()
	if err != nil {
		log.Fatal(err)
	}

	// init and ping datastore
	dbName := os.Getenv("DB")
	dbURL := os.Getenv("DB_URL")

	ds, err := datastore.NewDatastore(dbName, dbURL)
	if err != nil {
		log.Fatal(err)
	}

	defer ds.Close()
	if err = ds.Ping(); err != nil {
		log.Fatal(err)
	}

	// init routers
	mainRouter := chi.NewRouter()
	mainRouter.Use(logger.Request)
	mainRouter.Use(middleware.StripSlashes)

	// mainRouter.Use(logger.Request) // TODO convert to http.Handler
	apiRouter := api.NewRouter(ds, []byte(os.Getenv("API_SIGNING_KEY")))
	mainRouter.Mount("/api", apiRouter)

	// init and ping api backend
	apiURL := os.Getenv("API_URL")
	backend, err := datastore.NewDatastore("apiClient", apiURL)
	if err != nil {
		log.Fatal(err)
	}
	defer ds.Close()
	if err = ds.Ping(); err != nil {
		log.Fatal(err)
	}

	web.Register(mainRouter, backend)

	// parse view templates
	render.Templates = template.Must(template.ParseGlob("./views/**/*.html"))

	// config server
	addr := "localhost:8080"

	// init server
	s := &http.Server{
		Addr:           addr,
		Handler:        mainRouter,
		ReadTimeout:    10 * time.Second,
		WriteTimeout:   10 * time.Second,
		MaxHeaderBytes: 1 << 20,
	}

	fmt.Println(fmt.Sprintf("Listening on %s", addr))

	log.Fatal(s.ListenAndServe())
}
<|endoftext|>


<commit_before>package main

import (
	"os"

	"github.com/op/go-logging"
	"github.com/urfave/cli"
)

// logger
var log *logging.Logger

func init() {
	log = logging.MustGetLogger("hod")
	var format = "%{color}%{level} %{shortfile} %{time:Jan 02 15:04:05} %{color:reset} ▶ %{message}"
	var logBackend = logging.NewLogBackend(os.Stderr, "", 0)
	logBackendLeveled := logging.AddModuleLevel(logBackend)
	logging.SetBackend(logBackendLeveled)
	logging.SetFormatter(logging.MustStringFormatter(format))
}

func main() {
	app := cli.NewApp()
	app.Name = "hod"
	app.Version = "0.4.4"
	app.Usage = "BRICK database and query engine"

	app.Commands = []cli.Command{
		{
			Name:   "load",
			Usage:  "Load dataset into hoddb",
			Action: load,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "config, c",
					Usage: "Path to hoddb config file",
				},
			},
		},
		{
			Name:   "cli",
			Usage:  "Start hoddb from existing database",
			Action: startCLI,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "config, c",
					Usage: "Path to hoddb config file",
				},
			},
		},
		{
			Name:   "server",
			Usage:  "Start hoddb server from existing database. Default to HTTP server only, but can do both that and BOSSWAVE",
			Action: startServer,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "config, c",
					Usage: "Path to hoddb config file",
				},
			},
		},
		{
			Name:   "query",
			Usage:  "Query from command line (non-interactive)",
			Action: doQuery,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "file, f",
					Usage: "Path to hoddb query file",
				},
				cli.StringFlag{
					Name:  "config, c",
					Usage: "Path to hoddb config file",
				},
				cli.StringFlag{
					Name:  "query, q",
					Usage: "Query string",
				},
				cli.BoolFlag{
					Name:  "prefixes, p",
					Usage: "If true, abbreviate all namespaces. Else, just print the full URI",
				},
			},
		},
		{
			Name:   "search",
			Usage:  "Query for triples using just text matching",
			Action: doSearch,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "config, c",
					Usage: "Path to hoddb config file",
				},
				cli.StringFlag{
					Name:  "query, q",
					Usage: "Query string",
				},
				cli.IntFlag{
					Name:  "number, n",
					Usage: "Number of results to return",
					Value: 20,
				},
			},
		},
		{
			Name:   "benchload",
			Usage:  "Benchmark loading a turtle file",
			Action: benchLoad,
		},
		{
			Name:   "dump",
			Usage:  "Dump contents of turtle file",
			Action: dump,
		},
		{
			Name:   "viewclass",
			Usage:  "PDF visualization of class structure of file",
			Action: classGraph,
		},
		{
			Name:   "dumpgraph",
			Usage:  "PDF visualization of TTL file. WARNING this can get really big",
			Action: dumpGraph,
		},
		{
			Name:   "ttlstat",
			Usage:  "Outputs statistics on the provided TTL file. Loads all file provided as arguments",
			Action: ttlStat,
		},
	}
	app.Run(os.Args)
}
<commit_msg>make sure we annotate as v0.5.0<commit_after>package main

import (
	"os"

	"github.com/op/go-logging"
	"github.com/urfave/cli"
)

// logger
var log *logging.Logger

func init() {
	log = logging.MustGetLogger("hod")
	var format = "%{color}%{level} %{shortfile} %{time:Jan 02 15:04:05} %{color:reset} ▶ %{message}"
	var logBackend = logging.NewLogBackend(os.Stderr, "", 0)
	logBackendLeveled := logging.AddModuleLevel(logBackend)
	logging.SetBackend(logBackendLeveled)
	logging.SetFormatter(logging.MustStringFormatter(format))
}

func main() {
	app := cli.NewApp()
	app.Name = "hod"
	app.Version = "0.5.0"
	app.Usage = "BRICK database and query engine"

	app.Commands = []cli.Command{
		{
			Name:   "load",
			Usage:  "Load dataset into hoddb",
			Action: load,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "config, c",
					Usage: "Path to hoddb config file",
				},
			},
		},
		{
			Name:   "cli",
			Usage:  "Start hoddb from existing database",
			Action: startCLI,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "config, c",
					Usage: "Path to hoddb config file",
				},
			},
		},
		{
			Name:   "server",
			Usage:  "Start hoddb server from existing database. Default to HTTP server only, but can do both that and BOSSWAVE",
			Action: startServer,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "config, c",
					Usage: "Path to hoddb config file",
				},
			},
		},
		{
			Name:   "query",
			Usage:  "Query from command line (non-interactive)",
			Action: doQuery,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "file, f",
					Usage: "Path to hoddb query file",
				},
				cli.StringFlag{
					Name:  "config, c",
					Usage: "Path to hoddb config file",
				},
				cli.StringFlag{
					Name:  "query, q",
					Usage: "Query string",
				},
				cli.BoolFlag{
					Name:  "prefixes, p",
					Usage: "If true, abbreviate all namespaces. Else, just print the full URI",
				},
			},
		},
		{
			Name:   "search",
			Usage:  "Query for triples using just text matching",
			Action: doSearch,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "config, c",
					Usage: "Path to hoddb config file",
				},
				cli.StringFlag{
					Name:  "query, q",
					Usage: "Query string",
				},
				cli.IntFlag{
					Name:  "number, n",
					Usage: "Number of results to return",
					Value: 20,
				},
			},
		},
		{
			Name:   "benchload",
			Usage:  "Benchmark loading a turtle file",
			Action: benchLoad,
		},
		{
			Name:   "dump",
			Usage:  "Dump contents of turtle file",
			Action: dump,
		},
		{
			Name:   "viewclass",
			Usage:  "PDF visualization of class structure of file",
			Action: classGraph,
		},
		{
			Name:   "dumpgraph",
			Usage:  "PDF visualization of TTL file. WARNING this can get really big",
			Action: dumpGraph,
		},
		{
			Name:   "ttlstat",
			Usage:  "Outputs statistics on the provided TTL file. Loads all file provided as arguments",
			Action: ttlStat,
		},
	}
	app.Run(os.Args)
}
<|endoftext|>


<commit_before>// Workflow written in SciPipe.
// For more information about SciPipe, see: http://scipipe.org
package main

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	str "strings"

	"github.com/flowbase/flowbase"
	"github.com/knakk/rdf"
)

const (
	BUFSIZE = 16
)

func main() {
	flowbase.InitLogInfo()

	inFileName := flag.String("infile", "", "The input file name")
	flag.Parse()
	if *inFileName == "" {
		fmt.Println("No filename specified to --infile")
		os.Exit(1)
	}

	// Create a pipeline runner
	pipeRunner := flowbase.NewPipelineRunner()

	// Initialize processes and add to runner
	fileReader := NewFileReader()
	pipeRunner.AddProcess(fileReader)

	//tripleParser := NewTripleParser()
	//pipeRunner.AddProcess(tripleParser)

	tripleParser := NewTripleParser()
	pipeRunner.AddProcess(tripleParser)

	tripleAggregator := NewAggregateTriplesPerSubject()
	pipeRunner.AddProcess(tripleAggregator)

	tripleAggregatePrinter := NewTripleAggregatePrinter()
	pipeRunner.AddProcess(tripleAggregatePrinter)

	// Connect workflow dependency network
	tripleParser.In = fileReader.OutLine
	tripleAggregator.In = tripleParser.Out
	tripleAggregatePrinter.In = tripleAggregator.Out

	// Run the pipeline!
	go func() {
		defer close(fileReader.InFileName)
		fileReader.InFileName <- *inFileName
	}()

	pipeRunner.Run()

}

// ================================================================================
// Components
// ================================================================================

// --------------------------------------------------------------------------------
// FileReader
// --------------------------------------------------------------------------------

type FileReader struct {
	InFileName chan string
	OutLine    chan string
}

func NewFileReader() *FileReader {
	return &FileReader{
		InFileName: make(chan string, BUFSIZE),
		OutLine:    make(chan string, BUFSIZE),
	}
}

func (p *FileReader) Run() {
	defer close(p.OutLine)

	flowbase.Debug.Println("Starting loop")
	for fileName := range p.InFileName {
		flowbase.Debug.Printf("Starting processing file %s\n", fileName)
		fh, err := os.Open(fileName)
		if err != nil {
			log.Fatal(err)
		}
		defer fh.Close()

		sc := bufio.NewScanner(fh)
		for sc.Scan() {
			if err := sc.Err(); err != nil {
				log.Fatal(err)
			}
			p.OutLine <- sc.Text()
		}
	}
}

// --------------------------------------------------------------------------------
// TripleParser
// --------------------------------------------------------------------------------

type TripleParser struct {
	In  chan string
	Out chan rdf.Triple
}

func NewTripleParser() *TripleParser {
	return &TripleParser{
		In:  make(chan string, BUFSIZE),
		Out: make(chan rdf.Triple, BUFSIZE),
	}
}

func (p *TripleParser) Run() {
	defer close(p.Out)
	for line := range p.In {
		lineReader := str.NewReader(line)
		dec := rdf.NewTripleDecoder(lineReader, rdf.Turtle)
		for triple, err := dec.Decode(); err != io.EOF; triple, err = dec.Decode() {
			p.Out <- triple
		}
	}
}

// --------------------------------------------------------------------------------
// AggregateTriplesPerSubject
// --------------------------------------------------------------------------------

type AggregateTriplesPerSubject struct {
	In  chan rdf.Triple
	Out chan *TripleAggregate
}

func NewAggregateTriplesPerSubject() *AggregateTriplesPerSubject {
	return &AggregateTriplesPerSubject{
		In:  make(chan rdf.Triple, BUFSIZE),
		Out: make(chan *TripleAggregate, BUFSIZE),
	}
}

func (p *AggregateTriplesPerSubject) Run() {
	defer close(p.Out)
	tripleIndex := make(map[rdf.Subject][]rdf.Triple)
	for triple := range p.In {
		tripleIndex[triple.Subj] = append(tripleIndex[triple.Subj], triple)
	}
	for subj, triples := range tripleIndex {
		tripleAggregate := NewTripleAggregate(subj, triples)
		p.Out <- tripleAggregate
	}
}

// --------------------------------------------------------------------------------
// TriplePrinter
// --------------------------------------------------------------------------------

type TriplePrinter struct {
	In chan rdf.Triple
}

func NewTriplePrinter() *TriplePrinter {
	return &TriplePrinter{
		In: make(chan rdf.Triple, BUFSIZE),
	}
}

func (p *TriplePrinter) Run() {
	for tr := range p.In {
		fmt.Printf("S: %s\nP: %s\nO: %s\n\n", tr.Subj.String(), tr.Pred.String(), tr.Obj.String())
	}
}

// --------------------------------------------------------------------------------
// TripleAggregatePrinter
// --------------------------------------------------------------------------------

type TripleAggregatePrinter struct {
	In chan *TripleAggregate
}

func NewTripleAggregatePrinter() *TripleAggregatePrinter {
	return &TripleAggregatePrinter{
		In: make(chan *TripleAggregate, BUFSIZE),
	}
}

func (p *TripleAggregatePrinter) Run() {
	for trAggr := range p.In {
		fmt.Printf("Subject: %s\nTriples:\n", trAggr.Subject)
		for _, tr := range trAggr.Triples {
			fmt.Printf("\t<%s> <%s> <%s>\n", tr.Subj.String(), tr.Pred.String(), tr.Obj.String())
		}
		fmt.Println()
	}
}

// --------------------------------------------------------------------------------
// IP: RDFTriple
// --------------------------------------------------------------------------------
//
//type RDFTriple struct {
//	Subject   string
//	Predicate string
//	Object    string
//}
//
//func NewRDFTriple() *RDFTriple {
//	return &RDFTriple{}
//}

// --------------------------------------------------------------------------------
// IP: TripleAggregate
// --------------------------------------------------------------------------------

type TripleAggregate struct {
	Subject rdf.Subject
	Triples []rdf.Triple
}

func NewTripleAggregate(subj rdf.Subject, triples []rdf.Triple) *TripleAggregate {
	return &TripleAggregate{
		Subject: subj,
		Triples: triples,
	}
}
<commit_msg>Implement Triple Aggregate to Wiki Page converter<commit_after>// Workflow written in SciPipe.
// For more information about SciPipe, see: http://scipipe.org
package main

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	str "strings"

	"github.com/flowbase/flowbase"
	"github.com/knakk/rdf"
)

const (
	BUFSIZE = 16
)

func main() {
	flowbase.InitLogInfo()

	inFileName := flag.String("infile", "", "The input file name")
	flag.Parse()
	if *inFileName == "" {
		fmt.Println("No filename specified to --infile")
		os.Exit(1)
	}

	// ------------------------------------------
	// Initialize processes
	// ------------------------------------------

	// Create a pipeline runner
	pipeRunner := flowbase.NewPipelineRunner()

	// Read in-file
	fileRead := NewFileReader()
	pipeRunner.AddProcess(fileRead)

	// Parse triples
	parser := NewTripleParser()
	pipeRunner.AddProcess(parser)

	// Aggregate per subject
	aggregator := NewAggregateTriplesPerSubject()
	pipeRunner.AddProcess(aggregator)

	// Create an subject-indexed "index" of all triples
	indexCreator := NewCreateTripleIndex()
	pipeRunner.AddProcess(indexCreator)

	// Fan-out the triple index to the converter and serializer
	indexFanOut := NewTripleIndexFanOut()
	pipeRunner.AddProcess(indexFanOut)

	// Serialize the index back to individual subject-tripleaggregates
	indexToAggr := NewTripleIndexToTripleAggregates()
	pipeRunner.AddProcess(indexToAggr)

	// Convert TripleAggregate to WikiPage
	triplesToWikiConverter := NewTripleAggregateToWikiPageConverter()
	pipeRunner.AddProcess(triplesToWikiConverter)

	// Pretty-print wiki page data
	wikiPagePrinter := NewWikiPagePrinter()
	pipeRunner.AddProcess(wikiPagePrinter)

	// ------------------------------------------
	// Connect network
	// ------------------------------------------

	fileRead.OutLine = parser.In
	parser.Out = aggregator.In

	aggregator.Out = indexCreator.In

	indexCreator.Out = indexFanOut.In
	indexFanOut.Out["serialize"] = indexToAggr.In
	indexFanOut.Out["conv"] = triplesToWikiConverter.InIndex

	indexToAggr.Out = triplesToWikiConverter.InAggregate

	triplesToWikiConverter.OutPage = wikiPagePrinter.In

	// ------------------------------------------
	// Send in-data and run
	// ------------------------------------------

	go func() {
		defer close(fileRead.InFileName)
		fileRead.InFileName <- *inFileName
	}()

	pipeRunner.Run()

}

// ================================================================================
// Components
// ================================================================================

// --------------------------------------------------------------------------------
// FileReader
// --------------------------------------------------------------------------------

type FileReader struct {
	InFileName chan string
	OutLine    chan string
}

func NewFileReader() *FileReader {
	return &FileReader{
		InFileName: make(chan string, BUFSIZE),
		OutLine:    make(chan string, BUFSIZE),
	}
}

func (p *FileReader) Run() {
	defer close(p.OutLine)

	flowbase.Debug.Println("Starting loop")
	for fileName := range p.InFileName {
		flowbase.Debug.Printf("Starting processing file %s\n", fileName)
		fh, err := os.Open(fileName)
		if err != nil {
			log.Fatal(err)
		}
		defer fh.Close()

		sc := bufio.NewScanner(fh)
		for sc.Scan() {
			if err := sc.Err(); err != nil {
				log.Fatal(err)
			}
			p.OutLine <- sc.Text()
		}
	}
}

// --------------------------------------------------------------------------------
// TripleParser
// --------------------------------------------------------------------------------

type TripleParser struct {
	In  chan string
	Out chan rdf.Triple
}

func NewTripleParser() *TripleParser {
	return &TripleParser{
		In:  make(chan string, BUFSIZE),
		Out: make(chan rdf.Triple, BUFSIZE),
	}
}

func (p *TripleParser) Run() {
	defer close(p.Out)
	for line := range p.In {
		lineReader := str.NewReader(line)
		dec := rdf.NewTripleDecoder(lineReader, rdf.Turtle)
		for triple, err := dec.Decode(); err != io.EOF; triple, err = dec.Decode() {
			p.Out <- triple
		}
	}
}

// --------------------------------------------------------------------------------
// AggregateTriplesPerSubject
// --------------------------------------------------------------------------------

type AggregateTriplesPerSubject struct {
	In  chan rdf.Triple
	Out chan *TripleAggregate
}

func NewAggregateTriplesPerSubject() *AggregateTriplesPerSubject {
	return &AggregateTriplesPerSubject{
		In:  make(chan rdf.Triple, BUFSIZE),
		Out: make(chan *TripleAggregate, BUFSIZE),
	}
}

func (p *AggregateTriplesPerSubject) Run() {
	defer close(p.Out)
	tripleIndex := make(map[rdf.Subject][]rdf.Triple)
	for triple := range p.In {
		tripleIndex[triple.Subj] = append(tripleIndex[triple.Subj], triple)
	}
	for subj, triples := range tripleIndex {
		tripleAggregate := NewTripleAggregate(subj, triples)
		p.Out <- tripleAggregate
	}
}

type FanOutTripleAggregate struct {
	In  chan *TripleAggregate
	Out map[string](chan *TripleAggregate)
}

// NewFanOut creates a new FanOut process
func NewFanOutTripleAggregate() *FanOutTripleAggregate {
	return &FanOutTripleAggregate{
		In:  make(chan *TripleAggregate, BUFSIZE),
		Out: make(map[string](chan *TripleAggregate)),
	}
}

// Run runs the FanOut process
func (proc *FanOutTripleAggregate) Run() {
	for _, outPort := range proc.Out {
		defer close(outPort)
	}

	for ft := range proc.In {
		for _, outPort := range proc.Out {
			outPort <- ft
		}
	}
}

// --------------------------------------------------------------------------------
// Create Triple Index
// --------------------------------------------------------------------------------

type CreateTripleIndex struct {
	In  chan *TripleAggregate
	Out chan map[string]*TripleAggregate
}

func NewCreateTripleIndex() *CreateTripleIndex {
	return &CreateTripleIndex{
		In:  make(chan *TripleAggregate, BUFSIZE),
		Out: make(chan map[string]*TripleAggregate),
	}
}

func (p *CreateTripleIndex) Run() {
	defer close(p.Out)

	idx := make(map[string]*TripleAggregate)
	for aggr := range p.In {
		idx[aggr.SubjectStr] = aggr
	}

	p.Out <- idx
}

// --------------------------------------------------------------------------------
// Triple Index FanOut
// --------------------------------------------------------------------------------

type TripleIndexFanOut struct {
	In  chan map[string]*TripleAggregate
	Out map[string]chan map[string]*TripleAggregate
}

func NewTripleIndexFanOut() *TripleIndexFanOut {
	return &TripleIndexFanOut{
		In:  make(chan map[string]*TripleAggregate),
		Out: make(map[string]chan map[string]*TripleAggregate),
	}
}

func (p *TripleIndexFanOut) Run() {
	for _, outPort := range p.Out {
		defer close(outPort)
	}

	for idx := range p.In {
		for _, outPort := range p.Out {
			outPort <- idx
		}
	}
}

// --------------------------------------------------------------------------------
// Triple Index To Triple Aggregates
// --------------------------------------------------------------------------------

type TripleIndexToTripleAggregates struct {
	In  chan map[string]*TripleAggregate
	Out chan *TripleAggregate
}

func NewTripleIndexToTripleAggregates() *TripleIndexToTripleAggregates {
	return &TripleIndexToTripleAggregates{
		In:  make(chan map[string]*TripleAggregate, BUFSIZE),
		Out: make(chan *TripleAggregate, BUFSIZE),
	}
}

func (p *TripleIndexToTripleAggregates) Run() {
	defer close(p.Out)

	for idx := range p.In {
		for _, aggr := range idx {
			p.Out <- aggr
		}
	}
}

// --------------------------------------------------------------------------------
// TripleAggregateToWikiPageConverter
// --------------------------------------------------------------------------------

var titleProperties = []string{
	"http://semantic-mediawiki.org/swivt/1.0#page",
	"http://www.w3.org/2000/01/rdf-schema#label",
	"http://purl.org/dc/elements/1.1/title",
	"http://www.w3.org/2004/02/skos/core#preferredLabel",
	"http://xmlns.com/foaf/0.1/name",
}

var namespaceAbbreviations = map[string]string{
	"http://www.opentox.org/api/1.1#": "opentox",
}

type TripleAggregateToWikiPageConverter struct {
	InAggregate chan *TripleAggregate
	InIndex     chan map[string]*TripleAggregate
	OutPage     chan *WikiPage
}

func NewTripleAggregateToWikiPageConverter() *TripleAggregateToWikiPageConverter {
	return &TripleAggregateToWikiPageConverter{
		InAggregate: make(chan *TripleAggregate, BUFSIZE),
		InIndex:     make(chan map[string]*TripleAggregate, BUFSIZE),
		OutPage:     make(chan *WikiPage, BUFSIZE),
	}
}

func (p *TripleAggregateToWikiPageConverter) Run() {
	defer close(p.OutPage)
	tripleIndex := <-p.InIndex
	for aggr := range p.InAggregate {
		pageTitle, _ := p.convertUriToWikiTitle(aggr.SubjectStr, false, tripleIndex)

		page := NewWikiPage(pageTitle, []*Fact{})
		for _, tr := range aggr.Triples {
			fact := NewFact(tr.Pred.String(), tr.Obj.String())
			page.AddFact(fact)
		}
		p.OutPage <- page
	}
}

// For properties, the factTitle and pageTitle will be different (The page
// title including the "Property:" prefix), while for normal pages, they will
// be the same.
func (p *TripleAggregateToWikiPageConverter) convertUriToWikiTitle(uri string,
	isProperty bool, tripleIndex map[string]*TripleAggregate) (pageTitle string, factTitle string) {

	aggr := tripleIndex[uri]

	// Conversion strategies:
	// 1. Existing wiki title (in wiki, or cache)
	// 2. Use configured title-deciding properties
	for _, titleProp := range titleProperties {
		for _, tr := range aggr.Triples {
			if tr.Pred.String() == titleProp {
				factTitle = tr.Obj.String()
			}
		}
	}

	// 3. Shorten URI namespace to alias (e.g. http://purl.org/dc -> dc:)
	//    (Does this apply for properties only?)

	// 4. Remove namespace, keep only local part of URL (Split on '/' or '#')
	if factTitle == "" {
		bits := str.Split(uri, "#")
		lastBit := bits[len(bits)-1]
		bits = str.Split(lastBit, "/")
		lastBit = bits[len(bits)-1]
		factTitle = lastBit
	}

	if isProperty {
		pageTitle = "Property:" + factTitle
	} else {
		pageTitle = factTitle
	}

	return pageTitle, factTitle
}

// --------------------------------------------------------------------------------
// TriplePrinter
// --------------------------------------------------------------------------------

type TriplePrinter struct {
	In chan rdf.Triple
}

func NewTriplePrinter() *TriplePrinter {
	return &TriplePrinter{
		In: make(chan rdf.Triple, BUFSIZE),
	}
}

func (p *TriplePrinter) Run() {
	for tr := range p.In {
		fmt.Printf("S: %s\nP: %s\nO: %s\n\n", tr.Subj.String(), tr.Pred.String(), tr.Obj.String())
	}
}

// --------------------------------------------------------------------------------
// TripleAggregatePrinter
// --------------------------------------------------------------------------------

type TripleAggregatePrinter struct {
	In chan *TripleAggregate
}

func NewTripleAggregatePrinter() *TripleAggregatePrinter {
	return &TripleAggregatePrinter{
		In: make(chan *TripleAggregate, BUFSIZE),
	}
}

func (p *TripleAggregatePrinter) Run() {
	for trAggr := range p.In {
		fmt.Printf("Subject: %s\nTriples:\n", trAggr.Subject)
		for _, tr := range trAggr.Triples {
			fmt.Printf("\t<%s> <%s> <%s>\n", tr.Subj.String(), tr.Pred.String(), tr.Obj.String())
		}
		fmt.Println()
	}
}

// --------------------------------------------------------------------------------
// WikiPagePrinter
// --------------------------------------------------------------------------------

type WikiPagePrinter struct {
	In chan *WikiPage
}

func NewWikiPagePrinter() *WikiPagePrinter {
	return &WikiPagePrinter{
		In: make(chan *WikiPage, flowbase.BUFSIZE),
	}
}

func (p *WikiPagePrinter) Run() {
	for page := range p.In {
		fmt.Println("Title: ", page.Title)
		for _, fact := range page.Facts {
			fmt.Printf("[[%s::%s]]\n", fact.Property, fact.Value)
		}
		fmt.Println("") // Print an empty line
	}
}

// --------------------------------------------------------------------------------
// IP: RDFTriple
// --------------------------------------------------------------------------------
//
//type RDFTriple struct {
//	Subject   string
//	Predicate string
//	Object    string
//}
//
//func NewRDFTriple() *RDFTriple {
//	return &RDFTriple{}
//}

// --------------------------------------------------------------------------------
// IP: TripleAggregate
// --------------------------------------------------------------------------------

type TripleAggregate struct {
	Subject    rdf.Subject
	SubjectStr string
	Triples    []rdf.Triple
}

func NewTripleAggregate(subj rdf.Subject, triples []rdf.Triple) *TripleAggregate {
	return &TripleAggregate{
		Subject:    subj,
		SubjectStr: subj.String(),
		Triples:    triples,
	}
}

// --------------------------------------------------------------------------------
// IP: WikiPage
// --------------------------------------------------------------------------------

type WikiPage struct {
	Title string
	Facts []*Fact
}

func NewWikiPage(title string, facts []*Fact) *WikiPage {
	return &WikiPage{
		Title: title,
		Facts: facts,
	}
}

func (p *WikiPage) AddFact(fact *Fact) {
	p.Facts = append(p.Facts, fact)
}

// Helper type: Fact

type Fact struct {
	Property string
	Value    string
}

func NewFact(property string, value string) *Fact {
	return &Fact{
		Property: property,
		Value:    value,
	}
}
<|endoftext|>


<commit_before>// Main entry point for Rancher.
//
//
//
// Notes on code in Main: some of the code in runMain is copied from the copy-
// right holder, Mitchell Hashimoto (github.com/mitchellh), as I am using his
// cli package.
package main

import (
	"fmt"
	"os"
	"time"

	log "github.com/cihub/seelog"
	"github.com/mitchellh/cli"
	"github.com/mohae/rancher/ranchr"
)

var Logger log.LoggerInterface

func init() {
	// Set by default
}

func main() {
	// main wraps runMain() and ensures that the log gets flushed prior to exit.
	// Exit with return code from runMain()
	defer ranchr.FlushLog()
	defer log.Flush()
	rc := runMain()
	// wait for things to catch up because some log output seems to be missing
	// dev only? TODO hmmm, this helped me discover I've wasted a lot of time
	// and explains the craziness >.>
	os.Exit(rc)
}

func runMain() int {
	// runMain parses the Flag for glog, sets up CLI stuff for the supported sub-
	// commands and runs Rancher.
	var err error
	if err = ranchr.SetEnv(); err != nil {
		fmt.Println("An error while processing Rancher ranchr.Environment variables: ", err.Error())
		return -1
	}
	if err = SetLogging(); err != nil {
		fmt.Println("An error occurred while configuring application logging: ", err.Error())
		return -1
	}

	log.Info("Rancher starting with args: %v", os.Args[:])
	args := os.Args[1:]
	// Get the command line args. We shortcut "--version" and "-v" to
	// just show the version.
	for _, arg := range args {
		if arg == "-v" || arg == "--version" {
			newArgs := make([]string, len(args)+1)
			newArgs[0] = "version"
			copy(newArgs[1:], args)
			args = newArgs
			break
		}
	}
	cli := &cli.CLI{
		Args:     args,
		Commands: Commands,
		HelpFunc: cli.BasicHelpFunc("rancher"),
	}
	exitCode, err := cli.Run()
	if err != nil {
		log.Error("Rancher encountered an error: %s\n", err.Error())
		fmt.Println("Rancher encountered an error: %s\n", err.Error())
	}
	time.Sleep(time.Millisecond * 10000)
	log.Info("Rancher exiting with an exit code of %v", exitCode)
	return exitCode
}

func SetLogging() error {
	logger, err := log.LoggerFromConfigAsFile("seelog.xml")
	if err != nil {
		return err
	}
	ranchr.UseLogger(logger)
	log.ReplaceLogger(logger)
	return nil
}
<commit_msg>fixed fmt formatting issue in main<commit_after>// Main entry point for Rancher.
//
//
//
// Notes on code in Main: some of the code in runMain is copied from the copy-
// right holder, Mitchell Hashimoto (github.com/mitchellh), as I am using his
// cli package.
package main

import (
	"fmt"
	"os"
	"time"

	log "github.com/cihub/seelog"
	"github.com/mitchellh/cli"
	"github.com/mohae/rancher/ranchr"
)

var Logger log.LoggerInterface

func init() {
	// Set by default
}

func main() {
	// main wraps runMain() and ensures that the log gets flushed prior to exit.
	// Exit with return code from runMain()
	defer ranchr.FlushLog()
	defer log.Flush()
	rc := runMain()
	// wait for things to catch up because some log output seems to be missing
	// dev only? TODO hmmm, this helped me discover I've wasted a lot of time
	// and explains the craziness >.>
	os.Exit(rc)
}

func runMain() int {
	// runMain parses the Flag for glog, sets up CLI stuff for the supported sub-
	// commands and runs Rancher.
	var err error
	if err = ranchr.SetEnv(); err != nil {
		fmt.Println("An error while processing Rancher ranchr.Environment variables: ", err.Error())
		return -1
	}
	if err = SetLogging(); err != nil {
		fmt.Println("An error occurred while configuring application logging: ", err.Error())
		return -1
	}

	log.Info("Rancher starting with args: %v", os.Args[:])
	args := os.Args[1:]
	// Get the command line args. We shortcut "--version" and "-v" to
	// just show the version.
	for _, arg := range args {
		if arg == "-v" || arg == "--version" {
			newArgs := make([]string, len(args)+1)
			newArgs[0] = "version"
			copy(newArgs[1:], args)
			args = newArgs
			break
		}
	}
	cli := &cli.CLI{
		Args:     args,
		Commands: Commands,
		HelpFunc: cli.BasicHelpFunc("rancher"),
	}
	exitCode, err := cli.Run()
	if err != nil {
		log.Errorf("Rancher encountered an error: %s\n", err.Error())
		fmt.Printf("Rancher encountered an error: %s\n", err.Error())
	}
	time.Sleep(time.Millisecond * 10000)
	log.Info("Rancher exiting with an exit code of %v", exitCode)
	return exitCode
}

func SetLogging() error {
	logger, err := log.LoggerFromConfigAsFile("seelog.xml")
	if err != nil {
		return err
	}
	ranchr.UseLogger(logger)
	log.ReplaceLogger(logger)
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"github.com/globocom/tsuru/cmd"
	"github.com/globocom/tsuru/cmd/tsuru-base"
	"os"
)

const (
	version = "0.8.6.1"
	header  = "Supported-Tsuru"
)

func buildManager(name string) *cmd.Manager {
	m := cmd.BuildBaseManager(name, version, header)
	m.Register(&tsuru.AppRun{})
	m.Register(&tsuru.AppInfo{})
	m.Register(AppCreate{})
	m.Register(&AppRemove{})
	m.Register(&UnitAdd{})
	m.Register(&UnitRemove{})
	m.Register(tsuru.AppList{})
	m.Register(&tsuru.AppLog{})
	m.Register(&tsuru.AppGrant{})
	m.Register(&tsuru.AppRevoke{})
	m.Register(&tsuru.AppRestart{})
	m.Register(&tsuru.AppStart{})
	m.Register(&tsuru.SetCName{})
	m.Register(&tsuru.UnsetCName{})
	m.Register(&tsuru.EnvGet{})
	m.Register(&tsuru.EnvSet{})
	m.Register(&tsuru.EnvUnset{})
	m.Register(&KeyAdd{})
	m.Register(&KeyRemove{})
	m.Register(tsuru.ServiceList{})
	m.Register(tsuru.ServiceAdd{})
	m.Register(tsuru.ServiceRemove{})
	m.Register(tsuru.ServiceDoc{})
	m.Register(tsuru.ServiceInfo{})
	m.Register(tsuru.ServiceInstanceStatus{})
	m.Register(&tsuru.ServiceBind{})
	m.Register(&tsuru.ServiceUnbind{})
	m.Register(platformList{})
	m.Register(swap{})
	return m
}

func main() {
	name := cmd.ExtractProgramName(os.Args[0])
	manager := buildManager(name)
	manager.Run(os.Args[1:])
}
<commit_msg>update copyright data.<commit_after>// Copyright 2014 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"github.com/globocom/tsuru/cmd"
	"github.com/globocom/tsuru/cmd/tsuru-base"
	"os"
)

const (
	version = "0.8.6.1"
	header  = "Supported-Tsuru"
)

func buildManager(name string) *cmd.Manager {
	m := cmd.BuildBaseManager(name, version, header)
	m.Register(&tsuru.AppRun{})
	m.Register(&tsuru.AppInfo{})
	m.Register(AppCreate{})
	m.Register(&AppRemove{})
	m.Register(&UnitAdd{})
	m.Register(&UnitRemove{})
	m.Register(tsuru.AppList{})
	m.Register(&tsuru.AppLog{})
	m.Register(&tsuru.AppGrant{})
	m.Register(&tsuru.AppRevoke{})
	m.Register(&tsuru.AppRestart{})
	m.Register(&tsuru.AppStart{})
	m.Register(&tsuru.SetCName{})
	m.Register(&tsuru.UnsetCName{})
	m.Register(&tsuru.EnvGet{})
	m.Register(&tsuru.EnvSet{})
	m.Register(&tsuru.EnvUnset{})
	m.Register(&KeyAdd{})
	m.Register(&KeyRemove{})
	m.Register(tsuru.ServiceList{})
	m.Register(tsuru.ServiceAdd{})
	m.Register(tsuru.ServiceRemove{})
	m.Register(tsuru.ServiceDoc{})
	m.Register(tsuru.ServiceInfo{})
	m.Register(tsuru.ServiceInstanceStatus{})
	m.Register(&tsuru.ServiceBind{})
	m.Register(&tsuru.ServiceUnbind{})
	m.Register(platformList{})
	m.Register(swap{})
	return m
}

func main() {
	name := cmd.ExtractProgramName(os.Args[0])
	manager := buildManager(name)
	manager.Run(os.Args[1:])
}
<|endoftext|>