<commit_before>package fulltext

import (
	"bytes"
	"html/template"
	re "regexp"
	"strings"
)

var wordizeRe *re.Regexp

func init() {
	wordizeRe = re.MustCompile("[\\s,.;:!?[\\]()'\"]+")
}

type WordSplitter func(string)[]string;

// Split a string up into words
func Wordize(t string) []string {
	return wordizeRe.Split(t, -1)
}

type WordCleaner func(string)string;

// Make word appropriate for indexing
func IndexizeWord(w string) string {
	return strings.TrimSpace(strings.ToLower(w))
}

// This function copied from here: https://github.com/kennygrant/sanitize/blob/master/sanitize.go
// License is: https://github.com/kennygrant/sanitize/blob/master/License-BSD.txt
// Strip html tags, replace common entities, and escape <>&;'" in the result.
// Note the returned text may contain entities as it is escaped by HTMLEscapeString, and most entities are not translated.
func HTMLStripTags(s string) (output string) {

	output = ""

	// Shortcut strings with no tags in them
	if !strings.ContainsAny(s, "<>") {
		output = s
	} else {

		// First remove line breaks etc as these have no meaning outside html tags (except pre)
		// this means pre sections will lose formatting... but will result in less uninentional paras.
		s = strings.Replace(s, "\n", "", -1)

		// Then replace line breaks with newlines, to preserve that formatting
		s = strings.Replace(s, "
", "\n", -1)
		s = strings.Replace(s, "
", "\n", -1)
		s = strings.Replace(s, "
", "\n", -1)

		// Walk through the string removing all tags
		b := bytes.NewBufferString("")
		inTag := false
		for _, r := range s {
			switch r {
			case '<':
				inTag = true
			case '>':
				inTag = false
			default:
				if !inTag {
					b.WriteRune(r)
				}
			}
		}
		output = b.String()
	}

	// In case we have missed any tags above, escape the text - removes <, >, &, ' and ".
	output = template.HTMLEscapeString(output)

	// Remove a few common harmless entities, to arrive at something more like plain text
	// This relies on having removed *all* tags above
	output = strings.Replace(output, " ", " ", -1)
	output = strings.Replace(output, """, "\"", -1)
	output = strings.Replace(output, "'", "'", -1)
	output = strings.Replace(output, """, "\"", -1)
	output = strings.Replace(output, "'", "'", -1)
	// NB spaces here are significant - we only allow & not part of entity
	output = strings.Replace(output, "& ", "& ", -1)
	output = strings.Replace(output, "&amp; ", "& ", -1)

	return output
}

var titlere *re.Regexp
var descre *re.Regexp

func init() {
	titlere = re.MustCompile("(?i)<title>([^<]+)</title>")
	descre = re.MustCompile("(?i)<meta\\s+name\\s*=\\s*[\"']description[\"']\\s+content\\s*=\\s*[\"'](.*?)[\"']\\s*/?>")
}

// Helper to extract an HTML title from the title tag
func HTMLExtractTitle(html string) string {
	tret := titlere.FindSubmatch([]byte(html))
	title := ""
	if len(tret) > 1 {
		title = strings.TrimSpace(string(tret[1]))
	}
	return title
}

// Helper to extract an HTML description from the meta[name=description] tag
func HTMLExtractDescription(html string) string {
	tret := descre.FindSubmatch([]byte(html))
	desc := ""
	if len(tret) > 1 {
		desc = strings.TrimSpace(string(tret[1]))
	}
	return desc
}
<commit_msg>cosmetic format change<commit_after>package fulltext

import (
	"bytes"
	"html/template"
	re "regexp"
	"strings"
)

var wordizeRe *re.Regexp

func init() {
	wordizeRe = re.MustCompile("[\\s,.;:!?[\\]()'\"]+")
}

type WordSplitter func(string)[]string;

// Split a string up into words
func Wordize(t string) []string {
	return wordizeRe.Split(t, -1)
}

type WordCleaner func(string)string;

// Make word appropriate for indexing
func IndexizeWord(w string) string {
	return strings.TrimSpace(strings.ToLower(w))
}

// This function copied from here: https://github.com/kennygrant/sanitize/blob/master/sanitize.go
// License is: https://github.com/kennygrant/sanitize/blob/master/License-BSD.txt
// Strip html tags, replace common entities, and escape <>&;'" in the result.
// Note the returned text may contain entities as it is escaped by HTMLEscapeString,
// and most entities are not translated.
func HTMLStripTags(s string) (output string) {

	output = ""

	// Shortcut strings with no tags in them
	if !strings.ContainsAny(s, "<>") {
		output = s
	} else {

		// First remove line breaks etc as these have no meaning outside html tags (except pre)
		// this means pre sections will lose formatting... but will result in less uninentional paras.
		s = strings.Replace(s, "\n", "", -1)

		// Then replace line breaks with newlines, to preserve that formatting
		s = strings.Replace(s, "
", "\n", -1)
		s = strings.Replace(s, "
", "\n", -1)
		s = strings.Replace(s, "
", "\n", -1)

		// Walk through the string removing all tags
		b := bytes.NewBufferString("")
		inTag := false
		for _, r := range s {
			switch r {
			case '<':
				inTag = true
			case '>':
				inTag = false
			default:
				if !inTag {
					b.WriteRune(r)
				}
			}
		}
		output = b.String()
	}

	// In case we have missed any tags above, escape the text - removes <, >, &, ' and ".
	output = template.HTMLEscapeString(output)

	// Remove a few common harmless entities, to arrive at something more like plain text
	// This relies on having removed *all* tags above
	output = strings.Replace(output, " ", " ", -1)
	output = strings.Replace(output, """, "\"", -1)
	output = strings.Replace(output, "'", "'", -1)
	output = strings.Replace(output, """, "\"", -1)
	output = strings.Replace(output, "'", "'", -1)
	// NB spaces here are significant - we only allow & not part of entity
	output = strings.Replace(output, "& ", "& ", -1)
	output = strings.Replace(output, "&amp; ", "& ", -1)

	return output
}

var titlere *re.Regexp
var descre *re.Regexp

func init() {
	titlere = re.MustCompile("(?i)<title>([^<]+)</title>")
	descre = re.MustCompile("(?i)<meta\\s+name\\s*=\\s*[\"']description[\"']\\s+content\\s*=\\s*[\"'](.*?)[\"']\\s*/?>")
}

// Helper to extract an HTML title from the title tag
func HTMLExtractTitle(html string) string {
	tret := titlere.FindSubmatch([]byte(html))
	title := ""
	if len(tret) > 1 {
		title = strings.TrimSpace(string(tret[1]))
	}
	return title
}

// Helper to extract an HTML description from the meta[name=description] tag
func HTMLExtractDescription(html string) string {
	tret := descre.FindSubmatch([]byte(html))
	desc := ""
	if len(tret) > 1 {
		desc = strings.TrimSpace(string(tret[1]))
	}
	return desc
}
<|endoftext|>


<commit_before>package main

import (
	"io/ioutil"
	"net/http"
	"html/template"
	"log"
)

// Represents a page in the wiki.
type Page struct {
	Title string
	// io library works with bytes, so that's why we use a byte slice.
	Body []byte
}

// Persistence function.
// "p" is the receiver of this function. Interesting.
// Will return nil if save works.
func (p *Page) save() error {
	filename := "data/" + p.Title + ".txt"
	return ioutil.WriteFile(filename, p.Body, 0600)
}

// Load data from filesystem.
func loadPage(title string) (*Page, error) {
	filename := "data/" + title + ".txt"
	body, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}
	return &Page{Title: title, Body: body}, nil
}

func viewHandler(w http.ResponseWriter, r *http.Request) {
	title := r.URL.Path[len("/view/"):]
	p, err := loadPage(title)
	if err != nil {
		// If we attempt to view a page that doesn't exist, lets create one.
		http.Redirect(w, r, "/edit/" + title, http.StatusFound)
		return
	}
	renderTemplate(w, "view", p)
}

// i.e. if user accesses http://localhost:8080/, redirect to http://localhost:8080/view/index
func defaultHandler(w http.ResponseWriter, r *http.Request) {
	http.Redirect(w, r, "/view/index", http.StatusFound)
}

func editHandler(w http.ResponseWriter, r *http.Request) {
	title := r.URL.Path[len("/edit/"):]
	p, err := loadPage(title)
	tmpl := "edit"
	if err != nil {
		p = &Page{Title: title}
		tmpl = "create"
	}
	renderTemplate(w, tmpl, p)
}

func saveHandler(w http.ResponseWriter, r *http.Request) {

	title := r.URL.Path[len("/save/"):]
	body := r.FormValue("body")

	p := &Page{Title: title, Body: []byte(body)}
	p.save()

	http.Redirect(w, r, "/view/" + title, http.StatusFound)

}

func renderTemplate(w http.ResponseWriter, tmpl string, p *Page) {
	t, _ := template.ParseFiles("templates/" + tmpl + ".html")
	t.Execute(w, p)
}

func main() {

	port := ":8080"

	// Application handlers
	http.HandleFunc("/", defaultHandler)
	http.HandleFunc("/save/", saveHandler)
	http.HandleFunc("/view/", viewHandler)
	http.HandleFunc("/edit/", editHandler)

	log.Println("Server running, access via http://localhost" + port + "\n")

	http.ListenAndServe(port, nil)

}

<commit_msg>Added more error handling (looks nice and clean). Also added template caching. And more comments.<commit_after>package main

import (
	"io/ioutil"
	"net/http"
	"html/template"
	"log"
)

// Represents a page in the wiki.
type Page struct {
	Title string
	// io library works with bytes, so that's why we use a byte slice.
	Body []byte
}

// Load templates ONCE from filesystem, rather than on every request.
var templates = template.Must(template.ParseFiles(
	"templates/create.html",
	"templates/edit.html",
	"templates/view.html",
))

// Persistence function.
// "p" is the receiver of this function. Interesting.
// Will return nil if save works.
func (p *Page) save() error {
	filename := "data/" + p.Title + ".txt"
	return ioutil.WriteFile(filename, p.Body, 0600)
}

// Load data from filesystem.
func loadPage(title string) (*Page, error) {
	filename := "data/" + title + ".txt"
	body, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}
	return &Page{Title: title, Body: body}, nil
}

// View a wiki page in html!
func viewHandler(w http.ResponseWriter, r *http.Request) {

	title := r.URL.Path[len("/view/"):]

	p, err := loadPage(title)

	if err != nil {
		// If we attempt to view a page that doesn't exist, lets create one.
		http.Redirect(w, r, "/edit/" + title, http.StatusFound)
		return
	}

	renderTemplate(w, "view", p)

}

// Handle a redirect from base url to viewing a default wiki page.
// i.e. if user accesses http://localhost:8080/, redirect to http://localhost:8080/view/index
func defaultHandler(w http.ResponseWriter, r *http.Request) {
	http.Redirect(w, r, "/view/index", http.StatusFound)
}

// Either edit an existing wiki page, else create a new one if it doesn't exist.
func editHandler(w http.ResponseWriter, r *http.Request) {

	title := r.URL.Path[len("/edit/"):]

	p, err := loadPage(title)

	tmpl := "edit"

	if err != nil {
		p = &Page{Title: title}
		tmpl = "create"
	}

	renderTemplate(w, tmpl, p)

}

// Save a wiki page to filesystem.
func saveHandler(w http.ResponseWriter, r *http.Request) {

	title := r.URL.Path[len("/save/"):]
	body := r.FormValue("body")

	p := &Page{Title: title, Body: []byte(body)}
	err := p.save()

	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	http.Redirect(w, r, "/view/" + title, http.StatusFound)

}

// Helper function, given a Page, render using a certain template.
func renderTemplate(w http.ResponseWriter, tmpl string, p *Page) {

	err := templates.ExecuteTemplate(w, tmpl + ".html", p)

	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

}

// Load http server, bind handlers.
func main() {

	port := ":8080"

	// Application handlers
	http.HandleFunc("/", defaultHandler)
	http.HandleFunc("/save/", saveHandler)
	http.HandleFunc("/view/", viewHandler)
	http.HandleFunc("/edit/", editHandler)

	log.Println("Server running, access via http://localhost" + port + "\n")

	http.ListenAndServe(port, nil)

}

<|endoftext|>


<commit_before>package possum

import (
	"fmt"
	"net/http"
	"net/url"
)

// Resources wrap struct
type ResWrap struct{
	f HandlerFunc
	res interface{}
}

// Make a new resources wrap struct for `res` with `f` function
func NewResWrap(f HandlerFunc, res interface{}) (wrap *ResWrap, err error) {
	switch res.(type) {
	case Get, Post, Put, Delete, Patch:
	default:
		return nil, fmt.Errorf("`%T` is not a legal resource", res)
	}
	wrap = &Wrap{f, res}
	return
}

func (wrap *ResWrap) Get(params url.Values) (int, interface{}) {
	if status, err := wrap.f(params); status != http.StatusOK {
		return status, err
	}
	if res, ok := wrap.res.(Get); ok {
		return res.Get(params)
	}
	return (&NoGet{}).Get(params)
}

func (wrap *ResWrap) Post(params url.Values) (int, interface{}) {
	if status, err := wrap.f(params); status != http.StatusOK {
		return status, err
	}
	if res, ok := wrap.res.(Post); ok {
		return res.Post(params)
	}
	return (&NoPost{}).Post(params)
}

func (wrap *ResWrap) Put(params url.Values) (int, interface{}) {
	if status, err := wrap.f(params); status != http.StatusOK {
		return status, err
	}
	if res, ok := wrap.res.(Put); ok {
		return res.Put(params)
	}
	return (&NoPut{}).Put(params)
}

func (wrap *ResWrap) Delete(params url.Values) (int, interface{}) {
	if status, err := wrap.f(params); status != http.StatusOK {
		return status, err
	}
	if res, ok := wrap.res.(Delete); ok {
		return res.Delete(params)
	}
	return (&NoDelete{}).Delete(params)
}

func (wrap *ResWrap) Patch(params url.Values) (int, interface{}) {
	if status, err := wrap.f(params); status != http.StatusOK {
		return status, err
	}
	if res, ok := wrap.res.(Patch); ok {
		return res.Patch(params)
	}
	return (&NoPatch{}).Patch(params)
}
<commit_msg>refactoring wrap's name<commit_after>package possum

import (
	"fmt"
	"net/http"
	"net/url"
)

// Resources wrap struct
type Wrap struct{
	f HandlerFunc
	res interface{}
}

// Make a new resources wrap struct for `res` with `f` function
func NewWrap(f HandlerFunc, res interface{}) (wrap *Wrap, err error) {
	switch res.(type) {
	case Get, Post, Put, Delete, Patch:
	default:
		return nil, fmt.Errorf("`%T` is not a legal resource", res)
	}
	wrap = &Wrap{f, res}
	return
}

func (wrap *Wrap) Get(params url.Values) (int, interface{}) {
	if status, err := wrap.f(params); status != http.StatusOK {
		return status, err
	}
	if res, ok := wrap.res.(Get); ok {
		return res.Get(params)
	}
	return (&NoGet{}).Get(params)
}

func (wrap *Wrap) Post(params url.Values) (int, interface{}) {
	if status, err := wrap.f(params); status != http.StatusOK {
		return status, err
	}
	if res, ok := wrap.res.(Post); ok {
		return res.Post(params)
	}
	return (&NoPost{}).Post(params)
}

func (wrap *Wrap) Put(params url.Values) (int, interface{}) {
	if status, err := wrap.f(params); status != http.StatusOK {
		return status, err
	}
	if res, ok := wrap.res.(Put); ok {
		return res.Put(params)
	}
	return (&NoPut{}).Put(params)
}

func (wrap *Wrap) Delete(params url.Values) (int, interface{}) {
	if status, err := wrap.f(params); status != http.StatusOK {
		return status, err
	}
	if res, ok := wrap.res.(Delete); ok {
		return res.Delete(params)
	}
	return (&NoDelete{}).Delete(params)
}

func (wrap *Wrap) Patch(params url.Values) (int, interface{}) {
	if status, err := wrap.f(params); status != http.StatusOK {
		return status, err
	}
	if res, ok := wrap.res.(Patch); ok {
		return res.Patch(params)
	}
	return (&NoPatch{}).Patch(params)
}
<|endoftext|>


<commit_before>package main

import (
	"archive/tar"
	"compress/gzip"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"mime"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"syscall"

	"cloud.google.com/go/pubsub"
	"cloud.google.com/go/storage"
	"golang.org/x/net/context"
)

const updateTopicString string = "onesie-updates"
const updateSub string = "onesie-server"

func main() {
	log.Println("Starting")
	pubsubClient, err := pubsub.NewClient(context.Background(), "940380154622")
	if err != nil {
		log.Fatalf("Failed to create client: %v", err)
	}

	updateTopic := pubsubClient.Topic(updateTopicString)
	sub := pubsubClient.Subscription(updateSub)
	b, err := sub.Exists(context.Background())
	if err != nil {
		log.Fatal(err)
	}
	if !b {
		sub, err = pubsubClient.CreateSubscription(context.Background(), updateSub, pubsub.SubscriptionConfig{
			Topic: updateTopic,
		})
		if err != nil {
			log.Fatal(err)
		}
	}

	var mu sync.Mutex
	received := 0
	cctx, cancel := context.WithCancel(context.Background())
	err = sub.Receive(cctx, func(ctx context.Context, msg *pubsub.Message) {
		mu.Lock()
		defer mu.Unlock()
		received++
		if received >= 4 {
			cancel()
			msg.Nack()
			return
		}

		msgStr := string(msg.Data)
		fmt.Printf("Got message: %q\n", msgStr)

		if msgStr == "deploy" {
			domain := msg.Attributes["domain"]
			path := msg.Attributes["path"]
			log.Printf("Opening for archive: %s", path)

			// Open google storage client
			client, err := storage.NewClient(ctx)
			if err != nil {
				log.Panicf("Error connecting to Google Storage: %+v", err)
			}
			defer client.Close()
			bkt := client.Bucket("onesie")
			obj := bkt.Object(path)
			r, err := obj.NewReader(ctx)
			if err != nil {
				log.Panicf("Error opening object: %+v", err)
			}
			defer r.Close()

			// Expand into archive
			archive, err := gzip.NewReader(r)
			if err != nil {
				log.Panicf("Error creating gzip reader: %+v", err)
			}
			defer archive.Close()

			// Go through file by file
			tarReader := tar.NewReader(archive)
			buf := make([]byte, 160)
			for {
				header, err := tarReader.Next()
				if err == io.EOF {
					break
				} else if err != nil {
					log.Panicf("Error reading tar: %+v", err)
				}

				path := filepath.Join(domain, header.Name)
				switch header.Typeflag {
				case tar.TypeDir:
					continue
				case tar.TypeReg:
					w := bkt.Object(path).NewWriter(ctx)
					defer w.Close()
					w.ACL = []storage.ACLRule{{Entity: storage.AllUsers, Role: storage.RoleReader}}
					if filepath.Ext(path) != "" {
						w.ObjectAttrs.ContentType = mime.TypeByExtension(filepath.Ext(path))
					}
					wrtn, err := io.CopyBuffer(w, tarReader, buf)
					if err != nil {
						log.Printf("Error writing data to GCS: %+v", err)
					}
					log.Printf("Wrote %v bytes to %s", wrtn, path)
				default:
					log.Printf("Unable to figure out type: %v (%s)", header.Typeflag, path)
				}
			}
		}

		if msgStr == "update" {
			// Merge Certs
			files, err := ioutil.ReadDir("/opt/onesie-configs/certs/")
			if err != nil {
				log.Fatalf("Error iterating through files: %+v", err)
			}

			dhparam, err := ioutil.ReadFile(fmt.Sprintf("/opt/onesie-configs/dhparam.pem"))
			if err != nil {
				log.Fatalf("Error reading dhparam: %+v", err)
			}

			for _, file := range files {
				// cat /opt/onesie-configs/certs/$i/{privkey,fullchain}.pem /opt/onesie-configs/dhparam.pem > /opt/onesie-configs/hitch/$i.pem
				log.Printf("Parsing file: %+v", file)
				privkey, err := ioutil.ReadFile(fmt.Sprintf("/opt/onesie-configs/certs/%s/privkey.pem", file.Name()))
				if err != nil {
					log.Printf("Error reading privkey: %+v", err)
					continue
				}
				fullchain, err := ioutil.ReadFile(fmt.Sprintf("/opt/onesie-configs/certs/%s/fullchain.pem", file.Name()))
				if err != nil {
					log.Printf("Error reading fullchain: %+v", err)
					continue
				}

				// Write out
				f, err := os.OpenFile(fmt.Sprintf("/opt/onesie-configs/hitch/%s.pem", file.Name()), os.O_APPEND|os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0777)
				if err != nil {
					log.Printf("Error opening output pem: %+v", err)
					continue
				}
				defer f.Close()

				if _, err = f.Write(privkey); err != nil {
					log.Printf("Error writing privkey to output pem: %+v", err)
					continue
				}
				if _, err = f.Write(fullchain); err != nil {
					log.Printf("Error writing fullchain to output pem: %+v", err)
					continue
				}
				if _, err = f.Write(dhparam); err != nil {
					log.Printf("Error writing dhparam to output pem: %+v", err)
					continue
				}
			}

			// Get hitch PID, send sighup
			out, err := exec.Command("/bin/pidof", "hitch").Output()
			if err != nil {
				log.Printf("Error running pidof: %+v", err)
			}
			for _, pidStr := range strings.Split(string(out), " ") {
				pid, err := strconv.Atoi(strings.TrimSpace(pidStr))
				if err != nil {
					log.Printf("Error parsing string: %+v", err)
					continue
				}
				log.Printf("Sending SIGHUP to %+v", pid)
				syscall.Kill(pid, syscall.SIGHUP)
			}
		}
		msg.Ack()
	})

	if err != nil {
		log.Println(err)
	}

	log.Println("Finished.")
}
<commit_msg>Write read all to all<commit_after>package main

import (
	"archive/tar"
	"compress/gzip"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"mime"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"syscall"

	"cloud.google.com/go/pubsub"
	"cloud.google.com/go/storage"
	"golang.org/x/net/context"
)

const updateTopicString string = "onesie-updates"
const updateSub string = "onesie-server"

func main() {
	log.Println("Starting")
	pubsubClient, err := pubsub.NewClient(context.Background(), "940380154622")
	if err != nil {
		log.Fatalf("Failed to create client: %v", err)
	}

	updateTopic := pubsubClient.Topic(updateTopicString)
	sub := pubsubClient.Subscription(updateSub)
	b, err := sub.Exists(context.Background())
	if err != nil {
		log.Fatal(err)
	}
	if !b {
		sub, err = pubsubClient.CreateSubscription(context.Background(), updateSub, pubsub.SubscriptionConfig{
			Topic: updateTopic,
		})
		if err != nil {
			log.Fatal(err)
		}
	}

	// Make sure all files are public.
	go func(context ctx) {
		// Open google storage client
		client, err := storage.NewClient(ctx)
		if err != nil {
			log.Panicf("Error connecting to Google Storage: %+v", err)
		}
		defer client.Close()
		bkt := client.Bucket("onesie")
		it := bkt.Objects(ctx, nil)
		for {
			obj, err := it.Next()
			if err == iterator.Done {
				break
			}
			if err != nil {
				return err
			}

			// Let everyone read it.
			if err := obj.ACL().Set(ctx, storage.AllUsers, storage.RoleReader); err != nil {
				log.Infof("Could not set acl for %+v", obj)
				return err
			}
		}
	}(ctx)

	var mu sync.Mutex
	received := 0
	cctx, cancel := context.WithCancel(context.Background())
	err = sub.Receive(cctx, func(ctx context.Context, msg *pubsub.Message) {
		mu.Lock()
		defer mu.Unlock()
		received++
		if received >= 4 {
			cancel()
			msg.Nack()
			return
		}

		msgStr := string(msg.Data)
		fmt.Printf("Got message: %q\n", msgStr)

		if msgStr == "deploy" {
			domain := msg.Attributes["domain"]
			path := msg.Attributes["path"]
			log.Printf("Opening for archive: %s", path)

			// Open google storage client
			client, err := storage.NewClient(ctx)
			if err != nil {
				log.Panicf("Error connecting to Google Storage: %+v", err)
			}
			defer client.Close()
			bkt := client.Bucket("onesie")
			obj := bkt.Object(path)
			r, err := obj.NewReader(ctx)
			if err != nil {
				log.Panicf("Error opening object: %+v", err)
			}
			defer r.Close()

			// Expand into archive
			archive, err := gzip.NewReader(r)
			if err != nil {
				log.Panicf("Error creating gzip reader: %+v", err)
			}
			defer archive.Close()

			// Go through file by file
			tarReader := tar.NewReader(archive)
			buf := make([]byte, 160)
			for {
				header, err := tarReader.Next()
				if err == io.EOF {
					break
				} else if err != nil {
					log.Panicf("Error reading tar: %+v", err)
				}

				path := filepath.Join(domain, header.Name)
				switch header.Typeflag {
				case tar.TypeDir:
					continue
				case tar.TypeReg:
					w := bkt.Object(path).NewWriter(ctx)
					defer w.Close()
					w.ACL = []storage.ACLRule{{Entity: storage.AllUsers, Role: storage.RoleReader}}
					if filepath.Ext(path) != "" {
						w.ObjectAttrs.ContentType = mime.TypeByExtension(filepath.Ext(path))
					}
					wrtn, err := io.CopyBuffer(w, tarReader, buf)
					if err != nil {
						log.Printf("Error writing data to GCS: %+v", err)
					}
					log.Printf("Wrote %v bytes to %s", wrtn, path)
				default:
					log.Printf("Unable to figure out type: %v (%s)", header.Typeflag, path)
				}
			}
		}

		if msgStr == "update" {
			// Merge Certs
			files, err := ioutil.ReadDir("/opt/onesie-configs/certs/")
			if err != nil {
				log.Fatalf("Error iterating through files: %+v", err)
			}

			dhparam, err := ioutil.ReadFile(fmt.Sprintf("/opt/onesie-configs/dhparam.pem"))
			if err != nil {
				log.Fatalf("Error reading dhparam: %+v", err)
			}

			for _, file := range files {
				// cat /opt/onesie-configs/certs/$i/{privkey,fullchain}.pem /opt/onesie-configs/dhparam.pem > /opt/onesie-configs/hitch/$i.pem
				log.Printf("Parsing file: %+v", file)
				privkey, err := ioutil.ReadFile(fmt.Sprintf("/opt/onesie-configs/certs/%s/privkey.pem", file.Name()))
				if err != nil {
					log.Printf("Error reading privkey: %+v", err)
					continue
				}
				fullchain, err := ioutil.ReadFile(fmt.Sprintf("/opt/onesie-configs/certs/%s/fullchain.pem", file.Name()))
				if err != nil {
					log.Printf("Error reading fullchain: %+v", err)
					continue
				}

				// Write out
				f, err := os.OpenFile(fmt.Sprintf("/opt/onesie-configs/hitch/%s.pem", file.Name()), os.O_APPEND|os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0777)
				if err != nil {
					log.Printf("Error opening output pem: %+v", err)
					continue
				}
				defer f.Close()

				if _, err = f.Write(privkey); err != nil {
					log.Printf("Error writing privkey to output pem: %+v", err)
					continue
				}
				if _, err = f.Write(fullchain); err != nil {
					log.Printf("Error writing fullchain to output pem: %+v", err)
					continue
				}
				if _, err = f.Write(dhparam); err != nil {
					log.Printf("Error writing dhparam to output pem: %+v", err)
					continue
				}
			}

			// Get hitch PID, send sighup
			out, err := exec.Command("/bin/pidof", "hitch").Output()
			if err != nil {
				log.Printf("Error running pidof: %+v", err)
			}
			for _, pidStr := range strings.Split(string(out), " ") {
				pid, err := strconv.Atoi(strings.TrimSpace(pidStr))
				if err != nil {
					log.Printf("Error parsing string: %+v", err)
					continue
				}
				log.Printf("Sending SIGHUP to %+v", pid)
				syscall.Kill(pid, syscall.SIGHUP)
			}
		}
		msg.Ack()
	})

	if err != nil {
		log.Println(err)
	}

	log.Println("Finished.")
}
<|endoftext|>


<commit_before>// Copyright 2010 Hannes Baldursson. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This file is part of go-idn

// Package stringprep implements Stringprep as described in RFC 3454
package stringprep

import (
	"bytes"
	"code.google.com/p/go-idn/idna/norm32"
	"unicode/utf8"
)

// InvalidStringError represents an invalid string error in the input stream.
type InvalidStringError struct {
	// TODO: What should we report?
}

func (e InvalidStringError) Error() string {
	panic("TODO")
	return "Invalid string"
}

const (
	MAX_MAP_CHARS = 4
)

// A Profile...
//
// For a Profile p, this documentation uses the notation p(x) to mean the bytes
// or string x prepared with the given profile.
type Profile struct {
	// Defaults to true
	Normalize bool

	// Defaults to false
	AllowUnassigned bool

	// Defined by profile
	CheckBidi bool

	f          norm32.Form
	mappings   Table
	prohibited Table
}

// Append returns p(append(out, b...)). The buffer out must be nil, empty or
// equal to p(out).
func (p *Profile) Append(out []byte, src ...byte) []byte { return nil }

// AppendString returns p(append(out, []byte(s))). The buffer out must be nil,
//empty, or equal to p(out).
func (p *Profile) AppendString(out []byte, src string) []byte { return nil }

// Bytes returns p(b). May return b if p(b) = b.
func (p *Profile) Bytes(b []byte) []byte { return nil }

// Reader returns a new reader that implements Read by reading data from r and
// returning p(data).
func (p *Profile) Reader(r io.Reader) io.Reader { return nil }

// String returns p(s).
func (p *Profile) String(s string) string { return "" }

// Writer returns a new writer that implements Write(b) by writing p(b) to w.
// The returned writer may use an an internal buffer to maintain state across
// Write calls. Calling its Close method writes any buffered data to w.
func (p *Profile) Writer(w io.Writer) io.WriteCloser { return nil }

type d [MAX_MAP_CHARS]int

type ProfileElement struct {
	Step  int // see Step const's
	Table Table
}

func Stringprep(input string, profile Profile) string {
	input_runes := bytes.Runes([]byte(input))
	return stringify(StringprepRunes(input_runes, profile))
}

// Prepare the input rune array according to the stringprep profile,
// and return the results as a rune array.
func StringprepRunes(input []rune, profile Profile) []int {
	output := make([]int, len(input))
	copy(output[0:], input[0:])

	for i := 0; i < len(profile); i++ {
		switch profile[i].Step {
		case NFKC:
			output = normalization.NFKC(output)
			break
		case BIDI:
			done_prohibited := 0
			done_ral := 0
			done_l := 0
			contains_ral := -1
			contains_l := -1

			for j := 0; i < len(profile); j++ {
				switch profile[j].Step {
				case BIDI_PROHIBIT_TABLE:
					done_prohibited = 1
					for k := 0; k < len(output); k++ {
						if in_table(output[k], profile[j].Table) {
							return nil
						}
					}

				case BIDI_RAL_TABLE:
					done_ral = 1
					for k := 0; k < len(output); k++ {
						if in_table(output[k], profile[j].Table) {
							contains_ral = j
						}
					}

				case BIDI_L_TABLE:
					done_l = 1
					for k := 0; k < len(output); k++ {
						if in_table(output[k], profile[j].Table) {
							contains_l = j
						}
					}
				}
			}

			if done_prohibited != 1 || done_ral != 1 || done_l != 1 {
				return nil // PROFILE ERROR
			}

			if contains_ral != -1 && contains_l != -1 {
				return nil // BIDI BOTH L AND RAL
			}

			if contains_ral != -1 {
				return nil // Error?
			}

			break
		case MAP_TABLE:
			output = map_table(output, profile[i].Table)
			break
		case UNASSIGNED_TABLE:
			break
			//switch profile[i].Table
		case PROHIBIT_TABLE:
			for k := 0; k < len(output); k++ {
				if in_table(output[k], profile[i].Table) {
					return nil
				}
			}
			break
		case BIDI_PROHIBIT_TABLE:
			break
		case BIDI_RAL_TABLE:
			break
		case BIDI_L_TABLE:
			break
		default:
			return nil // PROFILE ERROR
		}
	}

	return output
}

// turn a slice of runes into an equivalent string
func stringify(runes []int) string {
	t := make([]byte, len(runes)*4) // kludge!
	i := 0
	for _, r := range runes {
		i += utf8.EncodeRune(t[i:], r)
	}
	return string(t)
}
<commit_msg>idna/stringprep: Remove deprecated functions.<commit_after>// Copyright 2010 Hannes Baldursson. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This file is part of go-idn

// Package stringprep implements Stringprep as described in RFC 3454
package stringprep

import (
	"bytes"
	"code.google.com/p/go-idn/idna/norm32"
	"io"
	"unicode/utf8"
)

// InvalidStringError represents an invalid string error in the input stream.
type InvalidStringError struct {
	// TODO: What should we report?
}

func (e InvalidStringError) Error() string {
	panic("TODO")
	return "Invalid string"
}

const (
	MAX_MAP_CHARS = 4
)

// A Profile...
//
// For a Profile p, this documentation uses the notation p(x) to mean the bytes
// or string x prepared with the given profile.
type Profile struct {
	// Defaults to true
	Normalize bool

	// Defaults to false
	AllowUnassigned bool

	// Defined by profile
	CheckBidi bool

	f          norm32.Form
	mappings   Table
	prohibited Table
}

// Append returns p(append(out, b...)). The buffer out must be nil, empty or
// equal to p(out).
func (p *Profile) Append(out []byte, src ...byte) []byte { return nil }

// AppendString returns p(append(out, []byte(s))). The buffer out must be nil,
//empty, or equal to p(out).
func (p *Profile) AppendString(out []byte, src string) []byte { return nil }

// Bytes returns p(b). May return b if p(b) = b.
func (p *Profile) Bytes(b []byte) []byte { return nil }

// Reader returns a new reader that implements Read by reading data from r and
// returning p(data).
func (p *Profile) Reader(r io.Reader) io.Reader { return nil }

// String returns p(s).
func (p *Profile) String(s string) string { return "" }

// Writer returns a new writer that implements Write(b) by writing p(b) to w.
// The returned writer may use an an internal buffer to maintain state across
// Write calls. Calling its Close method writes any buffered data to w.
func (p *Profile) Writer(w io.Writer) io.WriteCloser { return nil }

// Prepare the input rune array according to the stringprep profile,
// and return the results as a rune array.
func StringprepRunes(input []rune, profile Profile) []int {
	output := make([]int, len(input))
	copy(output[0:], input[0:])

	for i := 0; i < len(profile); i++ {
		switch profile[i].Step {
		case NFKC:
			output = normalization.NFKC(output)
			break
		case BIDI:
			done_prohibited := 0
			done_ral := 0
			done_l := 0
			contains_ral := -1
			contains_l := -1

			for j := 0; i < len(profile); j++ {
				switch profile[j].Step {
				case BIDI_PROHIBIT_TABLE:
					done_prohibited = 1
					for k := 0; k < len(output); k++ {
						if in_table(output[k], profile[j].Table) {
							return nil
						}
					}

				case BIDI_RAL_TABLE:
					done_ral = 1
					for k := 0; k < len(output); k++ {
						if in_table(output[k], profile[j].Table) {
							contains_ral = j
						}
					}

				case BIDI_L_TABLE:
					done_l = 1
					for k := 0; k < len(output); k++ {
						if in_table(output[k], profile[j].Table) {
							contains_l = j
						}
					}
				}
			}

			if done_prohibited != 1 || done_ral != 1 || done_l != 1 {
				return nil // PROFILE ERROR
			}

			if contains_ral != -1 && contains_l != -1 {
				return nil // BIDI BOTH L AND RAL
			}

			if contains_ral != -1 {
				return nil // Error?
			}

			break
		case MAP_TABLE:
			output = map_table(output, profile[i].Table)
			break
		case UNASSIGNED_TABLE:
			break
			//switch profile[i].Table
		case PROHIBIT_TABLE:
			for k := 0; k < len(output); k++ {
				if in_table(output[k], profile[i].Table) {
					return nil
				}
			}
			break
		case BIDI_PROHIBIT_TABLE:
			break
		case BIDI_RAL_TABLE:
			break
		case BIDI_L_TABLE:
			break
		default:
			return nil // PROFILE ERROR
		}
	}

	return output
}
<|endoftext|>


<commit_before>/* This file is part of VoltDB.
 * Copyright (C) 2008-2016 VoltDB Inc.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with VoltDB.  If not, see <http://www.gnu.org/licenses/>.
 */

package voltdbclient

import (
	"database/sql/driver"
	"errors"
	"fmt"
	"log"
	"sync"
	"sync/atomic"
	"time"
)

// the set of currently active connections
type distributer struct {
	handle int64
	// the set of active connections
	acs      []*nodeConn
	acsMutex sync.RWMutex
	// next conn to look at when finding by round robin.
	acsNextI int
	open     atomic.Value
	h        *hashinater
}

func newDistributer() *distributer {
	var d = new(distributer)
	d.handle = 0
	d.acs = make([]*nodeConn, 0)
	d.acsMutex = sync.RWMutex{}
	d.acsNextI = 0
	d.open = atomic.Value{}
	d.open.Store(true)
	d.h = newHashinater()
	return d
}

// add a conn that is newly connected
func (d *distributer) addConn(ac *nodeConn) {
	d.assertOpen()
	d.acsMutex.Lock()
	d.acs = append(d.acs, ac)
	d.acsMutex.Unlock()
}

// Begin starts a transaction.  VoltDB runs in auto commit mode, and so Begin
// returns an error.
func (d *distributer) Begin() (driver.Tx, error) {
	d.assertOpen()
	return nil, errors.New("VoltDB does not support transactions, VoltDB autocommits")
}

// Close closes the connection to the VoltDB server.  Connections to the server
// are meant to be long lived; it should not be necessary to continually close
// and reopen connections.  Close would typically be called using a defer.
// Operations using a closed connection cause a panic.
func (d *distributer) Close() (err error) {
	if d.isClosed() {
		return
	}
	d.setClosed()

	// once this is closed there shouldn't be any additional activity against
	// the connections.  They're touched here without getting a lock.
	for _, ac := range d.acs {
		err := ac.close()
		if err != nil {
			log.Println("Failed to close connection with %v", err)
		}
	}
	d.acs = nil
	return nil
}

func (d *distributer) removeConn(ci string) {
	d.assertOpen()
	d.acsMutex.Lock()
	for i, ac := range d.acs {
		if ac.connInfo == ci {
			d.acs = append(d.acs[:i], d.acs[i+1:]...)
			break
		}
	}
	d.acsMutex.Unlock()
}

// Drain blocks until all outstanding asynchronous requests have been satisfied.
// Asynchronous requests are processed in a background thread; this call blocks the
// current thread until that background thread has finished with all asynchronous requests.
func (d *distributer) Drain() {
	d.assertOpen()

	// drain can't work if the set of connections can change
	// while it's running.  Hold the lock for the duration, some
	// asyncs may time out.  The user thread is blocked on drain.
	d.acsMutex.Lock()
	for _, ac := range d.acs {
		ac.drain()
	}
	d.acsMutex.Unlock()
}

func (d *distributer) assertOpen() {
	if !(d.open.Load().(bool)) {
		panic("Tried to use closed connection pool")
	}
}

func (d *distributer) isClosed() bool {
	return !(d.open.Load().(bool))
}

func (d *distributer) setClosed() {
	d.open.Store(false)
}

func (d *distributer) numConns() int {
	d.assertOpen()
	d.acsMutex.RLock()
	num := len(d.acs)
	d.acsMutex.RUnlock()
	return num
}

// Exec executes a query that doesn't return rows, such as an INSERT or UPDATE.
// Exec is available on both VoltConn and on VoltStatement.  Uses DEFAULT_QUERY_TIMEOUT.
func (d *distributer) Exec(query string, args []driver.Value) (driver.Result, error) {
	return d.ExecTimeout(query, args, DEFAULT_QUERY_TIMEOUT)
}

// Exec executes a query that doesn't return rows, such as an INSERT or UPDATE.
// Exec is available on both VoltConn and on VoltStatement.  Specifies a duration for timeout.
func (d *distributer) ExecTimeout(query string, args []driver.Value, timeout time.Duration) (driver.Result, error) {
	pi := newProcedureInvocation(d.handle, query, args, timeout)
	d.handle++
	ac, err := d.getConn(pi)
	if err != nil {
		return nil, err
	}
	return ac.exec(pi)
}

// Exec executes a query that doesn't return rows, such as an INSERT or UPDATE.
// ExecAsync is analogous to Exec but is run asynchronously.  That is, an
// invocation of this method blocks only until a request is sent to the VoltDB
// server.  Uses DEFAULT_QUERY_TIMEOUT.
func (d *distributer) ExecAsync(resCons AsyncResponseConsumer, query string, args []driver.Value) error {
	return d.ExecAsyncTimeout(resCons, query, args, DEFAULT_QUERY_TIMEOUT)
}

// Exec executes a query that doesn't return rows, such as an INSERT or UPDATE.
// ExecAsync is analogous to Exec but is run asynchronously.  That is, an
// invocation of this method blocks only until a request is sent to the VoltDB
// server.  Specifies a duration for timeout.
func (d *distributer) ExecAsyncTimeout(resCons AsyncResponseConsumer, query string, args []driver.Value, timeout time.Duration) error {
	pi := newProcedureInvocation(d.handle, query, args, timeout)
	d.handle++
	ac, err := d.getConn(pi)
	if err != nil {
		return err
	}
	return ac.execAsync(resCons, pi)
}

// Prepare creates a prepared statement for later queries or executions.
// The Statement returned by Prepare is bound to this VoltConn.
func (d *distributer) Prepare(query string) (driver.Stmt, error) {
	stmt := newVoltStatement(d, query)
	return *stmt, nil
}

// Query executes a query that returns rows, typically a SELECT. The args are for any placeholder parameters in the query.
// Uses DEFAULT_QUERY_TIMEOUT.
func (d *distributer) Query(query string, args []driver.Value) (driver.Rows, error) {
	return d.QueryTimeout(query, args, DEFAULT_QUERY_TIMEOUT)
}

// Query executes a query that returns rows, typically a SELECT. The args are for any placeholder parameters in the query.
// Specifies a duration for timeout.
func (d *distributer) QueryTimeout(query string, args []driver.Value, timeout time.Duration) (driver.Rows, error) {
	pi := newProcedureInvocation(d.handle, query, args, timeout)
	d.handle++
	ac, err := d.getConn(pi)
	if err != nil {
		return nil, err
	}
	return ac.query(pi)
}

// QueryAsync executes a query asynchronously.  The invoking thread will block
// until the query is sent over the network to the server.  The eventual
// response will be handled by the given AsyncResponseConsumer, this processing
// happens in the 'response' thread.  Uses DEFAULT_QUERY_TIMEOUT.
func (d *distributer) QueryAsync(rowsCons AsyncResponseConsumer, query string, args []driver.Value) error {
	return d.QueryAsyncTimeout(rowsCons, query, args, DEFAULT_QUERY_TIMEOUT)
}

// QueryAsync executes a query asynchronously.  The invoking thread will block
// until the query is sent over the network to the server.  The eventual
// response will be handled by the given AsyncResponseConsumer, this processing
// happens in the 'response' thread.  Specifies a duration for timeout.
func (d *distributer) QueryAsyncTimeout(rowsCons AsyncResponseConsumer, query string, args []driver.Value, timeout time.Duration) error {
	pi := newProcedureInvocation(d.handle, query, args, timeout)
	d.handle++
	ac, err := d.getConn(pi)
	if err != nil {
		return err
	}
	return ac.queryAsync(rowsCons, pi)
}

// Get a connection from the hashinator.  If not, get one by round robin.  If not return nil.
func (d *distributer) getConn(pi *procedureInvocation) (c *nodeConn, err error) {

	d.assertOpen()
	d.acsMutex.RLock()
	c = d.h.getConn(pi)
	if c == nil {
		c, err = d.getConnByRR(pi.timeout)
	}
	d.acsMutex.RUnlock()
	if err != nil {
		return nil, err
	}
	return c, nil
}

func (d *distributer) getConnByRR(timeout time.Duration) (*nodeConn, error) {
	start := time.Now()
	currLen := len(d.acs)

	for {
		if d.acsNextI >= currLen {
			d.acsNextI = 0
		}
		c := d.acs[d.acsNextI]
		d.acsNextI++
		if !c.hasBP() {
			return c, nil
		}
		if time.Now().Sub(start) > timeout {
			return nil, errors.New("timeout")
		}
	}
}

type procedureInvocation struct {
	handle  int64
	query   string
	params  []driver.Value
	timeout time.Duration
}

func newProcedureInvocation(handle int64, query string, params []driver.Value, timeout time.Duration) *procedureInvocation {
	var pi = new(procedureInvocation)
	pi.handle = handle
	pi.query = query
	pi.params = params
	pi.timeout = timeout
	return pi
}
<commit_msg>ENG-10558 remove bad import<commit_after>/* This file is part of VoltDB.
 * Copyright (C) 2008-2016 VoltDB Inc.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with VoltDB.  If not, see <http://www.gnu.org/licenses/>.
 */

package voltdbclient

import (
	"database/sql/driver"
	"errors"
	"log"
	"sync"
	"sync/atomic"
	"time"
)

// the set of currently active connections
type distributer struct {
	handle int64
	// the set of active connections
	acs      []*nodeConn
	acsMutex sync.RWMutex
	// next conn to look at when finding by round robin.
	acsNextI int
	open     atomic.Value
	h        *hashinater
}

func newDistributer() *distributer {
	var d = new(distributer)
	d.handle = 0
	d.acs = make([]*nodeConn, 0)
	d.acsMutex = sync.RWMutex{}
	d.acsNextI = 0
	d.open = atomic.Value{}
	d.open.Store(true)
	d.h = newHashinater()
	return d
}

// add a conn that is newly connected
func (d *distributer) addConn(ac *nodeConn) {
	d.assertOpen()
	d.acsMutex.Lock()
	d.acs = append(d.acs, ac)
	d.acsMutex.Unlock()
}

// Begin starts a transaction.  VoltDB runs in auto commit mode, and so Begin
// returns an error.
func (d *distributer) Begin() (driver.Tx, error) {
	d.assertOpen()
	return nil, errors.New("VoltDB does not support transactions, VoltDB autocommits")
}

// Close closes the connection to the VoltDB server.  Connections to the server
// are meant to be long lived; it should not be necessary to continually close
// and reopen connections.  Close would typically be called using a defer.
// Operations using a closed connection cause a panic.
func (d *distributer) Close() (err error) {
	if d.isClosed() {
		return
	}
	d.setClosed()

	// once this is closed there shouldn't be any additional activity against
	// the connections.  They're touched here without getting a lock.
	for _, ac := range d.acs {
		err := ac.close()
		if err != nil {
			log.Println("Failed to close connection with %v", err)
		}
	}
	d.acs = nil
	return nil
}

func (d *distributer) removeConn(ci string) {
	d.assertOpen()
	d.acsMutex.Lock()
	for i, ac := range d.acs {
		if ac.connInfo == ci {
			d.acs = append(d.acs[:i], d.acs[i+1:]...)
			break
		}
	}
	d.acsMutex.Unlock()
}

// Drain blocks until all outstanding asynchronous requests have been satisfied.
// Asynchronous requests are processed in a background thread; this call blocks the
// current thread until that background thread has finished with all asynchronous requests.
func (d *distributer) Drain() {
	d.assertOpen()

	// drain can't work if the set of connections can change
	// while it's running.  Hold the lock for the duration, some
	// asyncs may time out.  The user thread is blocked on drain.
	d.acsMutex.Lock()
	for _, ac := range d.acs {
		ac.drain()
	}
	d.acsMutex.Unlock()
}

func (d *distributer) assertOpen() {
	if !(d.open.Load().(bool)) {
		panic("Tried to use closed connection pool")
	}
}

func (d *distributer) isClosed() bool {
	return !(d.open.Load().(bool))
}

func (d *distributer) setClosed() {
	d.open.Store(false)
}

func (d *distributer) numConns() int {
	d.assertOpen()
	d.acsMutex.RLock()
	num := len(d.acs)
	d.acsMutex.RUnlock()
	return num
}

// Exec executes a query that doesn't return rows, such as an INSERT or UPDATE.
// Exec is available on both VoltConn and on VoltStatement.  Uses DEFAULT_QUERY_TIMEOUT.
func (d *distributer) Exec(query string, args []driver.Value) (driver.Result, error) {
	return d.ExecTimeout(query, args, DEFAULT_QUERY_TIMEOUT)
}

// Exec executes a query that doesn't return rows, such as an INSERT or UPDATE.
// Exec is available on both VoltConn and on VoltStatement.  Specifies a duration for timeout.
func (d *distributer) ExecTimeout(query string, args []driver.Value, timeout time.Duration) (driver.Result, error) {
	pi := newProcedureInvocation(d.handle, query, args, timeout)
	d.handle++
	ac, err := d.getConn(pi)
	if err != nil {
		return nil, err
	}
	return ac.exec(pi)
}

// Exec executes a query that doesn't return rows, such as an INSERT or UPDATE.
// ExecAsync is analogous to Exec but is run asynchronously.  That is, an
// invocation of this method blocks only until a request is sent to the VoltDB
// server.  Uses DEFAULT_QUERY_TIMEOUT.
func (d *distributer) ExecAsync(resCons AsyncResponseConsumer, query string, args []driver.Value) error {
	return d.ExecAsyncTimeout(resCons, query, args, DEFAULT_QUERY_TIMEOUT)
}

// Exec executes a query that doesn't return rows, such as an INSERT or UPDATE.
// ExecAsync is analogous to Exec but is run asynchronously.  That is, an
// invocation of this method blocks only until a request is sent to the VoltDB
// server.  Specifies a duration for timeout.
func (d *distributer) ExecAsyncTimeout(resCons AsyncResponseConsumer, query string, args []driver.Value, timeout time.Duration) error {
	pi := newProcedureInvocation(d.handle, query, args, timeout)
	d.handle++
	ac, err := d.getConn(pi)
	if err != nil {
		return err
	}
	return ac.execAsync(resCons, pi)
}

// Prepare creates a prepared statement for later queries or executions.
// The Statement returned by Prepare is bound to this VoltConn.
func (d *distributer) Prepare(query string) (driver.Stmt, error) {
	stmt := newVoltStatement(d, query)
	return *stmt, nil
}

// Query executes a query that returns rows, typically a SELECT. The args are for any placeholder parameters in the query.
// Uses DEFAULT_QUERY_TIMEOUT.
func (d *distributer) Query(query string, args []driver.Value) (driver.Rows, error) {
	return d.QueryTimeout(query, args, DEFAULT_QUERY_TIMEOUT)
}

// Query executes a query that returns rows, typically a SELECT. The args are for any placeholder parameters in the query.
// Specifies a duration for timeout.
func (d *distributer) QueryTimeout(query string, args []driver.Value, timeout time.Duration) (driver.Rows, error) {
	pi := newProcedureInvocation(d.handle, query, args, timeout)
	d.handle++
	ac, err := d.getConn(pi)
	if err != nil {
		return nil, err
	}
	return ac.query(pi)
}

// QueryAsync executes a query asynchronously.  The invoking thread will block
// until the query is sent over the network to the server.  The eventual
// response will be handled by the given AsyncResponseConsumer, this processing
// happens in the 'response' thread.  Uses DEFAULT_QUERY_TIMEOUT.
func (d *distributer) QueryAsync(rowsCons AsyncResponseConsumer, query string, args []driver.Value) error {
	return d.QueryAsyncTimeout(rowsCons, query, args, DEFAULT_QUERY_TIMEOUT)
}

// QueryAsync executes a query asynchronously.  The invoking thread will block
// until the query is sent over the network to the server.  The eventual
// response will be handled by the given AsyncResponseConsumer, this processing
// happens in the 'response' thread.  Specifies a duration for timeout.
func (d *distributer) QueryAsyncTimeout(rowsCons AsyncResponseConsumer, query string, args []driver.Value, timeout time.Duration) error {
	pi := newProcedureInvocation(d.handle, query, args, timeout)
	d.handle++
	ac, err := d.getConn(pi)
	if err != nil {
		return err
	}
	return ac.queryAsync(rowsCons, pi)
}

// Get a connection from the hashinator.  If not, get one by round robin.  If not return nil.
func (d *distributer) getConn(pi *procedureInvocation) (c *nodeConn, err error) {

	d.assertOpen()
	d.acsMutex.RLock()
	c = d.h.getConn(pi)
	if c == nil {
		c, err = d.getConnByRR(pi.timeout)
	}
	d.acsMutex.RUnlock()
	if err != nil {
		return nil, err
	}
	return c, nil
}

func (d *distributer) getConnByRR(timeout time.Duration) (*nodeConn, error) {
	start := time.Now()
	currLen := len(d.acs)

	for {
		if d.acsNextI >= currLen {
			d.acsNextI = 0
		}
		c := d.acs[d.acsNextI]
		d.acsNextI++
		if !c.hasBP() {
			return c, nil
		}
		if time.Now().Sub(start) > timeout {
			return nil, errors.New("timeout")
		}
	}
}

type procedureInvocation struct {
	handle  int64
	query   string
	params  []driver.Value
	timeout time.Duration
}

func newProcedureInvocation(handle int64, query string, params []driver.Value, timeout time.Duration) *procedureInvocation {
	var pi = new(procedureInvocation)
	pi.handle = handle
	pi.query = query
	pi.params = params
	pi.timeout = timeout
	return pi
}
<|endoftext|>


<commit_before>package geo_skeleton_server

import (
	"bufio"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"net/textproto"
	"os"
	"os/exec"
	"strings"

	"./utils"
	"github.com/paulmach/go.geojson"
)

const (
	TCP_DEFAULT_CONN_HOST = "localhost"
	TCP_DEFAULT_CONN_PORT = "3333"
	TCP_DEFAULT_CONN_TYPE = "tcp"
)

type TcpServer struct {
	Host             string
	Port             string
	ConnType         string
	ActiveTcpClients int
}

func (self *TcpServer) getHost() string {
	if self.Host == "" {
		self.Host = TCP_DEFAULT_CONN_HOST
		return TCP_DEFAULT_CONN_HOST
	}
	return self.Host
}

func (self *TcpServer) getPort() string {
	if self.Port == "" {
		self.Port = TCP_DEFAULT_CONN_PORT
		return TCP_DEFAULT_CONN_PORT
	}
	return self.Port
}

func (self *TcpServer) getConnType() string {
	if self.ConnType == "" {
		self.ConnType = TCP_DEFAULT_CONN_TYPE
		return TCP_DEFAULT_CONN_TYPE
	}
	return self.ConnType
}

func (self TcpServer) Start() {
	go func() {
		// Check settings and apply defaults
		serv := fmt.Sprintf("%v:%v", self.getHost(), self.getPort())

		// Listen for incoming connections.
		l, err := net.Listen(self.getConnType(), serv)
		if err != nil {
			ServerLogger.Error("Error listening:", err.Error())
			panic(err)
		}
		ServerLogger.Info("Tcp Listening on " + serv)

		// Close the listener when the application closes.
		defer l.Close()

		for {
			// Listen for an incoming connection.
			conn, err := l.Accept()
			if err != nil {
				NetworkLogger.Error("Error accepting connection: ", err.Error())
				return
				// conn.Close()
			}

			NetworkLogger.Info("Connection open ", conn.RemoteAddr().String(), " [TCP]")

			// check for local connection
			if strings.Contains(conn.RemoteAddr().String(), "127.0.0.1") {
				// Handle connections in a new goroutine.
				go self.tcpClientHandler(conn)
			} else {
				// don't accept not local connections
				conn.Close()
			}

		}
	}()
}

// close tcp client
func (self *TcpServer) closeClient(conn net.Conn) {
	self.ActiveTcpClients--
	conn.Close()
}

// Handles incoming requests.
func (self *TcpServer) tcpClientHandler(conn net.Conn) {

	self.ActiveTcpClients++
	defer self.closeClient(conn)

	reader := bufio.NewReader(conn)
	tp := textproto.NewReader(reader)

	for {
		// will listen for message to process ending in newline (\n)
		message, _ := tp.ReadLine()

		// output message received
		NetworkLogger.Info("[TCP] Message Received: ", string([]byte(message)))

		// json parse message
		req := TcpMessage{}
		err := json.Unmarshal([]byte(message), &req)
		if err != nil {
			// invalid message
			// close connection
			// '\x04' end of transmittion character
			NetworkLogger.Warn("error:", err)
			resp := `{"status": "error", "error": "` + fmt.Sprintf("%v", err) + `",""}`
			conn.Write([]byte(resp + "\n"))
			NetworkLogger.Info("Connection closed", " [TCP]")
			return
		}

		switch {

		case req.Method == "ping":
			self.handleSuccess(`{"message": "pong", "version": "`+VERSION+`"}`, conn)

		case req.Method == "help":
			conn.Write([]byte("Methods:\n"))
			conn.Write([]byte("\t ping\n"))
			conn.Write([]byte("\t assign_datasource\n"))
			conn.Write([]byte("\t create_apikey\n"))
			conn.Write([]byte("\t insert_apikey\n"))
			conn.Write([]byte("\t insert_feature\n"))
			conn.Write([]byte("\t edit_feature\n"))
			conn.Write([]byte("\t create_datasource\n"))
			conn.Write([]byte("\t export_apikeys\n"))
			conn.Write([]byte("\t export_apikey\n"))
			conn.Write([]byte("\t export_datasources\n"))
			conn.Write([]byte("\t export_datasource\n"))
			conn.Write([]byte("\t import_file\n"))

		// APIKEYS
		case req.Method == "create_apikey":
			self.create_apikey(req, conn)

		case req.Method == "insert_apikey":
			self.insert_apikey(req, conn)

		case req.Method == "export_apikeys":
			self.export_apikeys(req, conn)

		case req.Method == "export_apikey":
			self.export_apikey(req, conn)

		// FEATURE
		case req.Method == "insert_feature":
			self.insert_feature(req, conn)

		case req.Method == "edit_feature":
			self.edit_feature(req, conn)

		// DATASOURCES
		case req.Method == "assign_datasource":
			self.assign_datasource(req, conn)

		case req.Method == "create_datasource":
			self.create_datasource(req, conn)

		case req.Method == "insert_layer":
			self.create_datasource(req, conn)

		case req.Method == "delete_layer":
			self.delete_datasource(req, conn)

		case req.Method == "delete_datasource":
			self.delete_datasource(req, conn)

		// TODO: ERROR HANDLING
		case req.Method == "export_datasources":
			self.export_datasources(req, conn)

		case req.Method == "export_datasource":
			self.export_datasource(req, conn)

		case req.Method == "export_layer":
			self.export_datasource(req, conn)

		case req.Method == "import_file":
			self.import_file(req, conn)

			/*

			   "export_datasource_snapshots
			   "export_datasource_by_snapshot"
			   "export_datasource_by_range"

			*/

		default:
			err := errors.New("Method not found")
			self.handleError(err, conn)
		}
	}
}

func (self TcpServer) handleError(err error, conn net.Conn) {
	conn.Write([]byte("{\"status\": \"error\", \"error\": \"" + err.Error() + "\"}\n"))
}

func (self TcpServer) handleSuccess(data string, conn net.Conn) {
	conn.Write([]byte("{\"status\": \"ok\", \"data\": " + data + "}\n"))
}

func (self TcpServer) missingParams(conn net.Conn) {
	err := errors.New("Missing required parameters")
	self.handleError(err, conn)
}

func (self TcpServer) mashalJsonFromStructResponse(data interface{}, conn net.Conn) {
	js, err := json.Marshal(data)
	if err != nil {
		self.handleError(err, conn)
		return
	}
	self.handleSuccess(string(js), conn)
}

// APIKEYS
func (self TcpServer) create_apikey(req TcpMessage, conn net.Conn) {
	// {"method":"create_apikey"}
	apikey := utils.NewAPIKey(12)
	customer := Customer{Apikey: apikey}
	err := DB.InsertCustomer(customer)
	if err != nil {
		self.handleError(err, conn)
		return
	}
	self.handleSuccess(`{"apikey": "`+apikey+`"}`, conn)
}

func (self TcpServer) insert_apikey(req TcpMessage, conn net.Conn) {
	// {"method": "insert_apikey"}
	if "" == req.Data.Apikey {
		self.missingParams(conn)
		return
	}
	customer := Customer{Apikey: req.Data.Apikey, Datasources: req.Data.Datasources}
	err := DB.InsertCustomer(customer)
	if err != nil {
		self.handleError(err, conn)
		return
	}
	self.handleSuccess(`{"apikey": "`+req.Data.Apikey+`"}`, conn)
}

func (self TcpServer) export_apikeys(req TcpMessage, conn net.Conn) {
	// {"method":"export_apikeys"}
	apikeys, err := DB.GetCustomers()
	if err != nil {
		self.handleError(err, conn)
		return
	}
	self.mashalJsonFromStructResponse(apikeys, conn)
}

func (self TcpServer) export_apikey(req TcpMessage, conn net.Conn) {
	// {"method":"export_apikey","apikey":"12dB6BlenIeB"}
	apikey, err := DB.GetCustomer(req.Apikey)
	if err != nil {
		self.handleError(err, conn)
		return
	}
	self.mashalJsonFromStructResponse(apikey, conn)
}

// DATASOURCES
func (self TcpServer) assign_datasource(req TcpMessage, conn net.Conn) {
	// {"method":"assign_datasource"}
	datasource_id := req.Datasource
	apikey := req.Apikey

	if "" == datasource_id || "" == apikey {
		self.missingParams(conn)
		return
	}

	customer, err := DB.GetCustomer(apikey)
	if err != nil {
		self.handleError(err, conn)
		return
	}

	_, err = GeoDB.GetLayer(datasource_id)
	if err != nil {
		self.handleError(err, conn)
		return
	}

	if !utils.StringInSlice(datasource_id, customer.Datasources) {
		customer.Datasources = append(customer.Datasources, datasource_id)
		DB.InsertCustomer(customer)
	}

	self.handleSuccess(`{}`, conn)
}

func (self TcpServer) create_datasource(req TcpMessage, conn net.Conn) {
	// {"method":"create_datasource"}
	datasource_id := req.Datasource
	var err error

	if "" != req.Datasource {
		err = GeoDB.InsertLayer(req.Datasource, req.Layer)
	} else {
		datasource_id, err = GeoDB.NewLayer()
	}

	if err != nil {
		self.handleError(err, conn)
		return
	}

	self.handleSuccess(`{"datasource_id":"`+datasource_id+`"}`, conn)
}

func (self TcpServer) export_datasources(req TcpMessage, conn net.Conn) {
	// {"method":"export_datasources"}
	layers, err := GeoDB.GetLayers()
	if err != nil {
		self.handleError(err, conn)
		return
	}
	self.mashalJsonFromStructResponse(layers, conn)
}

func (self TcpServer) export_datasource(req TcpMessage, conn net.Conn) {
	// {"method":"export_datasource","datasource":"20f3332781ea4d7b8d509d12517ac5fa"}
	layer, err := GeoDB.GetLayer(req.Datasource)
	if err != nil {
		self.handleError(err, conn)
		return
	}
	self.mashalJsonFromStructResponse(layer, conn)
}

func (self TcpServer) delete_datasource(req TcpMessage, conn net.Conn) {
	// {"method":"delete_layer", "datasource":"f79aac397a484998b94b56d345287096"}
	if "" == req.Datasource {
		self.missingParams(conn)
		return
	}
	err := GeoDB.DeleteLayer(req.Datasource)
	if err != nil {
		self.handleError(err, conn)
		return
	}
	self.handleSuccess(`{"datasource_id":"`+req.Data.Datasource+`", "message":"layer deleted"}`, conn)
}

// FEATURES
func (self TcpServer) insert_feature(req TcpMessage, conn net.Conn) {
	// {"method":"insert_feature"}
	if "" == req.Datasource {
		self.missingParams(conn)
		return
	}
	err := GeoDB.InsertFeature(req.Datasource, req.Feature)
	if err != nil {
		self.handleError(err, conn)
		return
	}
	self.handleSuccess(`{"datasource_id":"`+req.Data.Datasource+`", "message":"feature added"}`, conn)
}

func (self TcpServer) edit_feature(req TcpMessage, conn net.Conn) {
	// {"method":"edit_feature"}
	if "" == req.Datasource || "" == req.GeoId {
		self.missingParams(conn)
		return
	}
	err := GeoDB.EditFeature(req.Datasource, req.GeoId, req.Feature)
	if err != nil {
		self.handleError(err, conn)
		return
	}
	self.handleSuccess(`{"datasource_id":"`+req.Datasource+`", "message":"edited added"}`, conn)
}

// FILE
func (self TcpServer) import_file(req TcpMessage, conn net.Conn) {
	// {"method":"import_file","file":"springfield_projects_edit.geojson"}
	result, err := importDatasource(req.File)
	if err != nil {
		self.handleError(err, conn)
		return
	}
	self.handleSuccess(`{"datasource": "`+result+`"}`, conn)
}

func importDatasource(importFile string) (string, error) {
	//fmt.Println("Importing", importFile)
	// get geojson file
	var geojsonFile string
	// check if file exists
	if _, err := os.Stat(importFile); os.IsNotExist(err) {
		return "", err
	}
	// ERROR
	// CRASHES IF NO "." character FOUND
	ext := strings.Split(importFile, ".")[1]
	// convert shapefile
	if ext == "shp" {
		// Convert .shp to .geojson
		geojsonFile := strings.Replace(importFile, ".shp", ".geojson", -1)
		fmt.Println("ogr2ogr", "-f", "GeoJSON", "-t_srs", "crs:84", geojsonFile, importFile)
		out, err := exec.Command("ogr2ogr", "-f", "GeoJSON", "-t_srs", "crs:84", geojsonFile, importFile).Output()
		if err != nil {
			return fmt.Sprintf("%v", out), err
		}
	} else if ext == "geojson" {
		geojsonFile = importFile
	} else {
		return fmt.Sprintf("Unsupported file type: %v", ext), errors.New(fmt.Sprintf("Unsupported file type: %v", ext))
	}
	// Read .geojson file
	file, err := ioutil.ReadFile(geojsonFile)
	if err != nil {
		return "", err
	}
	// Unmarshal to geojson struct
	geojs, err := geojson.UnmarshalFeatureCollection(file)
	if err != nil {
		return "", err
	}
	// Create datasource
	ds, _ := utils.NewUUID()
	GeoDB.InsertLayer(ds, geojs)
	// Cleanup artifacts
	if geojsonFile != importFile {
		os.Remove(geojsonFile)
	}
	return ds, nil
}
<commit_msg>remove tile layers<commit_after>package geo_skeleton_server

import (
	"bufio"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"net/textproto"
	"os"
	"os/exec"
	"strings"

	"./utils"
	"github.com/paulmach/go.geojson"
)

const (
	TCP_DEFAULT_CONN_HOST = "localhost"
	TCP_DEFAULT_CONN_PORT = "3333"
	TCP_DEFAULT_CONN_TYPE = "tcp"
)

type TcpServer struct {
	Host             string
	Port             string
	ConnType         string
	ActiveTcpClients int
}

func (self *TcpServer) getHost() string {
	if self.Host == "" {
		self.Host = TCP_DEFAULT_CONN_HOST
		return TCP_DEFAULT_CONN_HOST
	}
	return self.Host
}

func (self *TcpServer) getPort() string {
	if self.Port == "" {
		self.Port = TCP_DEFAULT_CONN_PORT
		return TCP_DEFAULT_CONN_PORT
	}
	return self.Port
}

func (self *TcpServer) getConnType() string {
	if self.ConnType == "" {
		self.ConnType = TCP_DEFAULT_CONN_TYPE
		return TCP_DEFAULT_CONN_TYPE
	}
	return self.ConnType
}

func (self TcpServer) Start() {
	go func() {
		// Check settings and apply defaults
		serv := fmt.Sprintf("%v:%v", self.getHost(), self.getPort())

		// Listen for incoming connections.
		l, err := net.Listen(self.getConnType(), serv)
		if err != nil {
			ServerLogger.Error("Error listening:", err.Error())
			panic(err)
		}
		ServerLogger.Info("Tcp Listening on " + serv)

		// Close the listener when the application closes.
		defer l.Close()

		for {
			// Listen for an incoming connection.
			conn, err := l.Accept()
			if err != nil {
				NetworkLogger.Error("Error accepting connection: ", err.Error())
				return
				// conn.Close()
			}

			NetworkLogger.Info("Connection open ", conn.RemoteAddr().String(), " [TCP]")

			// check for local connection
			if strings.Contains(conn.RemoteAddr().String(), "127.0.0.1") {
				// Handle connections in a new goroutine.
				go self.tcpClientHandler(conn)
			} else {
				// don't accept not local connections
				conn.Close()
			}

		}
	}()
}

// close tcp client
func (self *TcpServer) closeClient(conn net.Conn) {
	self.ActiveTcpClients--
	conn.Close()
}

// Handles incoming requests.
func (self *TcpServer) tcpClientHandler(conn net.Conn) {

	self.ActiveTcpClients++
	defer self.closeClient(conn)

	reader := bufio.NewReader(conn)
	tp := textproto.NewReader(reader)

	for {
		// will listen for message to process ending in newline (\n)
		message, _ := tp.ReadLine()

		// output message received
		NetworkLogger.Info("[TCP] Message Received: ", string([]byte(message)))

		// json parse message
		req := TcpMessage{}
		err := json.Unmarshal([]byte(message), &req)
		if err != nil {
			// invalid message
			// close connection
			// '\x04' end of transmittion character
			NetworkLogger.Warn("error:", err)
			resp := `{"status": "error", "error": "` + fmt.Sprintf("%v", err) + `",""}`
			conn.Write([]byte(resp + "\n"))
			NetworkLogger.Info("Connection closed", " [TCP]")
			return
		}

		switch {

		case req.Method == "ping":
			self.handleSuccess(`{"message": "pong", "version": "`+VERSION+`"}`, conn)

		case req.Method == "help":
			conn.Write([]byte("Methods:\n"))
			conn.Write([]byte("\t ping\n"))
			conn.Write([]byte("\t assign_datasource\n"))
			conn.Write([]byte("\t create_apikey\n"))
			conn.Write([]byte("\t insert_apikey\n"))
			conn.Write([]byte("\t insert_feature\n"))
			conn.Write([]byte("\t edit_feature\n"))
			conn.Write([]byte("\t create_datasource\n"))
			conn.Write([]byte("\t export_apikeys\n"))
			conn.Write([]byte("\t export_apikey\n"))
			conn.Write([]byte("\t export_datasources\n"))
			conn.Write([]byte("\t export_datasource\n"))
			conn.Write([]byte("\t import_file\n"))

		// APIKEYS
		case req.Method == "create_apikey":
			self.create_apikey(req, conn)

		case req.Method == "insert_apikey":
			self.insert_apikey(req, conn)

		case req.Method == "export_apikeys":
			self.export_apikeys(req, conn)

		case req.Method == "export_apikey":
			self.export_apikey(req, conn)

		// FEATURE
		case req.Method == "insert_feature":
			self.insert_feature(req, conn)

		case req.Method == "edit_feature":
			self.edit_feature(req, conn)

		// DATASOURCES
		case req.Method == "assign_datasource":
			self.assign_datasource(req, conn)

		case req.Method == "create_datasource":
			self.create_datasource(req, conn)

		case req.Method == "insert_layer":
			self.create_datasource(req, conn)

		case req.Method == "delete_layer":
			self.delete_datasource(req, conn)

		case req.Method == "delete_datasource":
			self.delete_datasource(req, conn)

		// TODO: ERROR HANDLING
		case req.Method == "export_datasources":
			self.export_datasources(req, conn)

		case req.Method == "export_datasource":
			self.export_datasource(req, conn)

		case req.Method == "export_layer":
			self.export_datasource(req, conn)

		case req.Method == "import_file":
			self.import_file(req, conn)

			/*

			   "export_datasource_snapshots
			   "export_datasource_by_snapshot"
			   "export_datasource_by_range"

			*/

		default:
			err := errors.New("Method not found")
			self.handleError(err, conn)
		}
	}
}

func (self TcpServer) handleError(err error, conn net.Conn) {
	conn.Write([]byte("{\"status\": \"error\", \"error\": \"" + err.Error() + "\"}\n"))
}

func (self TcpServer) handleSuccess(data string, conn net.Conn) {
	conn.Write([]byte("{\"status\": \"ok\", \"data\": " + data + "}\n"))
}

func (self TcpServer) missingParams(conn net.Conn) {
	err := errors.New("Missing required parameters")
	self.handleError(err, conn)
}

func (self TcpServer) mashalJsonFromStructResponse(data interface{}, conn net.Conn) {
	js, err := json.Marshal(data)
	if err != nil {
		self.handleError(err, conn)
		return
	}
	self.handleSuccess(string(js), conn)
}

// APIKEYS
func (self TcpServer) create_apikey(req TcpMessage, conn net.Conn) {
	// {"method":"create_apikey"}
	apikey := utils.NewAPIKey(12)
	customer := Customer{Apikey: apikey}
	err := DB.InsertCustomer(customer)
	if err != nil {
		self.handleError(err, conn)
		return
	}
	self.handleSuccess(`{"apikey": "`+apikey+`"}`, conn)
}

func (self TcpServer) insert_apikey(req TcpMessage, conn net.Conn) {
	// {"method": "insert_apikey"}
	if "" == req.Data.Apikey {
		self.missingParams(conn)
		return
	}
	customer := Customer{Apikey: req.Data.Apikey, Datasources: req.Data.Datasources}
	err := DB.InsertCustomer(customer)
	if err != nil {
		self.handleError(err, conn)
		return
	}
	self.handleSuccess(`{"apikey": "`+req.Data.Apikey+`"}`, conn)
}

func (self TcpServer) export_apikeys(req TcpMessage, conn net.Conn) {
	// {"method":"export_apikeys"}
	apikeys, err := DB.GetCustomers()
	if err != nil {
		self.handleError(err, conn)
		return
	}
	self.mashalJsonFromStructResponse(apikeys, conn)
}

func (self TcpServer) export_apikey(req TcpMessage, conn net.Conn) {
	// {"method":"export_apikey","apikey":"12dB6BlenIeB"}
	apikey, err := DB.GetCustomer(req.Apikey)
	if err != nil {
		self.handleError(err, conn)
		return
	}
	self.mashalJsonFromStructResponse(apikey, conn)
}

// DATASOURCES
func (self TcpServer) assign_datasource(req TcpMessage, conn net.Conn) {
	// {"method":"assign_datasource"}
	datasource_id := req.Datasource
	apikey := req.Apikey

	if "" == datasource_id || "" == apikey {
		self.missingParams(conn)
		return
	}

	customer, err := DB.GetCustomer(apikey)
	if err != nil {
		self.handleError(err, conn)
		return
	}

	_, err = GeoDB.GetLayer(datasource_id)
	if err != nil {
		self.handleError(err, conn)
		return
	}

	if !utils.StringInSlice(datasource_id, customer.Datasources) {
		customer.Datasources = append(customer.Datasources, datasource_id)
		DB.InsertCustomer(customer)
	}

	self.handleSuccess(`{}`, conn)
}

func (self TcpServer) create_datasource(req TcpMessage, conn net.Conn) {
	// {"method":"create_datasource"}
	datasource_id := req.Datasource
	var err error

	fmt.Println(req.Datasource, req.Layer)

	if "" != req.Datasource {
		err = GeoDB.InsertLayer(req.Datasource, req.Layer)
	} else {
		datasource_id, err = GeoDB.NewLayer()
	}

	if err != nil {
		self.handleError(err, conn)
		return
	}

	self.handleSuccess(`{"datasource_id":"`+datasource_id+`"}`, conn)
}

func (self TcpServer) export_datasources(req TcpMessage, conn net.Conn) {
	// {"method":"export_datasources"}
	layers, err := GeoDB.GetLayers()
	if err != nil {
		self.handleError(err, conn)
		return
	}
	self.mashalJsonFromStructResponse(layers, conn)
}

func (self TcpServer) export_datasource(req TcpMessage, conn net.Conn) {
	// {"method":"export_datasource","datasource":"20f3332781ea4d7b8d509d12517ac5fa"}
	layer, err := GeoDB.GetLayer(req.Datasource)
	if err != nil {
		self.handleError(err, conn)
		return
	}
	self.mashalJsonFromStructResponse(layer, conn)
}

func (self TcpServer) delete_datasource(req TcpMessage, conn net.Conn) {
	// {"method":"delete_layer", "datasource":"f79aac397a484998b94b56d345287096"}
	if "" == req.Datasource {
		self.missingParams(conn)
		return
	}
	err := GeoDB.DeleteLayer(req.Datasource)
	if err != nil {
		self.handleError(err, conn)
		return
	}
	self.handleSuccess(`{"datasource_id":"`+req.Data.Datasource+`", "message":"layer deleted"}`, conn)
}

// FEATURES
func (self TcpServer) insert_feature(req TcpMessage, conn net.Conn) {
	// {"method":"insert_feature"}
	if "" == req.Datasource {
		self.missingParams(conn)
		return
	}
	err := GeoDB.InsertFeature(req.Datasource, req.Feature)
	if err != nil {
		self.handleError(err, conn)
		return
	}
	self.handleSuccess(`{"datasource_id":"`+req.Data.Datasource+`", "message":"feature added"}`, conn)
}

func (self TcpServer) edit_feature(req TcpMessage, conn net.Conn) {
	// {"method":"edit_feature"}
	if "" == req.Datasource || "" == req.GeoId {
		self.missingParams(conn)
		return
	}
	err := GeoDB.EditFeature(req.Datasource, req.GeoId, req.Feature)
	if err != nil {
		self.handleError(err, conn)
		return
	}
	self.handleSuccess(`{"datasource_id":"`+req.Datasource+`", "message":"edited added"}`, conn)
}

// FILE
func (self TcpServer) import_file(req TcpMessage, conn net.Conn) {
	// {"method":"import_file","file":"springfield_projects_edit.geojson"}
	result, err := importDatasource(req.File)
	if err != nil {
		self.handleError(err, conn)
		return
	}
	self.handleSuccess(`{"datasource": "`+result+`"}`, conn)
}

func importDatasource(importFile string) (string, error) {
	//fmt.Println("Importing", importFile)
	// get geojson file
	var geojsonFile string
	// check if file exists
	if _, err := os.Stat(importFile); os.IsNotExist(err) {
		return "", err
	}
	// ERROR
	// CRASHES IF NO "." character FOUND
	ext := strings.Split(importFile, ".")[1]
	// convert shapefile
	if ext == "shp" {
		// Convert .shp to .geojson
		geojsonFile := strings.Replace(importFile, ".shp", ".geojson", -1)
		fmt.Println("ogr2ogr", "-f", "GeoJSON", "-t_srs", "crs:84", geojsonFile, importFile)
		out, err := exec.Command("ogr2ogr", "-f", "GeoJSON", "-t_srs", "crs:84", geojsonFile, importFile).Output()
		if err != nil {
			return fmt.Sprintf("%v", out), err
		}
	} else if ext == "geojson" {
		geojsonFile = importFile
	} else {
		return fmt.Sprintf("Unsupported file type: %v", ext), errors.New(fmt.Sprintf("Unsupported file type: %v", ext))
	}
	// Read .geojson file
	file, err := ioutil.ReadFile(geojsonFile)
	if err != nil {
		return "", err
	}
	// Unmarshal to geojson struct
	geojs, err := geojson.UnmarshalFeatureCollection(file)
	if err != nil {
		return "", err
	}
	// Create datasource
	ds, _ := utils.NewUUID()
	GeoDB.InsertLayer(ds, geojs)
	// Cleanup artifacts
	if geojsonFile != importFile {
		os.Remove(geojsonFile)
	}
	return ds, nil
}
<|endoftext|>


<commit_before>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package feedback

import (
	"encoding/hex"
	"fmt"
	"log"

	"github.com/golang/dep/gps"
)

const (
	// ConsTypeConstraint represents a constraint
	ConsTypeConstraint = "constraint"

	// ConsTypeHint represents a constraint type hint
	ConsTypeHint = "hint"

	// DepTypeDirect represents a direct dependency
	DepTypeDirect = "direct dep"

	// DepTypeTransitive represents a transitive dependency,
	// or a dependency of a dependency
	DepTypeTransitive = "transitive dep"

	// DepTypeImported represents a dependency imported by an external tool
	DepTypeImported = "imported dep"
)

// ConstraintFeedback holds project constraint feedback data
type ConstraintFeedback struct {
	Constraint, LockedVersion, Revision, ConstraintType, DependencyType, ProjectPath string
}

// NewConstraintFeedback builds a feedback entry for a constraint in the manifest.
func NewConstraintFeedback(pc gps.ProjectConstraint, depType string) *ConstraintFeedback {
	cf := &ConstraintFeedback{
		Constraint:     pc.Constraint.String(),
		ProjectPath:    string(pc.Ident.ProjectRoot),
		DependencyType: depType,
	}

	if _, ok := pc.Constraint.(gps.Revision); ok {
		cf.ConstraintType = ConsTypeHint
	} else {
		cf.ConstraintType = ConsTypeConstraint
	}

	return cf
}

// NewLockedProjectFeedback builds a feedback entry for a project in the lock.
func NewLockedProjectFeedback(lp gps.LockedProject, depType string) *ConstraintFeedback {
	cf := &ConstraintFeedback{
		ProjectPath:    string(lp.Ident().ProjectRoot),
		DependencyType: depType,
	}

	switch vt := lp.Version().(type) {
	case gps.PairedVersion:
		cf.LockedVersion = vt.String()
		cf.Revision = vt.Revision().String()
	case gps.UnpairedVersion: // Logically this should never occur, but handle for completeness sake
		cf.LockedVersion = vt.String()
	case gps.Revision:
		cf.Revision = vt.String()
	}

	return cf
}

// LogFeedback logs feedback on changes made to the manifest or lock.
func (cf ConstraintFeedback) LogFeedback(logger *log.Logger) {
	if cf.Constraint != "" {
		logger.Printf("  %v", GetUsingFeedback(cf.Constraint, cf.ConstraintType, cf.DependencyType, cf.ProjectPath))
	}
	if cf.Revision != "" {
		logger.Printf("  %v", GetLockingFeedback(cf.LockedVersion, cf.Revision, cf.DependencyType, cf.ProjectPath))
	}
}

type brokenImport struct {
	projectPath string
	version     *gps.StringDiff
	revision    *gps.StringDiff
}

func (bi brokenImport) String() string {
	var pr string
	var cr string
	if bi.revision != nil {
		pr = fmt.Sprintf("(%s)", trimSHA(bi.revision.Previous))
		cr = fmt.Sprintf("(%s)", trimSHA(bi.revision.Current))
	}

	return fmt.Sprintf("%v %s for %s. Locking in %v %s",
		bi.version.Previous,
		pr,
		bi.projectPath,
		bi.version.Current,
		cr,
	)
}

// ConstraintFeedback holds project constraint feedback data
type BrokenImportFeedback struct {
	brokenImports []brokenImport
}

func NewBrokenImportFeedback(ld *gps.LockDiff) *BrokenImportFeedback {
	bi := &BrokenImportFeedback{}
	for _, lpd := range ld.Modify {
		bi.brokenImports = append(bi.brokenImports, brokenImport{
			projectPath: string(lpd.Name),
			version:     lpd.Version,
			revision:    lpd.Revision,
		})
	}

	return bi
}

func (b BrokenImportFeedback) LogFeedback(logger *log.Logger) {
	for _, bi := range b.brokenImports {
		logger.Printf("Warning: Unable to preserve imported lock %v\n", bi)
	}
}

// GetUsingFeedback returns a dependency "using" feedback message. For example:
//
//    Using ^1.0.0 as constraint for direct dep github.com/foo/bar
//    Using 1b8edb3 as hint for direct dep github.com/bar/baz
func GetUsingFeedback(version, consType, depType, projectPath string) string {
	if depType == DepTypeImported {
		return fmt.Sprintf("Using %s as initial %s for %s %s", version, consType, depType, projectPath)
	}
	return fmt.Sprintf("Using %s as %s for %s %s", version, consType, depType, projectPath)
}

// GetLockingFeedback returns a dependency "locking" feedback message. For
// example:
//
//    Locking in v1.1.4 (bc29b4f) for direct dep github.com/foo/bar
//    Locking in master (436f39d) for transitive dep github.com/baz/qux
func GetLockingFeedback(version, revision, depType, projectPath string) string {
	// Check if it's a valid SHA1 digest and trim to 7 characters.
	if len(revision) == 40 {
		if _, err := hex.DecodeString(revision); err == nil {
			// Valid SHA1 digest
			revision = revision[0:7]
		}
	}

	if depType == DepTypeImported {
		if version == "" {
			version = "*"
		}
		return fmt.Sprintf("Trying %s (%s) as initial lock for %s %s", version, revision, depType, projectPath)
	}
	return fmt.Sprintf("Locking in %s (%s) for %s %s", version, revision, depType, projectPath)
}

func trimSHA(revision string) string {
	if len(revision) == 40 {
		if _, err := hex.DecodeString(revision); err == nil {
			// Valid SHA1 digest
			revision = revision[0:7]
		}
	}

	return revision
}
<commit_msg>Add comments for new methods<commit_after>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package feedback

import (
	"encoding/hex"
	"fmt"
	"log"

	"github.com/golang/dep/gps"
)

const (
	// ConsTypeConstraint represents a constraint
	ConsTypeConstraint = "constraint"

	// ConsTypeHint represents a constraint type hint
	ConsTypeHint = "hint"

	// DepTypeDirect represents a direct dependency
	DepTypeDirect = "direct dep"

	// DepTypeTransitive represents a transitive dependency,
	// or a dependency of a dependency
	DepTypeTransitive = "transitive dep"

	// DepTypeImported represents a dependency imported by an external tool
	DepTypeImported = "imported dep"
)

// ConstraintFeedback holds project constraint feedback data
type ConstraintFeedback struct {
	Constraint, LockedVersion, Revision, ConstraintType, DependencyType, ProjectPath string
}

// NewConstraintFeedback builds a feedback entry for a constraint in the manifest.
func NewConstraintFeedback(pc gps.ProjectConstraint, depType string) *ConstraintFeedback {
	cf := &ConstraintFeedback{
		Constraint:     pc.Constraint.String(),
		ProjectPath:    string(pc.Ident.ProjectRoot),
		DependencyType: depType,
	}

	if _, ok := pc.Constraint.(gps.Revision); ok {
		cf.ConstraintType = ConsTypeHint
	} else {
		cf.ConstraintType = ConsTypeConstraint
	}

	return cf
}

// NewLockedProjectFeedback builds a feedback entry for a project in the lock.
func NewLockedProjectFeedback(lp gps.LockedProject, depType string) *ConstraintFeedback {
	cf := &ConstraintFeedback{
		ProjectPath:    string(lp.Ident().ProjectRoot),
		DependencyType: depType,
	}

	switch vt := lp.Version().(type) {
	case gps.PairedVersion:
		cf.LockedVersion = vt.String()
		cf.Revision = vt.Revision().String()
	case gps.UnpairedVersion: // Logically this should never occur, but handle for completeness sake
		cf.LockedVersion = vt.String()
	case gps.Revision:
		cf.Revision = vt.String()
	}

	return cf
}

// LogFeedback logs feedback on changes made to the manifest or lock.
func (cf ConstraintFeedback) LogFeedback(logger *log.Logger) {
	if cf.Constraint != "" {
		logger.Printf("  %v", GetUsingFeedback(cf.Constraint, cf.ConstraintType, cf.DependencyType, cf.ProjectPath))
	}
	if cf.Revision != "" {
		logger.Printf("  %v", GetLockingFeedback(cf.LockedVersion, cf.Revision, cf.DependencyType, cf.ProjectPath))
	}
}

type brokenImport struct {
	projectPath string
	version     *gps.StringDiff
	revision    *gps.StringDiff
}

func (bi brokenImport) String() string {
	var pr string
	var cr string
	if bi.revision != nil {
		pr = fmt.Sprintf("(%s)", trimSHA(bi.revision.Previous))
		cr = fmt.Sprintf("(%s)", trimSHA(bi.revision.Current))
	}

	return fmt.Sprintf("%v %s for %s. Locking in %v %s",
		bi.version.Previous,
		pr,
		bi.projectPath,
		bi.version.Current,
		cr,
	)
}

// BrokenImportFeedback holds problematic lock feedback data
type BrokenImportFeedback struct {
	brokenImports []brokenImport
}

// NewBrokenImportFeedback builds a feedback entry for problems with imports from a diff of the pre- and post- solved locks
func NewBrokenImportFeedback(ld *gps.LockDiff) *BrokenImportFeedback {
	bi := &BrokenImportFeedback{}
	for _, lpd := range ld.Modify {
		bi.brokenImports = append(bi.brokenImports, brokenImport{
			projectPath: string(lpd.Name),
			version:     lpd.Version,
			revision:    lpd.Revision,
		})
	}

	return bi
}

// LogFeedback logs a warning for all changes between the initially imported and post- solve locks
func (b BrokenImportFeedback) LogFeedback(logger *log.Logger) {
	for _, bi := range b.brokenImports {
		logger.Printf("Warning: Unable to preserve imported lock %v\n", bi)
	}
}

// GetUsingFeedback returns a dependency "using" feedback message. For example:
//
//    Using ^1.0.0 as constraint for direct dep github.com/foo/bar
//    Using 1b8edb3 as hint for direct dep github.com/bar/baz
func GetUsingFeedback(version, consType, depType, projectPath string) string {
	if depType == DepTypeImported {
		return fmt.Sprintf("Using %s as initial %s for %s %s", version, consType, depType, projectPath)
	}
	return fmt.Sprintf("Using %s as %s for %s %s", version, consType, depType, projectPath)
}

// GetLockingFeedback returns a dependency "locking" feedback message. For
// example:
//
//    Locking in v1.1.4 (bc29b4f) for direct dep github.com/foo/bar
//    Locking in master (436f39d) for transitive dep github.com/baz/qux
func GetLockingFeedback(version, revision, depType, projectPath string) string {
	// Check if it's a valid SHA1 digest and trim to 7 characters.
	if len(revision) == 40 {
		if _, err := hex.DecodeString(revision); err == nil {
			// Valid SHA1 digest
			revision = revision[0:7]
		}
	}

	if depType == DepTypeImported {
		if version == "" {
			version = "*"
		}
		return fmt.Sprintf("Trying %s (%s) as initial lock for %s %s", version, revision, depType, projectPath)
	}
	return fmt.Sprintf("Locking in %s (%s) for %s %s", version, revision, depType, projectPath)
}

func trimSHA(revision string) string {
	if len(revision) == 40 {
		if _, err := hex.DecodeString(revision); err == nil {
			// Valid SHA1 digest
			revision = revision[0:7]
		}
	}

	return revision
}
<|endoftext|>