<commit_before>// Copyright 2014, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package vtgate

// This is a V3 file. Do not intermix with V2.

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"sync"

	log "github.com/golang/glog"
	"golang.org/x/net/context"

	"github.com/youtube/vitess/go/acl"
	"github.com/youtube/vitess/go/cache"
	"github.com/youtube/vitess/go/vt/topo"
	"github.com/youtube/vitess/go/vt/vtgate/engine"
	"github.com/youtube/vitess/go/vt/vtgate/planbuilder"
	"github.com/youtube/vitess/go/vt/vtgate/vindexes"
)

// Planner is used to compute the plan. It contains
// the vschema, and has a cache of previous computed plans.
type Planner struct {
	serv    topo.SrvTopoServer
	cell    string
	mu      sync.Mutex
	vschema *vindexes.VSchema
	plans   *cache.LRUCache
}

var once sync.Once

// NewPlanner creates a new planner for VTGate.
// It will watch the vschema in the topology until the ctx is closed.
func NewPlanner(ctx context.Context, serv topo.SrvTopoServer, cell string, cacheSize int) *Planner {
	plr := &Planner{
		serv:  serv,
		cell:  cell,
		plans: cache.NewLRUCache(int64(cacheSize)),
	}
	plr.WatchVSchema(ctx)
	once.Do(func() {
		http.Handle("/debug/query_plans", plr)
		http.Handle("/debug/vschema", plr)
	})
	return plr
}

// WatchVSchema watches the VSchema from the topo. The function does
// not return an error. It instead logs warnings on failure.
// We get the list of keyspaces to watch at the beginning (because it's easier)
// and we don't watch that. So when adding a new keyspace, for now,
// vtgate needs to be restarted (after having rebuilt the serving graph).
// This could be fixed by adding a WatchSrvKeyspaceNames API to topo.Server,
// and when it triggers we'd diff the values, and terminate the missing
// SrvKeyspaces, and start watching the new SrvKeyspaces.
func (plr *Planner) WatchVSchema(ctx context.Context) {
	keyspaces, err := plr.serv.GetSrvKeyspaceNames(ctx, plr.cell)
	if err != nil {
		log.Warningf("Error loading vschema: could not read keyspaces: %v", err)
		return
	}

	// we will wait until we get the first value for each keyspace
	// before returning
	var wg sync.WaitGroup

	// mu protects formal
	var mu sync.Mutex
	formal := &vindexes.VSchemaFormal{
		Keyspaces: make(map[string]vindexes.KeyspaceFormal),
	}
	processKeyspace := func(keyspace, kschema string) error {
		// Note we use a closure here to allow for defer()
		// to work properly, and also to let the caller
		// do more things even if this fails.

		// unpack the new VSchema, or skip it
		var kformal vindexes.KeyspaceFormal
		if err := json.Unmarshal([]byte(kschema), &kformal); err != nil {
			return fmt.Errorf("Error unmarshalling vschema for keyspace %s: %v", keyspace, err)
		}

		// rebuild the new component
		mu.Lock()
		defer mu.Unlock()
		formal.Keyspaces[keyspace] = kformal
		vschema, err := vindexes.BuildVSchema(formal)
		if err != nil {
			return fmt.Errorf("Error creating VSchema: %v", err)
		}

		plr.mu.Lock()
		plr.vschema = vschema
		plr.mu.Unlock()
		plr.plans.Clear()

		return nil
	}

	lastValueIndex := len(keyspaces) - 1
	for i, keyspace := range keyspaces {
		wg.Add(1)
		go func(i int, keyspace string) {
			gotFirstValue := false

			notifications, err := plr.serv.WatchVSchema(ctx, keyspace)
			if err != nil {
				log.Warningf("Error watching vschema for keyspace %s, will not watch it: %v", keyspace, err)
				wg.Done()
				return
			}

			for kschema := range notifications {
				err := processKeyspace(keyspace, kschema)

				if !gotFirstValue {
					gotFirstValue = true

					// only log the last error, as other
					// errors may need the next values
					if err != nil && i == lastValueIndex {
						log.Warningf("%v", err)
					}

					wg.Done()
				} else {
					// always log a refresh that fails
					if err != nil {
						log.Warningf("%v", err)
					}
				}
			}
		}(i, keyspace)
	}

	wg.Wait()
}

// VSchema returns the VSchema.
func (plr *Planner) VSchema() *vindexes.VSchema {
	plr.mu.Lock()
	defer plr.mu.Unlock()
	return plr.vschema
}

// GetPlan computes the plan for the given query. If one is in
// the cache, it reuses it.
func (plr *Planner) GetPlan(sql, keyspace string) (*engine.Plan, error) {
	if plr.VSchema() == nil {
		return nil, errors.New("vschema not initialized")
	}
	key := sql
	if keyspace != "" {
		key = keyspace + ":" + sql
	}
	if result, ok := plr.plans.Get(key); ok {
		return result.(*engine.Plan), nil
	}
	plan, err := planbuilder.Build(sql, &wrappedVSchema{
		vschema:  plr.VSchema(),
		keyspace: keyspace,
	})
	if err != nil {
		return nil, err
	}
	plr.plans.Set(sql, plan)
	return plan, nil
}

// ServeHTTP shows the current plans in the query cache.
func (plr *Planner) ServeHTTP(response http.ResponseWriter, request *http.Request) {
	if err := acl.CheckAccessHTTP(request, acl.DEBUGGING); err != nil {
		acl.SendError(response, err)
		return
	}
	if request.URL.Path == "/debug/query_plans" {
		keys := plr.plans.Keys()
		response.Header().Set("Content-Type", "text/plain")
		response.Write([]byte(fmt.Sprintf("Length: %d\n", len(keys))))
		for _, v := range keys {
			response.Write([]byte(fmt.Sprintf("%#v\n", v)))
			if plan, ok := plr.plans.Peek(v); ok {
				if b, err := json.MarshalIndent(plan, "", "  "); err != nil {
					response.Write([]byte(err.Error()))
				} else {
					response.Write(b)
				}
				response.Write(([]byte)("\n\n"))
			}
		}
	} else if request.URL.Path == "/debug/vschema" {
		response.Header().Set("Content-Type", "application/json; charset=utf-8")
		b, err := json.MarshalIndent(plr.VSchema(), "", " ")
		if err != nil {
			response.Write([]byte(err.Error()))
			return
		}
		buf := bytes.NewBuffer(nil)
		json.HTMLEscape(buf, b)
		response.Write(buf.Bytes())
	} else {
		response.WriteHeader(http.StatusNotFound)
	}
}

type wrappedVSchema struct {
	vschema  *vindexes.VSchema
	keyspace string
}

func (vs *wrappedVSchema) Find(keyspace, tablename string) (table *vindexes.Table, err error) {
	if keyspace == "" {
		keyspace = vs.keyspace
	}
	return vs.vschema.Find(keyspace, tablename)
}
<commit_msg>Better version of the keyspace watcher in v3.<commit_after>// Copyright 2014, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package vtgate

// This is a V3 file. Do not intermix with V2.

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"sync"

	log "github.com/golang/glog"
	"golang.org/x/net/context"

	"github.com/youtube/vitess/go/acl"
	"github.com/youtube/vitess/go/cache"
	"github.com/youtube/vitess/go/vt/topo"
	"github.com/youtube/vitess/go/vt/vtgate/engine"
	"github.com/youtube/vitess/go/vt/vtgate/planbuilder"
	"github.com/youtube/vitess/go/vt/vtgate/vindexes"
)

// Planner is used to compute the plan. It contains
// the vschema, and has a cache of previous computed plans.
type Planner struct {
	serv    topo.SrvTopoServer
	cell    string
	mu      sync.Mutex
	vschema *vindexes.VSchema
	plans   *cache.LRUCache
}

var once sync.Once

// NewPlanner creates a new planner for VTGate.
// It will watch the vschema in the topology until the ctx is closed.
func NewPlanner(ctx context.Context, serv topo.SrvTopoServer, cell string, cacheSize int) *Planner {
	plr := &Planner{
		serv:  serv,
		cell:  cell,
		plans: cache.NewLRUCache(int64(cacheSize)),
	}
	plr.WatchVSchema(ctx)
	once.Do(func() {
		http.Handle("/debug/query_plans", plr)
		http.Handle("/debug/vschema", plr)
	})
	return plr
}

// WatchVSchema watches the VSchema from the topo. The function does
// not return an error. It instead logs warnings on failure.
// We get the list of keyspaces to watch at the beginning (because it's easier)
// and we don't watch that. So when adding a new keyspace, for now,
// vtgate needs to be restarted (after having rebuilt the serving graph).
// This could be fixed by adding a WatchSrvKeyspaceNames API to topo.Server,
// and when it triggers we'd diff the values, and terminate the missing
// SrvKeyspaces, and start watching the new SrvKeyspaces.
func (plr *Planner) WatchVSchema(ctx context.Context) {
	keyspaces, err := plr.serv.GetSrvKeyspaceNames(ctx, plr.cell)
	if err != nil {
		log.Warningf("Error loading vschema: could not read keyspaces: %v", err)
		return
	}

	// mu protects formal
	var mu sync.Mutex
	formal := &vindexes.VSchemaFormal{
		Keyspaces: make(map[string]vindexes.KeyspaceFormal),
	}

	notifications := make(map[string]<-chan string)
	for _, keyspace := range keyspaces {
		n, err := plr.serv.WatchVSchema(ctx, keyspace)
		if err != nil {
			log.Warningf("Error watching vschema for keyspace %s (will not watch it): %v", keyspace, err)
			continue
		}

		firstValue, ok := <-n
		if !ok {
			log.Warningf("Error getting first value for keyspace %s (will not watch it): %v", keyspace, err)
			continue
		}

		// unpack the first value, save it
		var kformal vindexes.KeyspaceFormal
		if err := json.Unmarshal([]byte(firstValue), &kformal); err != nil {
			log.Warningf("Error unmarshalling vschema for keyspace %s (will keep watching it): %v", keyspace, err)
		} else {
			formal.Keyspaces[keyspace] = kformal
		}

		notifications[keyspace] = n
	}

	// we got the initial set of values, now build the first version
	vschema, err := vindexes.BuildVSchema(formal)
	if err != nil {
		log.Warningf("Error creating initial VSchema (will keep watching keyspaces): %v", err)
	} else {
		plr.mu.Lock()
		plr.vschema = vschema
		plr.mu.Unlock()
	}

	processKeyspace := func(keyspace, kschema string) error {
		// Note we use a closure here to allow for defer()
		// to work properly

		// unpack the new VSchema, or skip it
		var kformal vindexes.KeyspaceFormal
		if err := json.Unmarshal([]byte(kschema), &kformal); err != nil {
			return fmt.Errorf("Error unmarshalling vschema for keyspace %s (will wait for a better version): %v", keyspace, err)
		}

		// rebuild the new component
		mu.Lock()
		defer mu.Unlock()
		formal.Keyspaces[keyspace] = kformal
		vschema, err := vindexes.BuildVSchema(formal)
		if err != nil {
			return fmt.Errorf("Error creating VSchema (will try again when newer parts come in): %v", err)
		}

		plr.mu.Lock()
		plr.vschema = vschema
		plr.mu.Unlock()
		plr.plans.Clear()

		return nil
	}

	// start the background watches
	for keyspace, n := range notifications {
		go func(keyspace string, n <-chan string) {
			for kschema := range n {
				if err := processKeyspace(keyspace, kschema); err != nil {
					log.Warning("%v", err)
				}
			}
		}(keyspace, n)
	}
}

// VSchema returns the VSchema.
func (plr *Planner) VSchema() *vindexes.VSchema {
	plr.mu.Lock()
	defer plr.mu.Unlock()
	return plr.vschema
}

// GetPlan computes the plan for the given query. If one is in
// the cache, it reuses it.
func (plr *Planner) GetPlan(sql, keyspace string) (*engine.Plan, error) {
	if plr.VSchema() == nil {
		return nil, errors.New("vschema not initialized")
	}
	key := sql
	if keyspace != "" {
		key = keyspace + ":" + sql
	}
	if result, ok := plr.plans.Get(key); ok {
		return result.(*engine.Plan), nil
	}
	plan, err := planbuilder.Build(sql, &wrappedVSchema{
		vschema:  plr.VSchema(),
		keyspace: keyspace,
	})
	if err != nil {
		return nil, err
	}
	plr.plans.Set(sql, plan)
	return plan, nil
}

// ServeHTTP shows the current plans in the query cache.
func (plr *Planner) ServeHTTP(response http.ResponseWriter, request *http.Request) {
	if err := acl.CheckAccessHTTP(request, acl.DEBUGGING); err != nil {
		acl.SendError(response, err)
		return
	}
	if request.URL.Path == "/debug/query_plans" {
		keys := plr.plans.Keys()
		response.Header().Set("Content-Type", "text/plain")
		response.Write([]byte(fmt.Sprintf("Length: %d\n", len(keys))))
		for _, v := range keys {
			response.Write([]byte(fmt.Sprintf("%#v\n", v)))
			if plan, ok := plr.plans.Peek(v); ok {
				if b, err := json.MarshalIndent(plan, "", "  "); err != nil {
					response.Write([]byte(err.Error()))
				} else {
					response.Write(b)
				}
				response.Write(([]byte)("\n\n"))
			}
		}
	} else if request.URL.Path == "/debug/vschema" {
		response.Header().Set("Content-Type", "application/json; charset=utf-8")
		b, err := json.MarshalIndent(plr.VSchema(), "", " ")
		if err != nil {
			response.Write([]byte(err.Error()))
			return
		}
		buf := bytes.NewBuffer(nil)
		json.HTMLEscape(buf, b)
		response.Write(buf.Bytes())
	} else {
		response.WriteHeader(http.StatusNotFound)
	}
}

type wrappedVSchema struct {
	vschema  *vindexes.VSchema
	keyspace string
}

func (vs *wrappedVSchema) Find(keyspace, tablename string) (table *vindexes.Table, err error) {
	if keyspace == "" {
		keyspace = vs.keyspace
	}
	return vs.vschema.Find(keyspace, tablename)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"github.com/vaughan0/go-ini"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"

	//	"errors"
	//	"log"
	//	"net"
	//	"crypto/sha512"
	//	"encoding/hex"
)

//	defaults:
var (
	WORKERS 	int64	= 10    //Number of workers
	IMGDIR  	string	= "img" //default download directory
	TAG     	string	= ""    //default string is empty, it can only ge extracted from command line
	STARTPAGE	int		= 1		//default start page, derpiboo.ru 1-indexed
	STOPPAGE	int		= 0		//default stop page, would stop parsing json when it ends
	)

func main() {

	config, err := ini.LoadFile("config.ini") // Loading default config file and checking for various errors.
	if os.IsNotExist(err) {
		panic("config.ini does not exist, create it")
	}

	if err != nil {
		panic(err)
	}

	//Getting stuff from config, overwriting defaults

	key, ok := config.Get("main", "key")
	if !ok {
		panic("'key' variable missing from 'main' section")
	}

	W_temp, _ := config.Get("main", "workers")
	if W_temp != "" {
		WORKERS, err = strconv.ParseInt(W_temp, 10, 0)
		if err != nil {
			fmt.Println("Wrong configuration: Amount of workers is not a number")
			os.Exit(1)
		}
	}

	ID_temp, _ := config.Get("main", "downdir")
	if ID_temp != "" {
		IMGDIR = ID_temp
	}

	//here shall be flag parser

	flag.StringVar(&TAG, "t", TAG, "Tag to download: Replace spaces with \"+\".")
	flag.Parse()

	fmt.Println("Derpiboo.ru Downloader version 0.0.6 \nWorking")

	length := flag.NArg()
	if length == 0 && TAG == "" {
		fmt.Println("Nothing to download, bye!")
		os.Exit(0)
	}

	//	creating directory for downloads if not yet done
	if err := os.MkdirAll(IMGDIR, 0777); err != nil {
		panic(err)
	}

	imgdat := make(chan Image, WORKERS)

	if TAG == "" {

		imgid := flag.Arg(length - 1)

		//	fmt.Println(key) //Just checking that I am not wrong

		_, err = strconv.ParseInt(imgid, 10, 0)
		if err != nil {
			fmt.Println("Wrong input: can not parse", imgid, "as a number")
			os.Exit(1)
		}

		fmt.Println("Processing image No " + imgid)

		parseImg(imgdat, imgid, key)

	}	else	{
	//	and here we parse tag and abuse stuff horribly

		fmt.Println("trying to process tags")
		parseTag(imgdat, TAG, key)
	}

	dlimage(imgdat)

}

type Image struct {
	url      string
	filename string
	hash     string
}

func parseImg(imgchan chan<- Image, imgid string, key string) {

	source := "http://derpiboo.ru/" + imgid + ".json?nofav=&nocomments="
	if key != "" {
		source = source + "&key=" + key
	}

	//	fmt.Println(source)

	resp, err := http.Get(source) //Getting our nice http response. Needs checking for 404 and other responses that are... less expected
	if err != nil {
		panic(err)
	}

	defer resp.Body.Close() //and not forgetting to close it when it's done

	var dat map[string]interface{}

	body, err := ioutil.ReadAll(resp.Body) //stolen from official documentation
	if err != nil {
		panic(err)
	}

	fmt.Println(body)

	if err := json.Unmarshal(body, &dat); //transforming json into native map

	err != nil {
		panic(err)
	}
	var imgdata Image
	imgdata.url = "http:" + dat["image"].(string)
	imgdata.hash = dat["sha512_hash"].(string) //for the future and checking that we got file right
	imgdata.filename = strconv.FormatFloat(dat["id_number"].(float64), 'f', -1, 64) + "." + dat["file_name"].(string) + "." + dat["original_format"].(string)

	//	fmt.Println(strconv.FormatFloat(dat["id_number"].(float64), 'f', -1, 64))

	//	fmt.Println(dat)

	//	for now and troubleshooting
	//	fmt.Println(imgdata.url)
	//	fmt.Println(imgdata.hash)
	//	fmt.Println(imgdata.filename)

	imgchan <- imgdata

	return
}

func dlimage(imgchan <-chan Image) {
	//	fmt.Println("reading channel")

	imgdata := <-imgchan

	if imgdata.filename == "" { fmt.Println("Empty filename. Oops?"); return }

	fmt.Println("Saving as ", imgdata.filename)
	PathSep, _ := strconv.Unquote(strconv.QuoteRune(os.PathSeparator))

	output, err := os.Create(IMGDIR + PathSep + imgdata.filename)
	if err != err {
		panic(err)
	}
	defer output.Close()

	response, err := http.Get(imgdata.url)
	if err != nil {
		panic(err)
		fmt.Println("Error while downloading", imgdata.url, "-", err)
		return
	}
	defer response.Body.Close()

	io.Copy(output, response.Body)

	/*	hash := sha512.New()
		io.Copy(hash, response.Body)
		b := make([]byte, hash.Size())
		hash.Sum(b[:0])

		fmt.Println("\n", hex.EncodeToString(b), "\n", imgdata.hash )

		if hex.EncodeToString(b) == imgdata.hash {
			fmt.Println("Hash correct")
		}	else {
			fmt.Println("Hash wrong")
		}

		fmt.Println("\n", hex.EncodeToString(hash.Sum(nil)), "\n", imgdata.hash )
	*/

}

func parseTag(imgchan chan<- Image, tag string, key string) {

	if tag == "" { fmt.Println("Something has gone horribly wrong, no tag found?"); os.Exit(1) }
	source := "http://derpiboo.ru/search.json?nofav=&nocomments=&utf8=false"
	if key != "" { source = source + "&key=" + key }

	fmt.Println(source + "&q=" + tag)

	resp, err := http.Get(source + "&q=" + tag) //Getting our nice http response. Needs checking for 404 and other responses that are... less expected
	if err != nil {
		panic(err)
	}

	defer resp.Body.Close() //and not forgetting to close it when it's done

	var dat []map[string]interface{}

	//fmt.Println(resp)

	body, err := ioutil.ReadAll(resp.Body) //stolen from official documentation
	if err != nil {
		panic(err)
	}

	//fmt.Println(body)

	if err := json.Unmarshal(body, &dat); //transforming json into native map

	err != nil {
		panic(err)

	}
	//fmt.Println(dat)
	close(imgchan)
}
<commit_msg>Downloading simutlaneously with parsing, single thread only<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"github.com/vaughan0/go-ini"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"

	//	"errors"
	//	"log"
	//	"net"
	//	"crypto/sha512"
	//	"encoding/hex"
)

//	defaults:
var (
	WORKERS 	int64	= 10    //Number of workers
	IMGDIR  	string	= "img" //default download directory
	TAG     	string	= ""    //default string is empty, it can only ge extracted from command line
	STARTPAGE	int		= 1		//default start page, derpiboo.ru 1-indexed
	STOPPAGE	int		= 0		//default stop page, would stop parsing json when it ends
	)

func main() {

	config, err := ini.LoadFile("config.ini") // Loading default config file and checking for various errors.
	if os.IsNotExist(err) {
		panic("config.ini does not exist, create it")
	}

	if err != nil {
		panic(err)
	}

	//Getting stuff from config, overwriting defaults

	key, ok := config.Get("main", "key")
	if !ok {
		panic("'key' variable missing from 'main' section")
	}

	W_temp, _ := config.Get("main", "workers")
	if W_temp != "" {
		WORKERS, err = strconv.ParseInt(W_temp, 10, 0)
		if err != nil {
			fmt.Println("Wrong configuration: Amount of workers is not a number")
			os.Exit(1)
		}
	}

	ID_temp, _ := config.Get("main", "downdir")
	if ID_temp != "" {
		IMGDIR = ID_temp
	}

	//here shall be flag parser

	flag.StringVar(&TAG, "t", TAG, "Tag to download: Replace spaces with \"+\".")
	flag.Parse()

	fmt.Println("Derpiboo.ru Downloader version 0.0.6 \nWorking")

	length := flag.NArg()
	if length == 0 && TAG == "" {
		fmt.Println("Nothing to download, bye!")
		os.Exit(0)
	}

	//	creating directory for downloads if not yet done
	if err := os.MkdirAll(IMGDIR, 0777); err != nil {
		panic(err)
	}

	imgdat := make(chan Image, WORKERS)
	done := make(chan bool)

	if TAG == "" {

		imgid := flag.Arg(length - 1)

		//	fmt.Println(key) //Just checking that I am not wrong

		_, err = strconv.ParseInt(imgid, 10, 0)
		if err != nil {
			fmt.Println("Wrong input: can not parse", imgid, "as a number")
			os.Exit(1)
		}

		fmt.Println("Processing image No " + imgid)

		go parseImg(imgdat, imgid, key)

	}	else	{
	//	and here we parse tag and abuse stuff horribly

		fmt.Println("Trying to process tags")
		go parseTag(imgdat, TAG, key)
	}

	fmt.Println("Starting worker")
	go dlimage(imgdat, done)

	<-done

}

type Image struct {
	url      string
	filename string
	hash     string
}

func parseImg(imgchan chan<- Image, imgid string, key string) {

	source := "http://derpiboo.ru/" + imgid + ".json?nofav=&nocomments="
	if key != "" {
		source = source + "&key=" + key
	}

		fmt.Println(source)

	resp, err := http.Get(source) //Getting our nice http response. Needs checking for 404 and other responses that are... less expected
	if err != nil {
		panic(err)
	}

	defer resp.Body.Close() //and not forgetting to close it when it's done

	var dat map[string]interface{}

	body, err := ioutil.ReadAll(resp.Body) //stolen from official documentation
	if err != nil {
		panic(err)
	}

	//fmt.Println(body)

	if err := json.Unmarshal(body, &dat); //transforming json into native map

	err != nil {
		panic(err)
	}
	var imgdata Image
	imgdata.url = "http:" + dat["image"].(string)
	imgdata.hash = dat["sha512_hash"].(string) //for the future and checking that we got file right
	imgdata.filename = strconv.FormatFloat(dat["id_number"].(float64), 'f', -1, 64) + "." + dat["file_name"].(string) + "." + dat["original_format"].(string)

	//	fmt.Println(strconv.FormatFloat(dat["id_number"].(float64), 'f', -1, 64))

	//	fmt.Println(dat)

	//	for now and troubleshooting
	//	fmt.Println(imgdata.url)
	//	fmt.Println(imgdata.hash)
	//	fmt.Println(imgdata.filename)

	imgchan <- imgdata

	close(imgchan)

	return
}

func dlimage(imgchan <-chan Image, done chan bool) {
	//	fmt.Println("reading channel")

	for {

	imgdata, more := <-imgchan

	if	more {
	if imgdata.filename == "" { fmt.Println("Empty filename. Oops?"); break }

	fmt.Println("Saving as ", imgdata.filename)
	PathSep, _ := strconv.Unquote(strconv.QuoteRune(os.PathSeparator))

	output, err := os.Create(IMGDIR + PathSep + imgdata.filename)
	if err != err {
		panic(err)
	}
	defer output.Close()

	response, err := http.Get(imgdata.url)
	if err != nil {
		panic(err)
		fmt.Println("Error while downloading", imgdata.url, "-", err)
		return
	}
	defer response.Body.Close()

	io.Copy(output, response.Body)

	/*	hash := sha512.New()
		io.Copy(hash, response.Body)
		b := make([]byte, hash.Size())
		hash.Sum(b[:0])

		fmt.Println("\n", hex.EncodeToString(b), "\n", imgdata.hash )

		if hex.EncodeToString(b) == imgdata.hash {
			fmt.Println("Hash correct")
		}	else {
			fmt.Println("Hash wrong")
		}

		fmt.Println("\n", hex.EncodeToString(hash.Sum(nil)), "\n", imgdata.hash )
	*/
		} else	{
			done <- true

		}

		}
}

func parseTag(imgchan chan<- Image, tag string, key string) {

	if tag == "" { fmt.Println("Something has gone horribly wrong, no tag found?"); os.Exit(1) }
	source := "http://derpiboo.ru/search.json?nofav=&nocomments=&utf8=false"
	if key != "" { source = source + "&key=" + key }

	fmt.Println(source + "&q=" + tag)

	resp, err := http.Get(source + "&q=" + tag) //Getting our nice http response. Needs checking for 404 and other responses that are... less expected
	if err != nil {
		panic(err)
	}

	defer resp.Body.Close() //and not forgetting to close it when it's done

	var dats []map[string]interface{}

	//fmt.Println(resp)

	body, err := ioutil.ReadAll(resp.Body) //stolen from official documentation
	if err != nil {
		panic(err)
	}

	//fmt.Println(body)

	if err := json.Unmarshal(body, &dats); //transforming json into native map

	err != nil {
		panic(err)

	}

	var imgdata Image

	for _, dat := range dats {

		imgdata.url = "http:" + dat["image"].(string)
		imgdata.hash = dat["sha512_hash"].(string) //for the future and checking that we got file right
		imgdata.filename = strconv.FormatFloat(dat["id_number"].(float64), 'f', -1, 64) + "." + dat["file_name"].(string) + "." + dat["original_format"].(string)

		imgchan <- imgdata
	}

	close(imgchan)
}
<|endoftext|>


<commit_before>package goFlags

import (
	"os"
	"testing"
)

type testStruct struct {
	A int    `flag:"A" flagDefault:"100"`
	B string `flag:"B" flagDefault:"200"`
	C string
	N string `flag:"-"`
	M int
	p string
	S testSub `flag:"S"`
}

type testSub struct {
	A int        `flag:"A" flagDefault:"300"`
	B string     `flag:"C" flagDefault:"400"`
	S testSubSub `flag:"S"`
}

type testSubSub struct {
	A int    `flag:"A" flagDefault:"500"`
	B string `flag:"S" flagDefault:"600"`
}

func TestParse(t *testing.T) {

	//os.Args = []string{"noop", "-flag1=val1", "arg1", "arg2"}
	//os.Args = []string{"program", "-h"}

	Setup("flag", "flagDefault")

	os.Args = []string{
		"program",
		"-a=900",
		"-b=TEST",
		"-s_s_a=99999",
	}

	s := &testStruct{A: 1, S: testSub{A: 1, B: "2"}}

	Preserve = false
	err := Parse(s)
	if err != nil {
		t.Fatal(err)
	}

	if s.A != 900 {
		t.Fatal("s.A != 900, s.A:", s.A)
	}

	if s.B != "TEST" {
		t.Fatal("s.B != \"TEST\", s.B:", s.B)
	}

	if s.S.S.A != 99999 {
		t.Fatal("s.S.S.A != 99999, s.S.S.A :", s.S.S.A)
	}

}

func TestPreserve(t *testing.T) {

	//os.Args = []string{"noop", "-flag1=val1", "arg1", "arg2"}
	//os.Args = []string{"program", "-h"}

	os.Args = []string{
		"program",
		"-a=8888",
		"-b=TEST",
		"-s_s_a=99999",
	}

	s := &testStruct{A: 1, S: testSub{A: 1, B: "2"}}

	Reset()
	Preserve = true
	err := Parse(s)
	if err != nil {
		t.Fatal(err)
	}

	if s.S.A != 1 {
		t.Fatal("s.S.A != 1, s.S.A:", s.S.A)
	}

	if s.A != 8888 {
		t.Fatal("s.A != 8888, s.A:", s.A)
	}

	if s.B != "TEST" {
		t.Fatal("s.B != \"TEST\", s.B:", s.B)
	}

	if s.S.S.A != 99999 {
		t.Fatal("s.S.S.A != 99999, s.S.S.A:", s.S.S.A)
	}
}
<commit_msg>add test to bool type<commit_after>package goFlags

import (
	"os"
	"testing"
)

type testStruct struct {
	A int    `flag:"A" flagDefault:"100"`
	B string `flag:"B" flagDefault:"200"`
	C string
	D bool   `cfg:"D" cfgDefault:"true"`
	N string `flag:"-"`
	M int
	p string
	S testSub `flag:"S"`
}

type testSub struct {
	A int        `flag:"A" flagDefault:"300"`
	B string     `flag:"C" flagDefault:"400"`
	S testSubSub `flag:"S"`
}

type testSubSub struct {
	A int    `flag:"A" flagDefault:"500"`
	B string `flag:"S" flagDefault:"600"`
}

func TestParse(t *testing.T) {

	//os.Args = []string{"noop", "-flag1=val1", "arg1", "arg2"}
	//os.Args = []string{"program", "-h"}

	Setup("flag", "flagDefault")

	os.Args = []string{
		"program",
		"-a=900",
		"-b=TEST",
		"-d=true",
		"-s_s_a=99999",
	}

	s := &testStruct{A: 1, S: testSub{A: 1, B: "2"}}

	Preserve = false
	err := Parse(s)
	if err != nil {
		t.Fatal(err)
	}

	if s.A != 900 {
		t.Fatal("s.A != 900, s.A:", s.A)
	}

	if s.B != "TEST" {
		t.Fatal("s.B != \"TEST\", s.B:", s.B)
	}

	if !s.D {
		t.Fatal("s.D == true, s.D:", s.D)
	}

	if s.S.S.A != 99999 {
		t.Fatal("s.S.S.A != 99999, s.S.S.A :", s.S.S.A)
	}

}

func TestPreserve(t *testing.T) {

	//os.Args = []string{"noop", "-flag1=val1", "arg1", "arg2"}
	//os.Args = []string{"program", "-h"}

	os.Args = []string{
		"program",
		"-a=8888",
		"-b=TEST",
		"-s_s_a=99999",
	}

	s := &testStruct{A: 1, S: testSub{A: 1, B: "2"}}

	Reset()
	Preserve = true
	err := Parse(s)
	if err != nil {
		t.Fatal(err)
	}

	if s.S.A != 1 {
		t.Fatal("s.S.A != 1, s.S.A:", s.S.A)
	}

	if s.A != 8888 {
		t.Fatal("s.A != 8888, s.A:", s.A)
	}

	if s.B != "TEST" {
		t.Fatal("s.B != \"TEST\", s.B:", s.B)
	}

	if s.S.S.A != 99999 {
		t.Fatal("s.S.S.A != 99999, s.S.S.A:", s.S.S.A)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/gob"
	"net"
	"os"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/alecthomas/kingpin"
	metrics_graphite "github.com/cyberdelia/go-metrics-graphite"
	gocache "github.com/patrickmn/go-cache"

	"github.com/resourced/resourced-master/application"
	"github.com/resourced/resourced-master/dal"
)

var (
	appConfDirFromEnv  = os.Getenv("RESOURCED_MASTER_CONFIG_DIR")
	appConfDirFromFlag = kingpin.Flag("conf", "Path to config directory").Short('c').String()

	appServerArg  = kingpin.Command("server", "Run resourced-master server.").Default()
	appMigrateArg = kingpin.Command("migrate", "CLI interface for resourced-master database migration.")

	appMigrateUpArg = appMigrateArg.Command("up", "Run all migrations to the most current.").Default()

	appVersion = "4.2.1"
)

func init() {
	gob.Register(&dal.UserRow{})
	gob.Register(&dal.ClusterRow{})
}

func main() {
	kingpin.UsageTemplate(kingpin.CompactUsageTemplate).Version(appVersion).Author("Didip Kerabat")
	parsedCLIArgs := kingpin.Parse()

	if appConfDirFromEnv == "" && *appConfDirFromFlag == "" {
		logrus.Fatal("Path to config directory is required. You must set RESOURCED_MASTER_CONFIG_DIR environment variable or -c flag.")
	}

	configDir := appConfDirFromEnv
	if configDir == "" {
		configDir = *appConfDirFromFlag
	}

	app, err := application.New(configDir)
	if err != nil {
		logrus.Fatal(err)
	}

	switch parsedCLIArgs {
	case "server":
		go app.MessageBus.ManageClients()

		go app.MessageBus.OnReceive(app.MessageBusHandlers())

		// Broadcast heartbeat
		go app.SendHeartbeat()

		go func() {
			// On boot, assign self to peers map and send a message to RefetchChecksChan
			app.Peers.Set(app.FullAddr(), true, gocache.DefaultExpiration)
			app.RefetchChecksChan <- true
		}()

		// Run all checks
		app.CheckAndRunTriggers()

		// Prune old timeseries data
		go app.PruneAll()

		// Publish metrics to local agent, which is a graphite endpoint.
		addr, err := net.ResolveTCPAddr("tcp", "localhost:"+app.GeneralConfig.LocalAgent.GraphiteTCPPort)
		if err != nil {
			logrus.Fatal(err)
		}
		statsInterval, err := time.ParseDuration(app.GeneralConfig.LocalAgent.ReportMetricsInterval)
		if err != nil {
			logrus.Fatal(err)
		}
		go metrics_graphite.Graphite(app.MetricsRegistry, statsInterval, "ResourcedMaster", addr)

		// Create HTTP server
		srv, err := app.NewHTTPServer()
		if err != nil {
			logrus.Fatal(err)
		}

		if app.GeneralConfig.HTTPS.CertFile != "" && app.GeneralConfig.HTTPS.KeyFile != "" {
			logrus.WithFields(logrus.Fields{"Addr": app.GeneralConfig.Addr}).Info("Running HTTPS server")
			srv.ListenAndServeTLS(app.GeneralConfig.HTTPS.CertFile, app.GeneralConfig.HTTPS.KeyFile)
		} else {
			logrus.WithFields(logrus.Fields{"Addr": app.GeneralConfig.Addr}).Info("Running HTTP server")
			srv.ListenAndServe()
		}

	case "migrate up":
		err := app.MigrateUpAll()
		if err != nil {
			logrus.Fatal(err)
		}
	}
}
<commit_msg>Bump version to 4.2.2<commit_after>package main

import (
	"encoding/gob"
	"net"
	"os"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/alecthomas/kingpin"
	metrics_graphite "github.com/cyberdelia/go-metrics-graphite"
	gocache "github.com/patrickmn/go-cache"

	"github.com/resourced/resourced-master/application"
	"github.com/resourced/resourced-master/dal"
)

var (
	appConfDirFromEnv  = os.Getenv("RESOURCED_MASTER_CONFIG_DIR")
	appConfDirFromFlag = kingpin.Flag("conf", "Path to config directory").Short('c').String()

	appServerArg  = kingpin.Command("server", "Run resourced-master server.").Default()
	appMigrateArg = kingpin.Command("migrate", "CLI interface for resourced-master database migration.")

	appMigrateUpArg = appMigrateArg.Command("up", "Run all migrations to the most current.").Default()

	appVersion = "4.2.2"
)

func init() {
	gob.Register(&dal.UserRow{})
	gob.Register(&dal.ClusterRow{})
}

func main() {
	kingpin.UsageTemplate(kingpin.CompactUsageTemplate).Version(appVersion).Author("Didip Kerabat")
	parsedCLIArgs := kingpin.Parse()

	if appConfDirFromEnv == "" && *appConfDirFromFlag == "" {
		logrus.Fatal("Path to config directory is required. You must set RESOURCED_MASTER_CONFIG_DIR environment variable or -c flag.")
	}

	configDir := appConfDirFromEnv
	if configDir == "" {
		configDir = *appConfDirFromFlag
	}

	app, err := application.New(configDir)
	if err != nil {
		logrus.Fatal(err)
	}

	switch parsedCLIArgs {
	case "server":
		go app.MessageBus.ManageClients()

		go app.MessageBus.OnReceive(app.MessageBusHandlers())

		// Broadcast heartbeat
		go app.SendHeartbeat()

		go func() {
			// On boot, assign self to peers map and send a message to RefetchChecksChan
			app.Peers.Set(app.FullAddr(), true, gocache.DefaultExpiration)
			app.RefetchChecksChan <- true
		}()

		// Run all checks
		app.CheckAndRunTriggers()

		// Prune old timeseries data
		go app.PruneAll()

		// Publish metrics to local agent, which is a graphite endpoint.
		addr, err := net.ResolveTCPAddr("tcp", "localhost:"+app.GeneralConfig.LocalAgent.GraphiteTCPPort)
		if err != nil {
			logrus.Fatal(err)
		}
		statsInterval, err := time.ParseDuration(app.GeneralConfig.LocalAgent.ReportMetricsInterval)
		if err != nil {
			logrus.Fatal(err)
		}
		go metrics_graphite.Graphite(app.MetricsRegistry, statsInterval, "ResourcedMaster", addr)

		// Create HTTP server
		srv, err := app.NewHTTPServer()
		if err != nil {
			logrus.Fatal(err)
		}

		if app.GeneralConfig.HTTPS.CertFile != "" && app.GeneralConfig.HTTPS.KeyFile != "" {
			logrus.WithFields(logrus.Fields{"Addr": app.GeneralConfig.Addr}).Info("Running HTTPS server")
			srv.ListenAndServeTLS(app.GeneralConfig.HTTPS.CertFile, app.GeneralConfig.HTTPS.KeyFile)
		} else {
			logrus.WithFields(logrus.Fields{"Addr": app.GeneralConfig.Addr}).Info("Running HTTP server")
			srv.ListenAndServe()
		}

	case "migrate up":
		err := app.MigrateUpAll()
		if err != nil {
			logrus.Fatal(err)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/golang/groupcache"
	"github.com/gorilla/mux"
	"github.com/mitchellh/goamz/aws"
	"github.com/mitchellh/goamz/s3"
	"log"
	"log/syslog"
	"net/http"
	"os"
	"vip/fetch"
	"vip/peer"
	"vip/store"
)

const (
	KeyFilePath  = "/etc/vip/application.key"
	CertFilePath = "/etc/vip/application.cert"
)

var (
	cache     *groupcache.Group
	peers     peer.CachePool
	storage   store.ImageStore
	authToken string

	verbose  *bool   = flag.Bool("verbose", false, "verbose logging")
	httpport *string = flag.String("httpport", "8080", "target port")
	secure   bool    = false
)

func listenHttp() {
	log.Printf("Listening on port :%s\n", *httpport)

	port := fmt.Sprintf(":%s", *httpport)

	if secure {
		log.Println("Serving via TSL")
		if err := http.ListenAndServeTLS(port, CertFilePath, KeyFilePath, nil); err != nil {
			log.Fatalf("Error starting server: %s\n", err.Error())
		}
	} else {
		if err := http.ListenAndServe(port, nil); err != nil {
			log.Fatalf("Error starting server: %s\n", err.Error())
		}
	}
}

func getRegion() aws.Region {
	region := os.Getenv("AWS_REGION")
	aws_region, ok := aws.Regions[region]
	if ok {
		return aws_region
	} else {
		log.Printf(
			"\"%s\" is not a valid AWS_REGION parameter provided, defaulting to us-east-1",
			region)
		return aws.USEast
	}
}

func init() {
	flag.Parse()
	var (
		err     error
		hasKey  bool
		hasCert bool
	)

	_, err = os.Stat(KeyFilePath)
	if err != nil {
		log.Printf("no key found at %s\n", KeyFilePath)
		hasKey = false
	}

	_, err = os.Stat(CertFilePath)
	if err != nil {
		log.Printf("no certificate found at %s\n", CertFilePath)
		hasCert = false
	}

	secure = hasCert && hasKey

	r := mux.NewRouter()

	authToken = os.Getenv("AUTH_TOKEN")
	if authToken == "" {
		log.Println("No AUTH_TOKEN parameter provided, uploads are insecure")
	}

	r.Handle("/upload/{bucket_id}", verifyAuth(handleUpload))
	r.HandleFunc("/{bucket_id}/{image_id}", handleImageRequest)
	r.HandleFunc("/ping", handlePing)
	http.Handle("/", r)
}

func main() {
	awsAuth, err := aws.EnvAuth()
	if err != nil {
		log.Fatalf(err.Error())
	}

	s3conn := s3.New(awsAuth, getRegion())
	storage = store.NewS3Store(s3conn)

	peers = peer.DebugPool()

	peers.SetContext(func(r *http.Request) groupcache.Context {
		return fetch.RequestContext(r)
	})

	cache = groupcache.NewGroup("ImageProxyCache", 64<<20, groupcache.GetterFunc(
		func(c groupcache.Context, key string, dest groupcache.Sink) error {
			log.Printf("Cache MISS for key -> %s", key)
			// Get image data from S3
			b, err := fetch.ImageData(storage, c)
			if err != nil {
				return err
			}

			return dest.SetBytes(b)
		}))

	if !*verbose {
		logwriter, err := syslog.Dial("udp", "app_syslog:514", syslog.LOG_NOTICE, "vip")
		if err != nil {
			log.Fatal(err.Error())
		}
		log.SetOutput(logwriter)
	}

	go peers.Listen()
	go listenHttp()

	log.Println("Cache listening on port :" + peers.Port())
	s := &http.Server{
		Addr:    ":" + peers.Port(),
		Handler: peers,
	}
	s.ListenAndServe()
}
<commit_msg>removed var block<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/golang/groupcache"
	"github.com/gorilla/mux"
	"github.com/mitchellh/goamz/aws"
	"github.com/mitchellh/goamz/s3"
	"log"
	"log/syslog"
	"net/http"
	"os"
	"vip/fetch"
	"vip/peer"
	"vip/store"
)

const (
	KeyFilePath  = "/etc/vip/application.key"
	CertFilePath = "/etc/vip/application.cert"
)

var (
	cache     *groupcache.Group
	peers     peer.CachePool
	storage   store.ImageStore
	authToken string

	verbose  *bool   = flag.Bool("verbose", false, "verbose logging")
	httpport *string = flag.String("httpport", "8080", "target port")
	secure   bool    = false
)

func listenHttp() {
	log.Printf("Listening on port :%s\n", *httpport)

	port := fmt.Sprintf(":%s", *httpport)

	if secure {
		log.Println("Serving via TSL")
		if err := http.ListenAndServeTLS(port, CertFilePath, KeyFilePath, nil); err != nil {
			log.Fatalf("Error starting server: %s\n", err.Error())
		}
	} else {
		if err := http.ListenAndServe(port, nil); err != nil {
			log.Fatalf("Error starting server: %s\n", err.Error())
		}
	}
}

func getRegion() aws.Region {
	region := os.Getenv("AWS_REGION")
	aws_region, ok := aws.Regions[region]
	if ok {
		return aws_region
	} else {
		log.Printf(
			"\"%s\" is not a valid AWS_REGION parameter provided, defaulting to us-east-1",
			region)
		return aws.USEast
	}
}

func init() {
	flag.Parse()
	var err error
	var hasKey bool
	var hasCert bool

	_, err = os.Stat(KeyFilePath)
	if err != nil {
		log.Printf("no key found at %s\n", KeyFilePath)
		hasKey = false
	}

	_, err = os.Stat(CertFilePath)
	if err != nil {
		log.Printf("no certificate found at %s\n", CertFilePath)
		hasCert = false
	}

	secure = hasCert && hasKey

	r := mux.NewRouter()

	authToken = os.Getenv("AUTH_TOKEN")
	if authToken == "" {
		log.Println("No AUTH_TOKEN parameter provided, uploads are insecure")
	}

	r.Handle("/upload/{bucket_id}", verifyAuth(handleUpload))
	r.HandleFunc("/{bucket_id}/{image_id}", handleImageRequest)
	r.HandleFunc("/ping", handlePing)
	http.Handle("/", r)
}

func main() {
	awsAuth, err := aws.EnvAuth()
	if err != nil {
		log.Fatalf(err.Error())
	}

	s3conn := s3.New(awsAuth, getRegion())
	storage = store.NewS3Store(s3conn)

	peers = peer.DebugPool()

	peers.SetContext(func(r *http.Request) groupcache.Context {
		return fetch.RequestContext(r)
	})

	cache = groupcache.NewGroup("ImageProxyCache", 64<<20, groupcache.GetterFunc(
		func(c groupcache.Context, key string, dest groupcache.Sink) error {
			log.Printf("Cache MISS for key -> %s", key)
			// Get image data from S3
			b, err := fetch.ImageData(storage, c)
			if err != nil {
				return err
			}

			return dest.SetBytes(b)
		}))

	if !*verbose {
		logwriter, err := syslog.Dial("udp", "app_syslog:514", syslog.LOG_NOTICE, "vip")
		if err != nil {
			log.Fatal(err.Error())
		}
		log.SetOutput(logwriter)
	}

	go peers.Listen()
	go listenHttp()

	log.Println("Cache listening on port :" + peers.Port())
	s := &http.Server{
		Addr:    ":" + peers.Port(),
		Handler: peers,
	}
	s.ListenAndServe()
}
<|endoftext|>


<commit_before>package core

import (
	"log"
	"os"
	"os/user"
)

/*
Tinzenite is the struct on which all important operations should be called.
*/
type Tinzenite struct {
	Path     string
	auth     *Authentication
	selfpeer *Peer
	channel  *Channel
	allPeers []*Peer
	model    *Model
}

/*
CreateTinzenite makes a directory a new Tinzenite directory. Will return error
if already so.
*/
func CreateTinzenite(dirname, dirpath, peername, username string, encrypted bool) (*Tinzenite, error) {
	// encrypted peer for now unsupported
	if encrypted {
		return nil, ErrUnsupported
	}
	if IsTinzenite(dirpath) {
		return nil, ErrIsTinzenite
	}
	hash, err := newIdentifier()
	if err != nil {
		return nil, err
	}
	/*TODO Bcrypt username!*/
	// Build
	tinzenite := &Tinzenite{
		Path: dirpath,
		auth: &Authentication{
			User:    username,
			Dirname: dirname,
			DirID:   hash,
			Key:     "TODO"}}
	// build channel
	channel, err := CreateChannel(peername, nil, tinzenite)
	if err != nil {
		return nil, err
	}
	tinzenite.channel = channel
	// build self peer
	address, err := channel.Address()
	if err != nil {
		return nil, err
	}
	peerhash, err := newIdentifier()
	if err != nil {
		return nil, err
	}
	peer := &Peer{
		Name:           peername,
		Address:        address,
		Protocol:       Tox,
		Identification: peerhash}
	tinzenite.selfpeer = peer
	tinzenite.allPeers = []*Peer{peer}
	// save
	err = tinzenite.store()
	if err != nil {
		return nil, err
	}
	// save that this directory is now a tinzenite dir
	err = tinzenite.storeGlobalConfig()
	if err != nil {
		return nil, err
	}
	// build model (can block for long!)
	m, err := LoadModel(dirpath)
	if err != nil {
		return nil, err
	}
	tinzenite.model = m
	/*TODO later implement that model updates are sent to all online peers --> channel and func must be init here*/
	return tinzenite, nil
}

/*
LoadTinzenite will try to load the given directory path as a Tinzenite directory.
If not one it won't work: use CreateTinzenite to create a new peer.
*/
func LoadTinzenite(dirpath string) (*Tinzenite, error) {
	if !IsTinzenite(dirpath) {
		return nil, ErrNotTinzenite
	}
	t := &Tinzenite{Path: dirpath}
	// load auth
	auth, err := loadAuthentication(dirpath)
	if err != nil {
		return nil, err
	}
	t.auth = auth
	// load model
	model, err := LoadModel(dirpath)
	if err != nil {
		return nil, err
	}
	t.model = model
	// load tox dump
	selfToxDump, err := loadToxDump(dirpath)
	if err != nil {
		return nil, err
	}
	t.selfpeer = selfToxDump.SelfPeer
	// prepare channel
	channel, err := CreateChannel(t.selfpeer.Name, selfToxDump.ToxData, t)
	if err != nil {
		return nil, err
	}
	t.channel = channel
	return t, nil
}

/*
RemoveTinzenite directory. Specifically leaves all user files but removes all
Tinzenite specific items.
*/
func RemoveTinzenite(path string) error {
	if !IsTinzenite(path) {
		return ErrNotTinzenite
	}
	/* TODO remove from directory list*/
	return os.RemoveAll(path + "/" + TINZENITEDIR)
}

/*
SyncModel TODO

fetches model from other peers and syncs (this is for manual sync)
*/
func (t *Tinzenite) SyncModel() error {
	// first ensure that local model is up to date
	err := t.model.Update()
	if err != nil {
		return err
	}
	// iterate over all known peers
	for _, peer := range t.allPeers {
		if peer == t.selfpeer {
			continue
		}
		/*TODO - also make this concurrent?*/
		t.channel.Send(peer.Address, "Want update!")
		// if online -> continue
		// if not init -> init
		// sync
		/*TODO must store init state in allPeers too, also in future encryption*/
	}
	return nil
}

/*
Address of this Tinzenite peer.
*/
func (t *Tinzenite) Address() string {
	return t.selfpeer.Address
}

/*
Close cleanly stores everything and shuts Tinzenite down.
*/
func (t *Tinzenite) Close() {
	/*TODO should I really update again? Maybe just call store explicitely?*/
	t.model.Update()
	t.channel.Close()
	t.store()
}

/*
write the tinzenite directory structure to disk.
*/
func (t *Tinzenite) store() error {
	// TODO
	/*
		Writes everything in the .tinzenite directory.
	*/
	root := t.Path + "/" + TINZENITEDIR
	// build directory structure
	err := makeDirectories(root,
		ORGDIR+"/"+PEERSDIR, "temp", "removed", LOCAL)
	if err != nil {
		return err
	}
	// write all peers to files
	for _, peer := range t.allPeers {
		err := peer.store(t.Path)
		if err != nil {
			return err
		}
	}
	// store local peer info with toxdata
	toxData, err := t.channel.ToxData()
	if err != nil {
		return err
	}
	toxPeerDump := &toxPeerDump{
		SelfPeer: t.selfpeer,
		ToxData:  toxData}
	err = toxPeerDump.store(t.Path)
	if err != nil {
		return err
	}
	// finally store auth file
	return t.auth.store(t.Path)
}

/*
Connect this tinzenite to another peer, beginning the bootstrap process.
*/
func (t *Tinzenite) Connect(address string) error {
	return t.channel.RequestConnection(address, t.selfpeer)
}

/*
CallbackNewConnection is called when a new connection request comes in.
*/
func (t *Tinzenite) CallbackNewConnection(address, message string) {
	log.Printf("New connection from <%s> with message <%s>\n", address, message)
	err := t.channel.AcceptConnection(address)
	if err != nil {
		log.Println(err.Error())
	}
	/*TODO actually this should be read from disk once the peer has synced... oO */
	/*
		tinzenite.allPeers = append(tinzenite.allPeers, &Peer{
			Name:     "Unknown",
			Address:  address,
			Protocol: Tox})
	*/
	// actually we just want to get type and confidence from the user here, and if everything
	// is okay we accept the connection --> then what? need to bootstrap him...
}

/*
CallbackMessage is called when a message is received.
*/
func (t *Tinzenite) CallbackMessage(address, message string) {
	log.Printf("Message from <%s> with message <%s>\n", address, message)
	t.channel.Send(address, "ACK")
	/*TODO switch if request or update*/
}

/*
storeGlobalConfig stores the path value into the user's home directory so that clients
can locate it.
*/
func (t *Tinzenite) storeGlobalConfig() error {
	// ready outside data
	user, err := user.Current()
	if err != nil {
		return err
	}
	path := user.HomeDir + "/.config/tinzenite"
	err = makeDirectory(path)
	if err != nil {
		return err
	}
	path += "/" + DIRECTORYLIST
	file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR|os.O_APPEND, FILEPERMISSIONMODE)
	if err != nil {
		return err
	}
	defer file.Close()
	// write path to file
	_, err = file.WriteString(t.Path + "\n")
	if err != nil {
		return err
	}
	// ensure that the file is valid
	return PrettifyDirectoryList()
}
<commit_msg>bug fix<commit_after>package core

import (
	"log"
	"os"
	"os/user"
)

/*
Tinzenite is the struct on which all important operations should be called.
*/
type Tinzenite struct {
	Path     string
	auth     *Authentication
	selfpeer *Peer
	channel  *Channel
	allPeers []*Peer
	model    *Model
}

/*
CreateTinzenite makes a directory a new Tinzenite directory. Will return error
if already so.
*/
func CreateTinzenite(dirname, dirpath, peername, username string, encrypted bool) (*Tinzenite, error) {
	// encrypted peer for now unsupported
	if encrypted {
		return nil, ErrUnsupported
	}
	if IsTinzenite(dirpath) {
		return nil, ErrIsTinzenite
	}
	hash, err := newIdentifier()
	if err != nil {
		return nil, err
	}
	/*TODO Bcrypt username!*/
	// Build
	tinzenite := &Tinzenite{
		Path: dirpath,
		auth: &Authentication{
			User:    username,
			Dirname: dirname,
			DirID:   hash,
			Key:     "TODO"}}
	// build channel
	channel, err := CreateChannel(peername, nil, tinzenite)
	if err != nil {
		return nil, err
	}
	tinzenite.channel = channel
	// build self peer
	address, err := channel.Address()
	if err != nil {
		return nil, err
	}
	peerhash, err := newIdentifier()
	if err != nil {
		return nil, err
	}
	peer := &Peer{
		Name:           peername,
		Address:        address,
		Protocol:       Tox,
		Identification: peerhash}
	tinzenite.selfpeer = peer
	tinzenite.allPeers = []*Peer{peer}
	// save
	err = tinzenite.store()
	if err != nil {
		return nil, err
	}
	// save that this directory is now a tinzenite dir
	err = tinzenite.storeGlobalConfig()
	if err != nil {
		return nil, err
	}
	// build model (can block for long!)
	m, err := LoadModel(dirpath)
	if err != nil {
		return nil, err
	}
	tinzenite.model = m
	/*TODO later implement that model updates are sent to all online peers --> channel and func must be init here*/
	return tinzenite, nil
}

/*
LoadTinzenite will try to load the given directory path as a Tinzenite directory.
If not one it won't work: use CreateTinzenite to create a new peer.
*/
func LoadTinzenite(dirpath string) (*Tinzenite, error) {
	if !IsTinzenite(dirpath) {
		return nil, ErrNotTinzenite
	}
	t := &Tinzenite{Path: dirpath}
	// load auth
	auth, err := loadAuthentication(dirpath)
	if err != nil {
		return nil, err
	}
	t.auth = auth
	// load model
	model, err := LoadModel(dirpath)
	if err != nil {
		return nil, err
	}
	t.model = model
	// load tox dump
	selfToxDump, err := loadToxDump(dirpath)
	if err != nil {
		return nil, err
	}
	t.selfpeer = selfToxDump.SelfPeer
	// prepare channel
	channel, err := CreateChannel(t.selfpeer.Name, selfToxDump.ToxData, t)
	if err != nil {
		return nil, err
	}
	t.channel = channel
	return t, nil
}

/*
RemoveTinzenite directory. Specifically leaves all user files but removes all
Tinzenite specific items.
*/
func RemoveTinzenite(path string) error {
	if !IsTinzenite(path) {
		return ErrNotTinzenite
	}
	/* TODO remove from directory list*/
	return os.RemoveAll(path + "/" + TINZENITEDIR)
}

/*
SyncModel TODO

fetches model from other peers and syncs (this is for manual sync)
*/
func (t *Tinzenite) SyncModel() error {
	// first ensure that local model is up to date
	err := t.model.Update()
	if err != nil {
		return err
	}
	// iterate over all known peers
	for _, peer := range t.allPeers {
		if peer == t.selfpeer {
			continue
		}
		/*TODO - also make this concurrent?*/
		t.channel.Send(peer.Address, "Want update!")
		// if online -> continue
		// if not init -> init
		// sync
		/*TODO must store init state in allPeers too, also in future encryption*/
	}
	return nil
}

/*
Address of this Tinzenite peer.
*/
func (t *Tinzenite) Address() string {
	return t.selfpeer.Address
}

/*
Close cleanly stores everything and shuts Tinzenite down.
*/
func (t *Tinzenite) Close() {
	/*TODO should I really update again? Maybe just call store explicitely?*/
	t.model.Update()
	t.store()
	// FINALLY close
	t.channel.Close()
}

/*
write the tinzenite directory structure to disk.
*/
func (t *Tinzenite) store() error {
	// TODO
	/*
		Writes everything in the .tinzenite directory.
	*/
	root := t.Path + "/" + TINZENITEDIR
	// build directory structure
	err := makeDirectories(root,
		ORGDIR+"/"+PEERSDIR, "temp", "removed", LOCAL)
	if err != nil {
		return err
	}
	// write all peers to files
	for _, peer := range t.allPeers {
		err := peer.store(t.Path)
		if err != nil {
			return err
		}
	}
	// store local peer info with toxdata
	toxData, err := t.channel.ToxData()
	if err != nil {
		return err
	}
	toxPeerDump := &toxPeerDump{
		SelfPeer: t.selfpeer,
		ToxData:  toxData}
	err = toxPeerDump.store(t.Path)
	if err != nil {
		return err
	}
	// finally store auth file
	return t.auth.store(t.Path)
}

/*
Connect this tinzenite to another peer, beginning the bootstrap process.
*/
func (t *Tinzenite) Connect(address string) error {
	return t.channel.RequestConnection(address, t.selfpeer)
}

/*
CallbackNewConnection is called when a new connection request comes in.
*/
func (t *Tinzenite) CallbackNewConnection(address, message string) {
	log.Printf("New connection from <%s> with message <%s>\n", address, message)
	err := t.channel.AcceptConnection(address)
	if err != nil {
		log.Println(err.Error())
	}
	/*TODO actually this should be read from disk once the peer has synced... oO */
	/*
		tinzenite.allPeers = append(tinzenite.allPeers, &Peer{
			Name:     "Unknown",
			Address:  address,
			Protocol: Tox})
	*/
	// actually we just want to get type and confidence from the user here, and if everything
	// is okay we accept the connection --> then what? need to bootstrap him...
}

/*
CallbackMessage is called when a message is received.
*/
func (t *Tinzenite) CallbackMessage(address, message string) {
	log.Printf("Message from <%s> with message <%s>\n", address, message)
	t.channel.Send(address, "ACK")
	/*TODO switch if request or update*/
}

/*
storeGlobalConfig stores the path value into the user's home directory so that clients
can locate it.
*/
func (t *Tinzenite) storeGlobalConfig() error {
	// ready outside data
	user, err := user.Current()
	if err != nil {
		return err
	}
	path := user.HomeDir + "/.config/tinzenite"
	err = makeDirectory(path)
	if err != nil {
		return err
	}
	path += "/" + DIRECTORYLIST
	file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR|os.O_APPEND, FILEPERMISSIONMODE)
	if err != nil {
		return err
	}
	defer file.Close()
	// write path to file
	_, err = file.WriteString(t.Path + "\n")
	if err != nil {
		return err
	}
	// ensure that the file is valid
	return PrettifyDirectoryList()
}
<|endoftext|>


<commit_before>package main

import (
	_ "expvar"
	"github.com/codegangsta/negroni"
	"github.com/meatballhat/negroni-logrus"
	"github.com/unrolled/render"
	"net/http"
)

func main() {
	Run()
}

func Run() {
	m := http.DefaultServeMux
	n := negroni.New(negroni.NewRecovery(), negroni.NewStatic(http.Dir("assets")))
	l := negronilogrus.NewMiddleware()
	r := render.New(render.Options{
		Layout: "layout",
	})

	n.Use(l)
	n.UseHandler(m)

	m.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
		r.HTML(w, http.StatusOK, "index", "world")
	})

	addr := ":3000"
	l.Logger.Infof("Listening on %s", addr)
	l.Logger.Fatal(http.ListenAndServe(addr, n))
}
<commit_msg>Allow port to be specified by environment variable<commit_after>package main

import (
	_ "expvar"
	"github.com/codegangsta/negroni"
	"github.com/meatballhat/negroni-logrus"
	"github.com/unrolled/render"
	"net/http"
	"os"
)

func main() {
	Run()
}

func Run() {
	m := http.DefaultServeMux
	n := negroni.New(negroni.NewRecovery(), negroni.NewStatic(http.Dir("assets")))
	l := negronilogrus.NewMiddleware()
	r := render.New(render.Options{
		Layout: "layout",
	})

	n.Use(l)
	n.UseHandler(m)

	m.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
		r.HTML(w, http.StatusOK, "index", "world")
	})

	var addr string
	if len(os.Getenv("PORT")) > 0 {
		addr = ":" + os.Getenv("PORT")
	} else {
		addr = ":3000"
	}

	l.Logger.Infof("Listening on %s", addr)
	l.Logger.Fatal(http.ListenAndServe(addr, n))
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/md5"
	"database/sql"
	"encoding/json"
	"fmt"
	"github.com/golang/groupcache"
	"github.com/jessevdk/go-flags"
	"github.com/jmoiron/sqlx"
	_ "github.com/mattn/go-sqlite3"
	"github.com/zenazn/goji"
	"github.com/zenazn/goji/web"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"strconv"
	"strings"
	"time"
)

var options struct {
	Hostname  string `long:"hostname" description:"externally accessible hostname" default:"localhost"`
	Bind      string `long:"bind" description:"Server port" default:":8000"`
	Tilesets  string `long:"tilesets" description:"path to tilesets" default:"./tilesets"`
	CachePort uint16 `long:"cacheport" description:"GroupCache port" default:"8001"`
	CacheSize int64  `long:"cachesize" description:"Size of Cache (MB)" default:"10"`
	MaxAge    uint   `long:"max_age" description:"Response max-age duration (seconds)" default:"3600"`
}

type DBClient struct {
	connection  *sqlx.DB
	imageStmt   *sql.Stmt
	infoStmt    *sqlx.Stmt
	contentType string
}

type KeyValuePair struct {
	Name  string
	Value string
}

var (
	pool       *groupcache.HTTPPool
	cache      *groupcache.Group
	dbClients  map[string]DBClient
	blankPNG   []byte
	cacheSince = time.Now().Format(http.TimeFormat)
)

func main() {
	_, err := flags.ParseArgs(&options, os.Args)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println("Flags:", options)
	blankPNG, _ = ioutil.ReadFile("blank.png")

	dbClients = make(map[string]DBClient)
	tilesets, _ := filepath.Glob(path.Join(options.Tilesets, "*.mbtiles"))

	for _, filename := range tilesets {
		_, service := filepath.Split(filename)
		service = strings.Split(service, ".")[0]

		db, err := sqlx.Open("sqlite3", filename)
		if err != nil {
			log.Fatal(err)
		}
		defer db.Close()

		infoStmt, err := db.Preparex("select * from metadata where value is not ''")
		if err != nil {
			log.Fatal(err)
		}
		defer infoStmt.Close()

		stmt, err := db.Prepare("select tile_data from tiles where zoom_level = ? and tile_column = ? and tile_row = ?")
		if err != nil {
			log.Fatal(err)
		}
		defer stmt.Close()

		//query a sample tile to determine if png or jpg, since metadata from tilemill doesn't give this to us
		var tileData []byte
		err = db.QueryRow("select tile_data from images limit 1").Scan(&tileData)
		if err != nil {
			log.Fatal(err)
		}

		dbClients[service] = DBClient{
			connection:  db,
			infoStmt:    infoStmt,
			imageStmt:   stmt,
			contentType: http.DetectContentType(tileData),
		}
	}

	pool = groupcache.NewHTTPPool(fmt.Sprintf("http://127.0.0.1:%v", options.CachePort))
	cache = groupcache.NewGroup("TileCache", options.CacheSize*1048576, groupcache.GetterFunc(
		func(ctx groupcache.Context, key string, dest groupcache.Sink) error {
			pathParams := strings.Split(key, "/")
			service := pathParams[0]
			yParams := strings.Split(pathParams[3], ".")
			z, _ := strconv.ParseUint(pathParams[1], 0, 64)
			x, _ := strconv.ParseUint(pathParams[2], 0, 64)
			y, _ := strconv.ParseUint(yParams[0], 0, 64)
			//flip y to match TMS spec
			y = (1 << z) - 1 - y

			var tileData []byte
			err := dbClients[service].imageStmt.QueryRow(uint8(z), uint16(x), uint16(y)).Scan(&tileData)
			if err != nil {
				if err != sql.ErrNoRows {
					log.Fatal(err)
				}
			}
			dest.SetBytes(tileData)
			return nil
		}))

	//TODO: add gzip
	goji.Get("/services", ListServices)
	goji.Get("/services/:service", GetService)
	goji.Get("/services/:service/tiles/:z/:x/:filename", GetTile)
	//TODO:  goji.Get("/:service/grids/:z/:x/:filename", GetGrid) //return UTF8 grid

	goji.Serve()
}

type ServiceInfo struct {
	URI       string `json:"uri"`
	ImageType string `json:"imageType"`
}

func ListServices(c web.C, w http.ResponseWriter, r *http.Request) {
	services := make([]ServiceInfo, len(dbClients))
	i := 0
	for service, _ := range dbClients {
		services[i] = ServiceInfo{
			URI:       fmt.Sprintf("/services/%s", service),
			ImageType: strings.Split(dbClients[service].contentType, "/")[1],
		}
		i++
	}
	w.Header().Add("Content-Type", "application/json")
	json, _ := json.Marshal(services)
	w.Write(json)
}

func GetService(c web.C, w http.ResponseWriter, r *http.Request) {
	//https://github.com/mapbox/tilejson-spec/tree/master/2.1.0
	//FIXME: https://a.tiles.mapbox.com/v4/bcward.salcc.json?access_token=<removed>
	service := c.URLParams["service"]
	if _, exists := dbClients[service]; !exists {
		http.Error(w, fmt.Sprintf("Service not found: %s", service), http.StatusNotFound)
		return
	}
	results := make(map[string]string)
	rows, err := dbClients[service].infoStmt.Queryx()
	if err != nil {
		http.Error(w, fmt.Sprintf("Metadata query failed for: %s", service), http.StatusInternalServerError)
		return
	}
	for rows.Next() {
		var record KeyValuePair
		rows.StructScan(&record)
		results[record.Name] = record.Value
	}
	rootURL := fmt.Sprintf("http://%s", options.Hostname)
	if options.Bind != ":80" {
		rootURL = fmt.Sprintf("%s%s", rootURL, options.Bind)
	}
	out := map[string]interface{}{
		"tilejson": "2.1.0",
		"id":       service,
		"scheme":   "tms",
		"format":   strings.Split(dbClients[service].contentType, "/")[1],
		"tiles":    []string{fmt.Sprintf("%s/services/%s/tiles/{z}/{x}/{y}", rootURL, service)},
	}

	for k := range results {
		out[k] = results[k]
	}
	var value string
	var ok bool
	multiFloatFields := []string{"bounds", "center"}
	intFields := []string{"maxzoom", "minzoom"}
	for _, field := range multiFloatFields {
		if value, ok = results[field]; ok {
			out[field] = stringToFloats(value)
		}
	}
	for _, field := range intFields {
		if value, ok = results[field]; ok {
			out[field], _ = strconv.Atoi(results[field])
		}
	}

	w.Header().Add("Content-Type", "application/json")
	json, _ := json.Marshal(out)
	w.Write(json)
}

func GetTile(c web.C, w http.ResponseWriter, r *http.Request) {
	var (
		data        []byte
		contentType string
	)
	//TODO: validate x, y, z

	service := c.URLParams["service"]
	if _, exists := dbClients[service]; !exists {
		http.Error(w, fmt.Sprintf("Service not found: %s", service), http.StatusNotFound)
		return
	}

	key := strings.Join([]string{c.URLParams["service"], c.URLParams["z"], c.URLParams["x"], c.URLParams["filename"]}, "/")
	fmt.Println("URL key: ", key)

	err := cache.Get(nil, key, groupcache.AllocatingByteSliceSink(&data))
	if err != nil {
		log.Println("Error fetching key", key)
		http.Error(w, fmt.Sprintf("Cache get failed for key: %s", key), http.StatusInternalServerError)
		return
	}
	etag := fmt.Sprintf("%x", md5.Sum(data))

	if r.Header.Get("If-None-Match") == etag {
		w.WriteHeader(http.StatusNotModified)
		return
	}

	if len(data) <= 1 {
		data = blankPNG
		contentType = "image/png"
	} else {
		contentType = dbClients[service].contentType
	}

	w.Header().Add("Cache-Control", fmt.Sprintf("max-age=%v", options.MaxAge))
	w.Header().Add("Last-Modified", cacheSince)
	w.Header().Add("Content-Type", contentType)
	w.Header().Add("ETag", etag)
	w.Write(data)
}

func stringToFloats(str string) []float32 {
	split := strings.Split(str, ",")
	var out = make([]float32, len(split))
	for i, v := range split {
		value, _ := strconv.ParseFloat(v, 32)
		out[i] = float32(value)
	}
	return out
}
<commit_msg>Added CORS and fixed tileJson<commit_after>package main

import (
	"crypto/md5"
	"database/sql"
	"encoding/json"
	"fmt"
	"github.com/golang/groupcache"
	"github.com/jessevdk/go-flags"
	"github.com/jmoiron/sqlx"
	_ "github.com/mattn/go-sqlite3"
	"github.com/rs/cors"
	"github.com/zenazn/goji"
	"github.com/zenazn/goji/web"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"strconv"
	"strings"
	"time"
)

var options struct {
	Hostname  string `long:"hostname" description:"externally accessible hostname" default:"localhost"`
	Bind      string `long:"bind" description:"Server port" default:":8000"`
	Tilesets  string `long:"tilesets" description:"path to tilesets" default:"./tilesets"`
	CachePort uint16 `long:"cacheport" description:"GroupCache port" default:"8001"`
	CacheSize int64  `long:"cachesize" description:"Size of Cache (MB)" default:"10"`
	MaxAge    uint   `long:"max_age" description:"Response max-age duration (seconds)" default:"3600"`
}

type DBClient struct {
	connection  *sqlx.DB
	imageStmt   *sql.Stmt
	infoStmt    *sqlx.Stmt
	contentType string
}

type KeyValuePair struct {
	Name  string
	Value string
}

var (
	pool       *groupcache.HTTPPool
	cache      *groupcache.Group
	dbClients  map[string]DBClient
	blankPNG   []byte
	cacheSince = time.Now().Format(http.TimeFormat)
)

func main() {
	_, err := flags.ParseArgs(&options, os.Args)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println("Flags:", options)
	blankPNG, _ = ioutil.ReadFile("blank.png")

	dbClients = make(map[string]DBClient)
	tilesets, _ := filepath.Glob(path.Join(options.Tilesets, "*.mbtiles"))

	for _, filename := range tilesets {
		_, service := filepath.Split(filename)
		service = strings.Split(service, ".")[0]

		db, err := sqlx.Open("sqlite3", filename)
		if err != nil {
			log.Fatal(err)
		}
		defer db.Close()

		infoStmt, err := db.Preparex("select * from metadata where value is not ''")
		if err != nil {
			log.Fatal(err)
		}
		defer infoStmt.Close()

		stmt, err := db.Prepare("select tile_data from tiles where zoom_level = ? and tile_column = ? and tile_row = ?")
		if err != nil {
			log.Fatal(err)
		}
		defer stmt.Close()

		//query a sample tile to determine if png or jpg, since metadata from tilemill doesn't give this to us
		var tileData []byte
		err = db.QueryRow("select tile_data from images limit 1").Scan(&tileData)
		if err != nil {
			log.Fatal(err)
		}

		dbClients[service] = DBClient{
			connection:  db,
			infoStmt:    infoStmt,
			imageStmt:   stmt,
			contentType: http.DetectContentType(tileData),
		}
	}

	pool = groupcache.NewHTTPPool(fmt.Sprintf("http://127.0.0.1:%v", options.CachePort))
	cache = groupcache.NewGroup("TileCache", options.CacheSize*1048576, groupcache.GetterFunc(
		func(ctx groupcache.Context, key string, dest groupcache.Sink) error {
			pathParams := strings.Split(key, "/")
			service := pathParams[0]
			yParams := strings.Split(pathParams[3], ".")
			z, _ := strconv.ParseUint(pathParams[1], 0, 64)
			x, _ := strconv.ParseUint(pathParams[2], 0, 64)
			y, _ := strconv.ParseUint(yParams[0], 0, 64)
			//flip y to match TMS spec
			y = (1 << z) - 1 - y

			var tileData []byte
			err := dbClients[service].imageStmt.QueryRow(uint8(z), uint16(x), uint16(y)).Scan(&tileData)
			if err != nil {
				if err != sql.ErrNoRows {
					log.Fatal(err)
				}
			}
			dest.SetBytes(tileData)
			return nil
		}))

	c := cors.New(cors.Options{
		AllowedOrigins: []string{"*"},
	})
	goji.Use(c.Handler)

	//TODO: add gzip
	goji.Get("/services", ListServices)
	goji.Get("/services/:service", GetService)
	goji.Get("/services/:service/tiles/:z/:x/:filename", GetTile)
	//TODO:  goji.Get("/:service/grids/:z/:x/:filename", GetGrid) //return UTF8 grid

	goji.Serve()
}

type ServiceInfo struct {
	URI       string `json:"uri"`
	ImageType string `json:"imageType"`
}

func ListServices(c web.C, w http.ResponseWriter, r *http.Request) {
	services := make([]ServiceInfo, len(dbClients))
	i := 0
	for service, _ := range dbClients {
		services[i] = ServiceInfo{
			URI:       fmt.Sprintf("/services/%s", service),
			ImageType: strings.Split(dbClients[service].contentType, "/")[1],
		}
		i++
	}
	w.Header().Add("Content-Type", "application/json")
	json, _ := json.Marshal(services)
	w.Write(json)
}

func GetService(c web.C, w http.ResponseWriter, r *http.Request) {
	service := c.URLParams["service"]
	if _, exists := dbClients[service]; !exists {
		http.Error(w, fmt.Sprintf("Service not found: %s", service), http.StatusNotFound)
		return
	}
	results := make(map[string]string)
	rows, err := dbClients[service].infoStmt.Queryx()
	if err != nil {
		http.Error(w, fmt.Sprintf("Metadata query failed for: %s", service), http.StatusInternalServerError)
		return
	}
	for rows.Next() {
		var record KeyValuePair
		rows.StructScan(&record)
		results[record.Name] = record.Value
	}
	rootURL := fmt.Sprintf("http://%s", options.Hostname)
	if options.Bind != ":80" {
		rootURL = fmt.Sprintf("%s%s", rootURL, options.Bind)
	}

	imgFormat := strings.Split(dbClients[service].contentType, "/")[1]

	out := map[string]interface{}{
		"tilejson": "2.1.0",
		"id":       service,
		"scheme":   "xyz",
		"format":   imgFormat,
		"tiles":    []string{fmt.Sprintf("%s/services/%s/tiles/{z}/{x}/{y}.%s", rootURL, service, strings.Replace(imgFormat, "jpeg", "jpg", 1))},
	}

	for k := range results {
		out[k] = results[k]
	}
	var value string
	var ok bool
	multiFloatFields := []string{"bounds", "center"}
	intFields := []string{"maxzoom", "minzoom"}
	for _, field := range multiFloatFields {
		if value, ok = results[field]; ok {
			out[field] = stringToFloats(value)
		}
	}
	for _, field := range intFields {
		if value, ok = results[field]; ok {
			out[field], _ = strconv.Atoi(results[field])
		}
	}

	w.Header().Add("Content-Type", "application/json")
	json, _ := json.Marshal(out)
	w.Write(json)
}

func GetTile(c web.C, w http.ResponseWriter, r *http.Request) {
	var (
		data        []byte
		contentType string
	)
	//TODO: validate x, y, z

	service := c.URLParams["service"]
	if _, exists := dbClients[service]; !exists {
		http.Error(w, fmt.Sprintf("Service not found: %s", service), http.StatusNotFound)
		return
	}

	key := strings.Join([]string{c.URLParams["service"], c.URLParams["z"], c.URLParams["x"], c.URLParams["filename"]}, "/")
	fmt.Println("URL key: ", key)

	err := cache.Get(nil, key, groupcache.AllocatingByteSliceSink(&data))
	if err != nil {
		log.Println("Error fetching key", key)
		http.Error(w, fmt.Sprintf("Cache get failed for key: %s", key), http.StatusInternalServerError)
		return
	}
	etag := fmt.Sprintf("%x", md5.Sum(data))

	if r.Header.Get("If-None-Match") == etag {
		w.WriteHeader(http.StatusNotModified)
		return
	}

	if len(data) <= 1 {
		data = blankPNG
		contentType = "image/png"
	} else {
		contentType = dbClients[service].contentType
	}

	w.Header().Add("Cache-Control", fmt.Sprintf("max-age=%v", options.MaxAge))
	w.Header().Add("Last-Modified", cacheSince)
	w.Header().Add("Content-Type", contentType)
	w.Header().Add("ETag", etag)
	w.Write(data)
}

func stringToFloats(str string) []float32 {
	split := strings.Split(str, ",")
	var out = make([]float32, len(split))
	for i, v := range split {
		value, _ := strconv.ParseFloat(v, 32)
		out[i] = float32(value)
	}
	return out
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"errors"
	"io/ioutil"
	"log"
	"os"
	"os/user"
	"path"
	"strings"
	"syscall"

	"github.com/codegangsta/cli"
	"github.com/hpcloud/tail"
)

type EdwardConfiguration struct {
	Dir    string
	LogDir string
	PidDir string
}

var EdwardConfig EdwardConfiguration = EdwardConfiguration{}

func createDirIfNeeded(path string) {
	if _, err := os.Stat(path); os.IsNotExist(err) {
		os.MkdirAll(path, 0777)
	}
}

func (e *EdwardConfiguration) initialize() error {
	user, err := user.Current()
	if err != nil {
		return err
	}
	e.Dir = path.Join(user.HomeDir, ".edward")
	e.LogDir = path.Join(e.Dir, "logs")
	e.PidDir = path.Join(e.Dir, "pidFiles")
	createDirIfNeeded(e.Dir)
	createDirIfNeeded(e.LogDir)
	createDirIfNeeded(e.PidDir)
	return nil
}

var groups map[string]*ServiceGroupConfig
var services map[string]*ServiceConfig

func thirdPartyService(name string, startCommand string, stopCommand string, started string) *ServiceConfig {
	pathStr := "$ALPHA"
	return &ServiceConfig{
		Name: name,
		Path: &pathStr,
		Commands: ServiceConfigCommands{
			Launch: startCommand,
			Stop:   stopCommand,
		},
		Properties: ServiceConfigProperties{
			Started: started,
		},
	}
}

func playService(name string) *ServiceConfig {
	pathStr := "$ALPHA"
	return &ServiceConfig{
		Name: name,
		Path: &pathStr,
		Commands: ServiceConfigCommands{
			Build:  "python tools/icbm/build.py :" + name + "_dev",
			Launch: "YEXT_RABBITMQ=localhost thirdparty/play/play test src/com/yext/" + name,
		},
		Properties: ServiceConfigProperties{
			Started: "Server is up and running",
		},
	}
}

func javaService(name string) *ServiceConfig {
	pathStr := "$ALPHA"
	return &ServiceConfig{
		Name: name,
		Path: &pathStr,
		Commands: ServiceConfigCommands{
			Build:  "python tools/icbm/build.py :" + name,
			Launch: "YEXT_RABBITMQ=localhost YEXT_SITE=office JVM_ARGS='-Xmx3G' build/" + name + "/" + name,
		},
		Properties: ServiceConfigProperties{
			Started: "Listening",
		},
	}
}

func goService(name string, goPackage string) *ServiceConfig {
	pathStr := "$ALPHA"
	return &ServiceConfig{
		Name: name,
		Path: &pathStr,
		Commands: ServiceConfigCommands{
			Build:  "go install " + goPackage,
			Launch: "YEXT_RABBITMQ=localhost " + name,
		},
		Properties: ServiceConfigProperties{
			Started: "Listening",
		},
	}
}

func loadConfig() {
	// TODO: Load configuration from the config file and populate the service and groups variables

	groups = make(map[string]*ServiceGroupConfig)
	services = make(map[string]*ServiceConfig)

	services["rabbitmq"] = thirdPartyService("rabbitmq", "sudo rabbitmq-server", "sudo rabbitmqctl stop", "completed")
	// TODO: haproxy actually needs a kill -9 to effectively die
	// TODO: haproxy also doesn't have an effective start output
	services["haproxy"] = thirdPartyService("haproxy", "sudo $ALPHA/tools/bin/haproxy_localhost.sh", "", "backend")

	services["admin2"] = playService("admin2")
	services["users"] = playService("users")
	services["storm"] = playService("storm")
	services["ProfileServer"] = javaService("ProfileServer")

	services["sites-staging"] = goService("sites-staging", "yext/pages/sites/sites-staging")
	services["sites-storm"] = goService("sites-storm", "yext/pages/sites/sites-storm")
	services["sites-cog"] = goService("sites-cog", "yext/pages/sites/sites-cog")

	groups["thirdparty"] = &ServiceGroupConfig{
		Name: "thirdparty",
		Services: []*ServiceConfig{
			services["rabbitmq"],
			services["haproxy"],
		},
	}

	groups["base"] = &ServiceGroupConfig{
		Name: "base",
		Groups: []*ServiceGroupConfig{
			groups["thirdparty"],
		},
		Services: []*ServiceConfig{
			services["admin2"],
			services["users"],
			services["storm"],
			services["ProfileServer"],
		},
	}

	groups["pages"] = &ServiceGroupConfig{
		Name: "pages",
		Groups: []*ServiceGroupConfig{
			groups["base"],
		},
		Services: []*ServiceConfig{
			services["sites-staging"],
			services["sites-storm"],
			services["sites-cog"],
		},
	}
}

func getServiceOrGroup(name string) (ServiceOrGroup, error) {
	if group, ok := groups[name]; ok {
		return group, nil
	}
	if service, ok := services[name]; ok {
		return service, nil
	}
	return nil, errors.New("Service or group not found")
}

func list(c *cli.Context) {
	println("Services and groups")
	println("Groups:")
	for name, _ := range groups {
		println("\t", name)
	}
	println("Services:")
	for name, _ := range services {
		println("\t", name)
	}
}

func generate(c *cli.Context) {
	println("Generate config")
}

func status(c *cli.Context) {
	name := c.Args()[0]
	s, err := getServiceOrGroup(name)
	if err != nil {
		log.Fatal(err)
	}
	statuses := s.GetStatus()
	for _, status := range statuses {
		println(status.Service.Name, ":", status.Status)
	}
}

func messages(c *cli.Context) {
	println("Messages")
}

func start(c *cli.Context) {
	name := c.Args()[0]
	s, err := getServiceOrGroup(name)
	if err != nil {
		log.Fatal(err)
	}
	err = s.Build()
	if err != nil {
		log.Fatal(err)
	}
	err = s.Start()
	if err != nil {
		log.Fatal(err)
	}
}

func stop(c *cli.Context) {
	name := c.Args()[0]
	s, err := getServiceOrGroup(name)
	if err != nil {
		log.Fatal(err)
	}
	err = s.Stop()
	if err != nil {
		log.Fatal(err)
	}
}

func restart(c *cli.Context) {
	name := c.Args()[0]
	s, err := getServiceOrGroup(name)
	if err != nil {
		log.Fatal(err)
	}
	err = s.Stop()
	if err != nil {
		log.Fatal(err)
	}
	err = s.Build()
	if err != nil {
		log.Fatal(err)
	}
	err = s.Start()
	if err != nil {
		log.Fatal(err)
	}
}

func doLog(c *cli.Context) {
	name := c.Args()[0]
	if _, ok := groups[name]; ok {
		log.Fatal(errors.New("Cannot output group logs"))
	}
	if service, ok := services[name]; ok {
		command := service.GetCommand()
		runLog := command.Logs.Run
		t, err := tail.TailFile(runLog, tail.Config{Follow: true})
		if err != nil {
			log.Fatal(err)
		}
		for line := range t.Lines {
			println(line.Text)
		}
		return
	}
	log.Fatal("Service not found:", name)
}

func checkNotSudo() {
	user, err := user.Current()
	if err != nil {
		log.Fatal(err)
	}
	if user.Uid == "0" {
		log.Fatal("edward should not be run with sudo")
	}
}

func createScriptFile(suffix string, content string) (*os.File, error) {
	file, err := ioutil.TempFile(os.TempDir(), suffix)
	if err != nil {
		return nil, err
	}
	file.WriteString(content)

	err = os.Chmod(file.Name(), 0777)
	if err != nil {
		return nil, err
	}

	return file, nil
}

func ensureSudoAble() {
	var buffer bytes.Buffer

	buffer.WriteString("#!/bin/bash\n")
	buffer.WriteString("sudo echo Test > /dev/null\n")
	buffer.WriteString("ISCHILD=YES ")
	buffer.WriteString(strings.Join(os.Args, " "))
	buffer.WriteString("\n")

	file, err := createScriptFile("sudoAbility", buffer.String())
	if err != nil {
		log.Fatal(err)
	}

	err = syscall.Exec(file.Name(), []string{file.Name()}, os.Environ())
	if err != nil {
		log.Fatal(err)
	}
}

func main() {

	checkNotSudo()

	isChild := os.Getenv("ISCHILD")
	if isChild == "" {
		ensureSudoAble()
		return
	}

	app := cli.NewApp()
	app.Name = "Edward"
	app.Usage = "Manage local microservices"
	app.Before = func(c *cli.Context) error {
		err := EdwardConfig.initialize()
		if err != nil {
			return err
		}
		loadConfig()
		return nil
	}
	app.Commands = []cli.Command{
		{
			Name:   "list",
			Usage:  "List available services",
			Action: list,
		},
		{
			Name:   "generate",
			Usage:  "Generate Edward config for a source tree",
			Action: generate,
		},
		{
			Name:   "status",
			Usage:  "Display service status",
			Action: status,
		},
		{
			Name:   "messages",
			Usage:  "Show messages from services",
			Action: messages,
		},
		{
			Name:   "start",
			Usage:  "Build and launch a service",
			Action: start,
		},
		{
			Name:   "stop",
			Usage:  "Stop a service",
			Action: stop,
		},
		{
			Name:   "restart",
			Usage:  "Rebuild and relaunch a service",
			Action: restart,
		},
		{
			Name:   "log",
			Usage:  "Tail the log for a service",
			Action: doLog,
		},
	}

	app.Run(os.Args)
}
<commit_msg>Show all service statuses when no service/group specified. Add locationsstorm to storm group (name changed from base).<commit_after>package main

import (
	"bytes"
	"errors"
	"io/ioutil"
	"log"
	"os"
	"os/user"
	"path"
	"strings"
	"syscall"

	"github.com/codegangsta/cli"
	"github.com/hpcloud/tail"
)

type EdwardConfiguration struct {
	Dir    string
	LogDir string
	PidDir string
}

var EdwardConfig EdwardConfiguration = EdwardConfiguration{}

func createDirIfNeeded(path string) {
	if _, err := os.Stat(path); os.IsNotExist(err) {
		os.MkdirAll(path, 0777)
	}
}

func (e *EdwardConfiguration) initialize() error {
	user, err := user.Current()
	if err != nil {
		return err
	}
	e.Dir = path.Join(user.HomeDir, ".edward")
	e.LogDir = path.Join(e.Dir, "logs")
	e.PidDir = path.Join(e.Dir, "pidFiles")
	createDirIfNeeded(e.Dir)
	createDirIfNeeded(e.LogDir)
	createDirIfNeeded(e.PidDir)
	return nil
}

var groups map[string]*ServiceGroupConfig
var services map[string]*ServiceConfig

func thirdPartyService(name string, startCommand string, stopCommand string, started string) *ServiceConfig {
	pathStr := "$ALPHA"
	return &ServiceConfig{
		Name: name,
		Path: &pathStr,
		Commands: ServiceConfigCommands{
			Launch: startCommand,
			Stop:   stopCommand,
		},
		Properties: ServiceConfigProperties{
			Started: started,
		},
	}
}

func playService(name string) *ServiceConfig {
	pathStr := "$ALPHA"
	return &ServiceConfig{
		Name: name,
		Path: &pathStr,
		Commands: ServiceConfigCommands{
			Build:  "python tools/icbm/build.py :" + name + "_dev",
			Launch: "YEXT_RABBITMQ=localhost thirdparty/play/play test src/com/yext/" + name,
		},
		Properties: ServiceConfigProperties{
			Started: "Server is up and running",
		},
	}
}

func javaService(name string) *ServiceConfig {
	pathStr := "$ALPHA"
	return &ServiceConfig{
		Name: name,
		Path: &pathStr,
		Commands: ServiceConfigCommands{
			Build:  "python tools/icbm/build.py :" + name,
			Launch: "YEXT_RABBITMQ=localhost YEXT_SITE=office JVM_ARGS='-Xmx3G' build/" + name + "/" + name,
		},
		Properties: ServiceConfigProperties{
			Started: "Listening",
		},
	}
}

func goService(name string, goPackage string) *ServiceConfig {
	pathStr := "$ALPHA"
	return &ServiceConfig{
		Name: name,
		Path: &pathStr,
		Commands: ServiceConfigCommands{
			Build:  "go install " + goPackage,
			Launch: "YEXT_RABBITMQ=localhost " + name,
		},
		Properties: ServiceConfigProperties{
			Started: "Listening",
		},
	}
}

func loadConfig() {
	// TODO: Load configuration from the config file and populate the service and groups variables

	groups = make(map[string]*ServiceGroupConfig)
	services = make(map[string]*ServiceConfig)

	services["rabbitmq"] = thirdPartyService("rabbitmq", "sudo rabbitmq-server", "sudo rabbitmqctl stop", "completed")
	// TODO: haproxy actually needs a kill -9 to effectively die
	// TODO: haproxy also doesn't have an effective start output
	services["haproxy"] = thirdPartyService("haproxy", "sudo $ALPHA/tools/bin/haproxy_localhost.sh", "", "backend")

	services["admin2"] = playService("admin2")
	services["users"] = playService("users")
	services["storm"] = playService("storm")
	services["locationsstorm"] = playService("locationsstorm")
	services["ProfileServer"] = javaService("ProfileServer")

	services["sites-staging"] = goService("sites-staging", "yext/pages/sites/sites-staging")
	services["sites-storm"] = goService("sites-storm", "yext/pages/sites/sites-storm")
	services["sites-cog"] = goService("sites-cog", "yext/pages/sites/sites-cog")

	groups["thirdparty"] = &ServiceGroupConfig{
		Name: "thirdparty",
		Services: []*ServiceConfig{
			services["rabbitmq"],
			services["haproxy"],
		},
	}

	groups["storm"] = &ServiceGroupConfig{
		Name: "storm",
		Groups: []*ServiceGroupConfig{
			groups["thirdparty"],
		},
		Services: []*ServiceConfig{
			services["admin2"],
			services["users"],
			services["storm"],
			services["locationsstorm"],
			services["ProfileServer"],
		},
	}

	groups["pages"] = &ServiceGroupConfig{
		Name: "pages",
		Groups: []*ServiceGroupConfig{
			groups["storm"],
		},
		Services: []*ServiceConfig{
			services["sites-staging"],
			services["sites-storm"],
			services["sites-cog"],
		},
	}
}

func getServiceOrGroup(name string) (ServiceOrGroup, error) {
	if group, ok := groups[name]; ok {
		return group, nil
	}
	if service, ok := services[name]; ok {
		return service, nil
	}
	return nil, errors.New("Service or group not found")
}

func list(c *cli.Context) {
	println("Services and groups")
	println("Groups:")
	for name, _ := range groups {
		println("\t", name)
	}
	println("Services:")
	for name, _ := range services {
		println("\t", name)
	}
}

func generate(c *cli.Context) {
	println("Generate config")
}

func allStatus() {
	var statuses []ServiceStatus
	for _, service := range services {
		statuses = append(statuses, service.GetStatus()...)
	}
	for _, status := range statuses {
		println(status.Service.Name, ":", status.Status)
	}
}

func status(c *cli.Context) {

	if len(c.Args()) == 0 {
		allStatus()
		return
	}

	name := c.Args()[0]
	s, err := getServiceOrGroup(name)
	if err != nil {
		log.Fatal(err)
	}
	statuses := s.GetStatus()
	for _, status := range statuses {
		println(status.Service.Name, ":", status.Status)
	}
}

func messages(c *cli.Context) {
	println("Messages")
}

func start(c *cli.Context) {
	name := c.Args()[0]
	s, err := getServiceOrGroup(name)
	if err != nil {
		log.Fatal(err)
	}
	err = s.Build()
	if err != nil {
		log.Fatal(err)
	}
	err = s.Start()
	if err != nil {
		log.Fatal(err)
	}
}

func stop(c *cli.Context) {
	name := c.Args()[0]
	s, err := getServiceOrGroup(name)
	if err != nil {
		log.Fatal(err)
	}
	err = s.Stop()
	if err != nil {
		log.Fatal(err)
	}
}

func restart(c *cli.Context) {
	name := c.Args()[0]
	s, err := getServiceOrGroup(name)
	if err != nil {
		log.Fatal(err)
	}
	err = s.Stop()
	if err != nil {
		log.Fatal(err)
	}
	err = s.Build()
	if err != nil {
		log.Fatal(err)
	}
	err = s.Start()
	if err != nil {
		log.Fatal(err)
	}
}

func doLog(c *cli.Context) {
	name := c.Args()[0]
	if _, ok := groups[name]; ok {
		log.Fatal(errors.New("Cannot output group logs"))
	}
	if service, ok := services[name]; ok {
		command := service.GetCommand()
		runLog := command.Logs.Run
		t, err := tail.TailFile(runLog, tail.Config{Follow: true})
		if err != nil {
			log.Fatal(err)
		}
		for line := range t.Lines {
			println(line.Text)
		}
		return
	}
	log.Fatal("Service not found:", name)
}

func checkNotSudo() {
	user, err := user.Current()
	if err != nil {
		log.Fatal(err)
	}
	if user.Uid == "0" {
		log.Fatal("edward should not be run with sudo")
	}
}

func createScriptFile(suffix string, content string) (*os.File, error) {
	file, err := ioutil.TempFile(os.TempDir(), suffix)
	if err != nil {
		return nil, err
	}
	file.WriteString(content)

	err = os.Chmod(file.Name(), 0777)
	if err != nil {
		return nil, err
	}

	return file, nil
}

func ensureSudoAble() {
	var buffer bytes.Buffer

	buffer.WriteString("#!/bin/bash\n")
	buffer.WriteString("sudo echo Test > /dev/null\n")
	buffer.WriteString("ISCHILD=YES ")
	buffer.WriteString(strings.Join(os.Args, " "))
	buffer.WriteString("\n")

	file, err := createScriptFile("sudoAbility", buffer.String())
	if err != nil {
		log.Fatal(err)
	}

	err = syscall.Exec(file.Name(), []string{file.Name()}, os.Environ())
	if err != nil {
		log.Fatal(err)
	}
}

func main() {

	checkNotSudo()

	isChild := os.Getenv("ISCHILD")
	if isChild == "" {
		ensureSudoAble()
		return
	}

	app := cli.NewApp()
	app.Name = "Edward"
	app.Usage = "Manage local microservices"
	app.Before = func(c *cli.Context) error {
		err := EdwardConfig.initialize()
		if err != nil {
			return err
		}
		loadConfig()
		return nil
	}
	app.Commands = []cli.Command{
		{
			Name:   "list",
			Usage:  "List available services",
			Action: list,
		},
		{
			Name:   "generate",
			Usage:  "Generate Edward config for a source tree",
			Action: generate,
		},
		{
			Name:   "status",
			Usage:  "Display service status",
			Action: status,
		},
		{
			Name:   "messages",
			Usage:  "Show messages from services",
			Action: messages,
		},
		{
			Name:   "start",
			Usage:  "Build and launch a service",
			Action: start,
		},
		{
			Name:   "stop",
			Usage:  "Stop a service",
			Action: stop,
		},
		{
			Name:   "restart",
			Usage:  "Rebuild and relaunch a service",
			Action: restart,
		},
		{
			Name:   "log",
			Usage:  "Tail the log for a service",
			Action: doLog,
		},
	}

	app.Run(os.Args)
}
<|endoftext|>


<commit_before>package types

// A Place isn't something directly used by the Navitia.io api
// However, it allows the library user to use idiomatic go when working with the library
// If you want a countainer, see PlaceCountainer
// Place is satisfied by:
// - StopArea
// - POI
// - Address
// - StopPoint
// - AdministrativeRegion
type Place interface {
	// PlaceID returns the ID associated with the Place
	PlaceID() ID

	// PlaceName returns the name of the Place
	PlaceName() string

	// PlaceType returns the name of the type of the Place
	PlaceType() string
}

// A StopArea represents a stop area: a place where a public transportation method may stop for a traveller.
type StopArea struct {
	ID   ID     `json:"id"`
	Name string `json:"name"`

	// Label of the stop area.
	// The name is directly taken from the data whereas the label is something computed by navitia for better traveler information.
	// If you don't know what to display, display the label
	Label string `json:"label"`

	// Coordinates of the stop area
	Coord Coordinates `json:"coord"`

	// Administrative regions of the stop area in which is placed the stop area
	AdministrativeRegions []AdministrativeRegion `json:"administrative_regions"`

	// Stop points countained in this stop area
	StopPoints []StopPoint `json:"stop_points"`
}

// PlaceID returns the ID associated with the StopArea
// Helps satisfy Place
func (sa StopArea) PlaceID() ID {
	return sa.ID
}

// PlaceName returns the name of the StopArea
// Helps satisfy Place
func (sa StopArea) PlaceName() string {
	return sa.Name
}

// PlaceType returns the type of place, in this case "stop_area"
// Helps satisfy Place
func (sa StopArea) PlaceType() string {
	return "stop_area"
}

// A POI is a Point Of Interest. A loosely-defined place.
type POI struct {
	ID   ID     `json:"id"`
	Name string `json:"name"`

	// The name is directly taken from the data whereas the label is something computed by navitia for better traveler information.
	// If you don't know what to display, display the label
	Label string `json:"label"`

	// The type of the POI
	Type POIType `json:"poi_type"`
}

// PlaceID returns the ID associated with the POI.
// Helps satisfy Place
func (poi POI) PlaceID() ID {
	return poi.ID
}

// PlaceName returns the name of the POI
// Helps satisfy Place
func (poi POI) PlaceName() string {
	return poi.Name
}

// PlaceType returns the type of place, in this case "poi"
// Helps satisfy Place
func (poi POI) PlaceType() string {
	return "poi"
}

// A POIType codes for the type of the point of interest
// TODO: A list of usual types ?
type POIType struct {
	ID   ID     `json:"id"`
	Name string `json:"name"`
}

// An Address codes for a real-world address
type Address struct {
	ID   ID     `json:"id"`
	Name string `json:"name"`

	// Label of the address
	// The name is directly taken from the data whereas the label is something computed by navitia for better traveler information.
	// If you don't know what to display, display the label
	Label string `json:"label"`

	// Coordinates of the address
	Coord Coordinates `json:"coord"`

	// House number of the address
	HouseNumber uint `json:"house_number"`

	// Administrative regions of the stop area in which is placed the stop area
	AdministrativeRegions []AdministrativeRegion `json:"administrative_regions"`
}

// PlaceID returns the ID associated with the Address
// Helps satisfy Place
func (add Address) PlaceID() ID {
	return add.ID
}

// PlaceName returns the name of the Address
// Helps satisfy Place
func (add Address) PlaceName() string {
	return add.Name
}

// PlaceType returns the type of place, in this case "address"
// Helps satisfy Place
func (add Address) PlaceType() string {
	return "address"
}

// A StopPoint codes for a stop point in a line
type StopPoint struct {
	ID ID `json:"id"`

	// Name of the stop point
	Name string `json:"name"`

	// Coordinates of the stop point
	Coord Coordinates `json:"coord"`

	// Administrative regions of the stop point
	AdministrativeRegions []AdministrativeRegion `json:"administrative_regions"`

	// List of equipments of the stop point
	Equipments []Equipment `json:"equipment"`

	// Stop Area countaining the stop point
	StopArea *StopArea `json:"stop_area"`
}

// PlaceID returns the ID associated with the Stop Point
// Helps satisfy Place
func (sp StopPoint) PlaceID() ID {
	return sp.ID
}

// PlaceName returns the name of the Stop Point
// Helps satisfy Place
func (sp StopPoint) PlaceName() string {
	return sp.Name
}

// PlaceType returns the type of place, in this case "stop_point"
// Helps satisfy Place
func (sp StopPoint) PlaceType() string {
	return "stop_point"
}

// An AdministrativeRegion represents an administrative region: a region under the control/responsibility of a specific organisation.
type AdministrativeRegion struct {
	ID   ID     `json:"id"`
	Name string `json:"name"`

	// Label of the address
	// The name is directly taken from the data whereas the label is something computed by navitia for better traveler information.
	// If you don't know what to display, display the label
	Label string `json:"label"`

	// Coordinates of the administrative region
	Coord Coordinates `json:"coord"`

	// Level of the administrative region
	Level int `json:"level"`

	// Zip code of the administrative region
	ZipCode string
}

// PlaceID returns the ID associated with the AdministrativeRegion
// Helps satisfy Place
func (ar AdministrativeRegion) PlaceID() ID {
	return ar.ID
}

// PlaceName returns the name of the AdministrativeRegion
// Helps satisfy Place
func (ar AdministrativeRegion) PlaceName() string {
	return ar.Name
}

// PlaceType returns the type of place, in this case "administrative_region"
// Helps satisfy Place
func (ar AdministrativeRegion) PlaceType() string {
	return "administrative_region"
}
<commit_msg>Add PlaceCountainer and associated Place method<commit_after>package types

//import "github.com/pkg/errors"

// A Place isn't something directly used by the Navitia.io api
// However, it allows the library user to use idiomatic go when working with the library
// If you want a countainer, see PlaceCountainer
// Place is satisfied by:
// - StopArea
// - POI
// - Address
// - StopPoint
// - AdministrativeRegion
type Place interface {
	// PlaceID returns the ID associated with the Place
	PlaceID() ID

	// PlaceName returns the name of the Place
	PlaceName() string

	// PlaceType returns the name of the type of the Place
	PlaceType() string
}

// PlaceCountainer is the ugly countainer sent by Navitia to make us all cry.
// However, as this can be useful. May be removed from the public API in gonavitia v0.
type PlaceCountainer struct {
	ID           ID     `json:"id"`
	Name         string `json:"name"`
	Quality      uint   `json:"quality,omitempty"`
	EmbeddedType string `json:"embedded_type"`

	// Four possibilitiess
	StopArea             *StopArea             `json:"stop_area,omitempty"`
	POI                  *POI                  `json:"POI,omitempty"`
	Address              *Address              `json:"address,omitempty"`
	StopPoint            *StopPoint            `json:"stop_point,omitempty"`
	AdministrativeRegion *AdministrativeRegion `json:"administrative_region,omitempty"`
}

// Place returns the Place countained in the PlaceCountainer
func (pc PlaceCountainer) Place() (Place, error) {
	switch pc.EmbeddedType {
	case "stop_area":
		return pc.StopArea, nil
	case "poi":
		return pc.POI, nil
	case "address":
		return pc.Address, nil
	case "stop_point":
		return pc.StopPoint, nil
	case "administrative_region":
		return pc.AdministrativeRegion, nil
	default:
		return nil, nil //errors.Errorf("No known embedded type indicated (we have \"%s\"), can't return a place !", pc.EmbeddedType)
	}
}

// A StopArea represents a stop area: a place where a public transportation method may stop for a traveller.
type StopArea struct {
	ID   ID     `json:"id"`
	Name string `json:"name"`

	// Label of the stop area.
	// The name is directly taken from the data whereas the label is something computed by navitia for better traveler information.
	// If you don't know what to display, display the label
	Label string `json:"label"`

	// Coordinates of the stop area
	Coord Coordinates `json:"coord"`

	// Administrative regions of the stop area in which is placed the stop area
	AdministrativeRegions []AdministrativeRegion `json:"administrative_regions"`

	// Stop points countained in this stop area
	StopPoints []StopPoint `json:"stop_points"`
}

// PlaceID returns the ID associated with the StopArea
// Helps satisfy Place
func (sa StopArea) PlaceID() ID {
	return sa.ID
}

// PlaceName returns the name of the StopArea
// Helps satisfy Place
func (sa StopArea) PlaceName() string {
	return sa.Name
}

// PlaceType returns the type of place, in this case "stop_area"
// Helps satisfy Place
func (sa StopArea) PlaceType() string {
	return "stop_area"
}

// A POI is a Point Of Interest. A loosely-defined place.
type POI struct {
	ID   ID     `json:"id"`
	Name string `json:"name"`

	// The name is directly taken from the data whereas the label is something computed by navitia for better traveler information.
	// If you don't know what to display, display the label
	Label string `json:"label"`

	// The type of the POI
	Type POIType `json:"poi_type"`
}

// PlaceID returns the ID associated with the POI.
// Helps satisfy Place
func (poi POI) PlaceID() ID {
	return poi.ID
}

// PlaceName returns the name of the POI
// Helps satisfy Place
func (poi POI) PlaceName() string {
	return poi.Name
}

// PlaceType returns the type of place, in this case "poi"
// Helps satisfy Place
func (poi POI) PlaceType() string {
	return "poi"
}

// A POIType codes for the type of the point of interest
// TODO: A list of usual types ?
type POIType struct {
	ID   ID     `json:"id"`
	Name string `json:"name"`
}

// An Address codes for a real-world address
type Address struct {
	ID   ID     `json:"id"`
	Name string `json:"name"`

	// Label of the address
	// The name is directly taken from the data whereas the label is something computed by navitia for better traveler information.
	// If you don't know what to display, display the label
	Label string `json:"label"`

	// Coordinates of the address
	Coord Coordinates `json:"coord"`

	// House number of the address
	HouseNumber uint `json:"house_number"`

	// Administrative regions of the stop area in which is placed the stop area
	AdministrativeRegions []AdministrativeRegion `json:"administrative_regions"`
}

// PlaceID returns the ID associated with the Address
// Helps satisfy Place
func (add Address) PlaceID() ID {
	return add.ID
}

// PlaceName returns the name of the Address
// Helps satisfy Place
func (add Address) PlaceName() string {
	return add.Name
}

// PlaceType returns the type of place, in this case "address"
// Helps satisfy Place
func (add Address) PlaceType() string {
	return "address"
}

// A StopPoint codes for a stop point in a line
type StopPoint struct {
	ID ID `json:"id"`

	// Name of the stop point
	Name string `json:"name"`

	// Coordinates of the stop point
	Coord Coordinates `json:"coord"`

	// Administrative regions of the stop point
	AdministrativeRegions []AdministrativeRegion `json:"administrative_regions"`

	// List of equipments of the stop point
	Equipments []Equipment `json:"equipment"`

	// Stop Area countaining the stop point
	StopArea *StopArea `json:"stop_area"`
}

// PlaceID returns the ID associated with the Stop Point
// Helps satisfy Place
func (sp StopPoint) PlaceID() ID {
	return sp.ID
}

// PlaceName returns the name of the Stop Point
// Helps satisfy Place
func (sp StopPoint) PlaceName() string {
	return sp.Name
}

// PlaceType returns the type of place, in this case "stop_point"
// Helps satisfy Place
func (sp StopPoint) PlaceType() string {
	return "stop_point"
}

// An AdministrativeRegion represents an administrative region: a region under the control/responsibility of a specific organisation.
type AdministrativeRegion struct {
	ID   ID     `json:"id"`
	Name string `json:"name"`

	// Label of the address
	// The name is directly taken from the data whereas the label is something computed by navitia for better traveler information.
	// If you don't know what to display, display the label
	Label string `json:"label"`

	// Coordinates of the administrative region
	Coord Coordinates `json:"coord"`

	// Level of the administrative region
	Level int `json:"level"`

	// Zip code of the administrative region
	ZipCode string
}

// PlaceID returns the ID associated with the AdministrativeRegion
// Helps satisfy Place
func (ar AdministrativeRegion) PlaceID() ID {
	return ar.ID
}

// PlaceName returns the name of the AdministrativeRegion
// Helps satisfy Place
func (ar AdministrativeRegion) PlaceName() string {
	return ar.Name
}

// PlaceType returns the type of place, in this case "administrative_region"
// Helps satisfy Place
func (ar AdministrativeRegion) PlaceType() string {
	return "administrative_region"
}
<|endoftext|>