<commit_before>package main

import (
	"archive/zip"
	"io"
	"log"
	"net/http"
	"os"
	"strconv"
	"sync"
	"time"

	"github.com/sethgrid/pester"
	"github.com/urfave/negroni"

	_ "github.com/heroku/x/hmetrics/onload"
)

type FileMetadata struct {
	path string
	url  string
}

type RemoteFile struct {
	path     string
	response *http.Response
}

func main() {
	port := os.Getenv("PORT")
	if port == "" {
		port = "7777"
	}
	http.ListenAndServe(":"+port, server())
}
func server() http.Handler {
	n := negroni.Classic()
	n.UseHandlerFunc(handleZip)
	return n
}

func handleZip(responseWriter http.ResponseWriter, request *http.Request) {
	downloadConcurrency, err := strconv.Atoi(os.Getenv("DOWNLOAD_CONCURRENCY"))
	if err != nil || downloadConcurrency <= 0 {
		downloadConcurrency = 10
	}
	request.ParseForm()
	fileName := request.FormValue("filename")
	if fileName == "" {
		fileName = "download.zip"
	}
	responseWriter.Header().Add("Content-Disposition", "attachment; filename=\""+fileName+"\"")

	downloaderQueue := make(chan FileMetadata)
	fileQueue := make(chan RemoteFile)
	zipperQueue := make(chan RemoteFile)
	downloadWaitGroup := sync.WaitGroup{}

	completionSignal := make(chan error)

	// Run zipper
	go zipper(responseWriter, zipperQueue, completionSignal)

	// Run downloader pool
	for i := 1; i <= downloadConcurrency; i++ {
		go downloader(downloaderQueue, fileQueue)
	}

	go func() {
		for df := range fileQueue {
			downloadWaitGroup.Done()
			zipperQueue <- df
		}
		close(zipperQueue)
	}()

	for path, urls := range request.PostForm {
		downloadWaitGroup.Add(1)
		downloaderQueue <- FileMetadata{path: path, url: urls[0]}
	}
	close(downloaderQueue)

	downloadWaitGroup.Wait()
	close(fileQueue)

	err = <-completionSignal

	if err != nil {
		handleError(err, responseWriter)
		return
	}

}

func handleError(err error, w http.ResponseWriter) {
	http.Error(w, "error", http.StatusBadRequest)
	log.Println(err)
	return
}

func zipper(writer io.Writer, incoming chan RemoteFile, complete chan error) {
	archive := zip.NewWriter(writer)
	for rf := range incoming {
		zipEntryHeader := &zip.FileHeader{
			Name:   rf.path,
			Method: zip.Deflate,
		}
		zipEntryHeader.SetModTime(time.Now())
		entryWriter, err := archive.CreateHeader(zipEntryHeader)
		if err != nil {
			complete <- err
			return
		}
		io.Copy(entryWriter, rf.response.Body)
		rf.response.Body.Close()
	}
	err := archive.Close()
	if err != nil {
		complete <- err
		return
	}
	complete <- nil
}

func downloader(inputQueue chan FileMetadata, outputQueue chan RemoteFile) {
	for df := range inputQueue {
		download, err := pester.Get(df.url)
		if err != nil {
			log.Println("ERROR: " + err.Error())
		} else {
			outputQueue <- RemoteFile{path: df.path, response: download}
		}
	}
}
<commit_msg>Added CORS headers<commit_after>package main

import (
	"archive/zip"
	"io"
	"log"
	"net/http"
	"os"
	"strconv"
	"sync"
	"time"

	"github.com/sethgrid/pester"
	"github.com/urfave/negroni"

	_ "github.com/heroku/x/hmetrics/onload"
)

type FileMetadata struct {
	path string
	url  string
}

type RemoteFile struct {
	path     string
	response *http.Response
}

func main() {
	port := os.Getenv("PORT")
	if port == "" {
		port = "7777"
	}
	http.ListenAndServe(":"+port, server())
}
func server() http.Handler {
	n := negroni.Classic()
	n.UseHandlerFunc(handleZip)
	return n
}

func handleZip(responseWriter http.ResponseWriter, request *http.Request) {
	downloadConcurrency, err := strconv.Atoi(os.Getenv("DOWNLOAD_CONCURRENCY"))
	if err != nil || downloadConcurrency <= 0 {
		downloadConcurrency = 10
	}
	request.ParseForm()
	fileName := request.FormValue("filename")
	if fileName == "" {
		fileName = "download.zip"
	}
	responseWriter.Header().Add("Content-Disposition", "attachment; filename=\""+fileName+"\"")
	responseWriter.Header().Add("Access-Control-Allow-Origin", "*")

	downloaderQueue := make(chan FileMetadata)
	fileQueue := make(chan RemoteFile)
	zipperQueue := make(chan RemoteFile)
	downloadWaitGroup := sync.WaitGroup{}

	completionSignal := make(chan error)

	// Run zipper
	go zipper(responseWriter, zipperQueue, completionSignal)

	// Run downloader pool
	for i := 1; i <= downloadConcurrency; i++ {
		go downloader(downloaderQueue, fileQueue)
	}

	go func() {
		for df := range fileQueue {
			downloadWaitGroup.Done()
			zipperQueue <- df
		}
		close(zipperQueue)
	}()

	for path, urls := range request.PostForm {
		downloadWaitGroup.Add(1)
		downloaderQueue <- FileMetadata{path: path, url: urls[0]}
	}
	close(downloaderQueue)

	downloadWaitGroup.Wait()
	close(fileQueue)

	err = <-completionSignal

	if err != nil {
		handleError(err, responseWriter)
		return
	}

}

func handleError(err error, w http.ResponseWriter) {
	http.Error(w, "error", http.StatusBadRequest)
	log.Println(err)
	return
}

func zipper(writer io.Writer, incoming chan RemoteFile, complete chan error) {
	archive := zip.NewWriter(writer)
	for rf := range incoming {
		zipEntryHeader := &zip.FileHeader{
			Name:   rf.path,
			Method: zip.Deflate,
		}
		zipEntryHeader.SetModTime(time.Now())
		entryWriter, err := archive.CreateHeader(zipEntryHeader)
		if err != nil {
			complete <- err
			return
		}
		io.Copy(entryWriter, rf.response.Body)
		rf.response.Body.Close()
	}
	err := archive.Close()
	if err != nil {
		complete <- err
		return
	}
	complete <- nil
}

func downloader(inputQueue chan FileMetadata, outputQueue chan RemoteFile) {
	for df := range inputQueue {
		download, err := pester.Get(df.url)
		if err != nil {
			log.Println("ERROR: " + err.Error())
		} else {
			outputQueue <- RemoteFile{path: df.path, response: download}
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
//  "flag"
  "fmt"
//  "os"
//  "strings"
)

func main() {
  fmt.Println("allo there from riddles")
}
<commit_msg>add flags<commit_after>package main

import (
	"flag"
	"fmt"
	//  "os"
	//  "strings"
)

func main() {

	idPtr := flag.Int("id", 0, "Access riddle with id.")

	rPtr := flag.Bool("r", false, "")
	randomPtr := flag.Bool("random", true, "Random riddle.")

	aPtr := flag.Bool("a", false, "")
	answerPtr := flag.Bool("answer", false, "Show answer.")
	flag.Parse()

	fmt.Printf("id: %d, random: %t %t, answer: %t %t\n", *idPtr, *rPtr, *randomPtr, *aPtr, *answerPtr)

}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net/http"

	"github.com/go-martini/martini"
	"github.com/martini-contrib/binding"
	"github.com/martini-contrib/render"
	"github.com/xyproto/auth"
	"github.com/xyproto/fizz"
	"github.com/xyproto/instapage"
)

const (
	Version       = "1.0"
	API           = "/api/" + Version + "/"
	Title         = "Score Server " + Version
	AdminUsername = "admin"
)

type RegisterAdmin struct {
	Password1 string `form:"password1" binding:"required"`
	Password2 string `form:"password2" binding:"required"`
	Email     string `form:"email" binding:"required"`
}

type LoginAdmin struct {
	Password string `form:"password" binding:"required"`
}

// Helper function for converting a bool to "yes" or "no"
func b2yn(yesno bool) string {
	if yesno {
		return "yes"
	}
	return "no"
}

func main() {
	fmt.Println(Title)

	// New Martini
	m := martini.Classic()

	// New Renderer
	r := render.Renderer(render.Options{})
	m.Use(r)

	// Initiate the user system
	fizz := fizz.NewWithRedisConf(7, "")
	userstate := fizz.UserState()

	// --- Public pages and admin panel ---

	// Public page
	m.Get("/", func(r render.Render) {
		msg := "Everything is fine."
		if !userstate.HasUser(AdminUsername) {
			msg = "No registered administrator. Please visit /register."
		}
		data := map[string]string{
			"title": Title,
			"msg":   msg,
		}

		// Uses templates/index.tmpl
		r.HTML(http.StatusOK, "index", data)
	})

	// Admin status
	m.Any("/status", func(r render.Render) {
		s := fmt.Sprintf("has administrator: %s, logged in: %s",
			b2yn(userstate.HasUser(AdminUsername)),
			b2yn(userstate.IsLoggedIn(AdminUsername)),
		)

		data := map[string]string{
			"title": Title,
			"msg":   s,
		}

		// Uses templates/index.tmpl
		r.HTML(http.StatusOK, "index", data)
	})

	// The admin panel
	m.Get("/admin", func(w http.ResponseWriter, req *http.Request, r render.Render) {
		// TODO: Write an admin panel for managing users

		data := map[string]string{
			"title": Title,
			"msg":   "Admin panel, work in progress",
		}

		// Uses templates/index.tmpl
		r.HTML(http.StatusOK, "index", data)
	})

	// Enable temporarily for removing and re-creating the admin user with a new pasword
	//m.Get("/remove", func() string {
	//	userstate.RemoveUser(AdminUsername)
	//	return "removed admin user"
	//})

	// --- Admin user management ---

	// Register the admin password
	m.Get("/register", func(w http.ResponseWriter, req *http.Request) {
		if userstate.HasUser(AdminUsername) {
			fmt.Fprint(w, "Error: Already has a registered administrator.")
			return
		}
		w.Header().Add("Content-Type", "text/html")
		fmt.Fprint(w, "<!doctype html>")
		fmt.Fprint(w, instapage.RegisterForm())
		fmt.Fprint(w, "")
	})
	m.Post("/register", binding.Bind(RegisterAdmin{}), func(ra RegisterAdmin) string {
		username := AdminUsername
		if !userstate.HasUser(username) {
			userstate.AddUser(username, ra.Password1, ra.Email)
			userstate.SetAdminStatus(username)
		}
		return "Success: Registered administrator: " + username + "."
	})

	// Login admin
	m.Get("/login", func(w http.ResponseWriter, req *http.Request) {
		if userstate.AdminRights(req) || userstate.UserRights(req) {
			fmt.Fprint(w, "Error: Already logged in as a user or as an administrator.")
			return
		}
		w.Header().Add("Content-Type", "text/html")
		fmt.Fprint(w, "<!doctype html>")
		fmt.Fprint(w, instapage.LoginForm())
		fmt.Fprint(w, "")
	})
	m.Post("/login", binding.Bind(LoginAdmin{}), func(la LoginAdmin) string {
		username := AdminUsername
		if !userstate.HasUser(username) {
			return "Error: User " + username + " does not exist."
		}
		if !userstate.CorrectPassword(username, la.Password) {
			return "Error: Incorrect password."
		}
		userstate.SetLoggedIn(username)
		return "Success: Logged in " + username + "."
	})

	// Logout admin
	m.Any("/logout", func(req *http.Request) string {
		username := AdminUsername
		if !userstate.AdminRights(req) {
			return "Error: Need administrator rights to log out the administrator user."
		}
		userstate.Logout(username)
		if userstate.IsLoggedIn(username) {
			// logout failed
			return "Error: Could not log out."
		}
		return "Success: Logged out " + username + "."
	})

	// --- REST methods ---

	// For testing the API
	m.Any(API, func(r render.Render) {
		r.JSON(http.StatusOK, map[string]interface{}{"hello": "fjaselus"})
	})

	// For adding users

	m.Post(API+"create/:username", func(params martini.Params, r render.Render) {
		username := params["username"]
		if userstate.HasUser(username) {
			r.JSON(http.StatusConflict, map[string]interface{}{"error": "user " + username + " already exists"})
			return
		}
		userstate.AddUser(username, "", "")
		if userstate.HasUser(username) {
			r.JSON(http.StatusOK, map[string]interface{}{"create": true})
		} else {
			r.JSON(http.StatusInternalServerError, map[string]interface{}{"error": "user " + username + " was not created"})
		}
	})

	m.Post(API+"register/:username/:password/:email", func(params martini.Params, r render.Render) {
		username := params["username"]
		password := params["password"]
		email := params["email"]
		if userstate.HasUser(username) {
			r.JSON(http.StatusConflict, map[string]interface{}{"error": "user " + username + " already exists"})
			return
		}
		userstate.AddUser(username, password, email)
		if userstate.HasUser(username) {
			r.JSON(http.StatusOK, map[string]interface{}{"create": true})
		} else {
			r.JSON(http.StatusInternalServerError, map[string]interface{}{"error": "user " + username + " was not created"})
		}
	})

	// For logging in
	m.Post(API+"login/:username/:password", func(w http.ResponseWriter, params martini.Params, r render.Render) {
		username := params["username"]
		password := params["password"]
		if userstate.CorrectPassword(username, password) {
			userstate.SetLoggedIn(username)

		}
		if !userstate.IsLoggedIn(username) {
			r.JSON(http.StatusUnauthorized, map[string]interface{}{"error": "could not log in " + username})
			return
		}
		r.JSON(http.StatusOK, map[string]interface{}{"login": true})
	})

	// For logging out
	m.Any(API+"logout/:username", func(params martini.Params, r render.Render) {
		username := params["username"]
		userstate.Logout(username)
		if userstate.IsLoggedIn(username) {
			r.JSON(http.StatusInternalServerError, map[string]interface{}{"error": "user " + username + " is still logged in!"})
			return
		}
		r.JSON(http.StatusOK, map[string]interface{}{"logout": true})
	})

	// For login status
	m.Any(API+"status/:username", func(params martini.Params, r render.Render) {
		username := params["username"]
		if userstate.IsLoggedIn(username) {
			r.JSON(http.StatusOK, map[string]interface{}{"login": true})
		} else {
			r.JSON(http.StatusOK, map[string]interface{}{"login": "false"})
		}
	})

	// Score POST og GET + timestamp
	m.Post(API+"score/:username/:score", func(params martini.Params, r render.Render) {
		username := params["username"]
		score := params["score"]

		if !userstate.HasUser(username) {
			r.JSON(http.StatusNotFound, map[string]interface{}{"error": "no such user " + username})
			return
		}

		users := userstate.GetUsers()
		users.Set(username, "score", score)

		r.JSON(http.StatusOK, map[string]interface{}{"score set": true})
	})
	m.Get(API+"score/:username", func(params martini.Params, r render.Render) {
		username := params["username"]

		if !userstate.HasUser(username) {
			r.JSON(http.StatusNotFound, map[string]interface{}{"error": "no such user " + username})
			return
		}

		users := userstate.GetUsers()
		score, err := users.Get(username, "score")
		if err != nil {
			r.JSON(http.StatusInternalServerError, map[string]interface{}{"error": "could not get score for " + username})
			return
		}

		r.JSON(http.StatusOK, map[string]interface{}{"score": score})
	})

	// Share the files in static
	m.Use(martini.Static("static"))

	// Only enable HTTP Basic Auth for paths that starts with "/api"
	m.Use(auth.BasicFunc(func(username, password string) bool {
		// Check if the admin user has the correct password, as registered for the admin user
		return auth.SecureCompare(AdminUsername, username) && userstate.CorrectPassword(AdminUsername, password)
	}, "/api"))

	// port 3000 by default, uses PORT and HOST environment variables
	m.Run()
}
<commit_msg>Improved admin functionality<commit_after>package main

import (
	"fmt"
	"net/http"

	"github.com/go-martini/martini"
	"github.com/martini-contrib/binding"
	"github.com/martini-contrib/render"
	"github.com/xyproto/auth"
	"github.com/xyproto/fizz"
	"github.com/xyproto/instapage"
)

const (
	Version       = "1.0"
	API           = "/api/" + Version + "/"
	Title         = "Score Server " + Version
	AdminUsername = "admin"
)

type RegisterAdmin struct {
	Password1 string `form:"password1" binding:"required"`
	Password2 string `form:"password2" binding:"required"`
	Email     string `form:"email" binding:"required"`
}

type LoginAdmin struct {
	Password string `form:"password" binding:"required"`
}

// Helper function for converting a bool to "yes" or "no"
func b2yn(yesno bool) string {
	if yesno {
		return "yes"
	}
	return "no"
}

func main() {
	fmt.Println(Title)

	// New Martini
	m := martini.Classic()

	// New Renderer
	r := render.Renderer(render.Options{})
	m.Use(r)

	// Initiate the user system
	fizz := fizz.NewWithRedisConf(7, "")
	userstate := fizz.UserState()

	// --- Public pages and admin panel ---

	// Public page
	m.Get("/", func(r render.Render) {
		msg := "Everything is fine."
		if !userstate.HasUser(AdminUsername) {
			msg = "No registered administrator. Please visit /register."
		}
		data := map[string]string{
			"title": Title,
			"msg":   msg,
		}

		// Uses templates/index.tmpl
		r.HTML(http.StatusOK, "index", data)
	})

	// Admin status
	m.Any("/status", func(r render.Render) {
		s := fmt.Sprintf("has administrator: %s, logged in: %s",
			b2yn(userstate.HasUser(AdminUsername)),
			b2yn(userstate.IsLoggedIn(AdminUsername)),
		)

		data := map[string]string{
			"title": Title,
			"msg":   s,
		}

		// Uses templates/index.tmpl
		r.HTML(http.StatusOK, "index", data)
	})

	// The admin panel
	m.Get("/admin", func(w http.ResponseWriter, req *http.Request, r render.Render) {
		// TODO: Write an admin panel for managing users

		data := map[string]string{
			"title": Title,
			"msg":   "Admin panel, work in progress",
		}

		// Uses templates/index.tmpl
		r.HTML(http.StatusOK, "index", data)
	})

	// Enable temporarily for removing and re-creating the admin user with a new pasword
	//m.Get("/remove", func() string {
	//	userstate.RemoveUser(AdminUsername)
	//	return "removed admin user"
	//})

	// --- Admin user management ---

	// Register the admin password
	m.Get("/register", func(w http.ResponseWriter, req *http.Request) {
		if userstate.HasUser(AdminUsername) {
			fmt.Fprint(w, "Error: Already has a registered administrator.")
			return
		}
		w.Header().Add("Content-Type", "text/html")
		fmt.Fprint(w, "<!doctype html>")
		fmt.Fprint(w, instapage.RegisterForm())
		fmt.Fprint(w, "")
	})
	m.Post("/register", binding.Bind(RegisterAdmin{}), func(ra RegisterAdmin) string {
		username := AdminUsername
		if !userstate.HasUser(username) {
			userstate.AddUser(username, ra.Password1, ra.Email)
			userstate.SetAdminStatus(username)
		}
		return "Success: Registered administrator: " + username + "."
	})

	// Login admin
	m.Get("/login", func(w http.ResponseWriter, req *http.Request) {
		if userstate.AdminRights(req) || userstate.UserRights(req) {
			fmt.Fprint(w, "Error: Already logged in as a user or as an administrator.")
			return
		}
		w.Header().Add("Content-Type", "text/html")
		fmt.Fprint(w, "<!doctype html>")
		fmt.Fprint(w, instapage.LoginForm())
		fmt.Fprint(w, "")
	})
	m.Post("/login", binding.Bind(LoginAdmin{}), func(la LoginAdmin, w http.ResponseWriter, req *http.Request) string {
		username := AdminUsername
		if !userstate.HasUser(username) {
			return "Error: User " + username + " does not exist."
		}
		if !userstate.CorrectPassword(username, la.Password) {
			return "Error: Incorrect password."
		}
		userstate.Login(w, username)
		if !userstate.AdminRights(req) {
			return "Error: User " + username + " was logged in, but does not have admin rights. Cookie problem?"
		}
		return "Success: Logged in " + username + "."
	})

	// Logout admin
	m.Any("/logout", func(req *http.Request) string {
		username := AdminUsername
		if !userstate.AdminRights(req) {
			return "Error: Need administrator rights to log out the administrator user."
		}
		userstate.Logout(username)
		if userstate.IsLoggedIn(username) {
			// logout failed
			return "Error: Could not log out " + username + "."
		}
		return "Success: Logged out " + username + "."
	})

	// --- REST methods ---

	// For testing the API
	m.Any(API, func(r render.Render) {
		r.JSON(http.StatusOK, map[string]interface{}{"hello": "fjaselus"})
	})

	// For adding users

	m.Post(API+"create/:username", func(params martini.Params, r render.Render) {
		username := params["username"]
		if userstate.HasUser(username) {
			r.JSON(http.StatusConflict, map[string]interface{}{"error": "user " + username + " already exists"})
			return
		}
		userstate.AddUser(username, "", "")
		if userstate.HasUser(username) {
			r.JSON(http.StatusOK, map[string]interface{}{"create": true})
		} else {
			r.JSON(http.StatusInternalServerError, map[string]interface{}{"error": "user " + username + " was not created"})
		}
	})

	m.Post(API+"register/:username/:password/:email", func(params martini.Params, r render.Render) {
		username := params["username"]
		password := params["password"]
		email := params["email"]
		if userstate.HasUser(username) {
			r.JSON(http.StatusConflict, map[string]interface{}{"error": "user " + username + " already exists"})
			return
		}
		userstate.AddUser(username, password, email)
		if userstate.HasUser(username) {
			r.JSON(http.StatusOK, map[string]interface{}{"create": true})
		} else {
			r.JSON(http.StatusInternalServerError, map[string]interface{}{"error": "user " + username + " was not created"})
		}
	})

	// For logging in
	m.Post(API+"login/:username/:password", func(w http.ResponseWriter, params martini.Params, r render.Render) {
		username := params["username"]
		password := params["password"]
		if userstate.CorrectPassword(username, password) {
			userstate.SetLoggedIn(username)

		}
		if !userstate.IsLoggedIn(username) {
			r.JSON(http.StatusUnauthorized, map[string]interface{}{"error": "could not log in " + username})
			return
		}
		r.JSON(http.StatusOK, map[string]interface{}{"login": true})
	})

	// For logging out
	m.Any(API+"logout/:username", func(params martini.Params, r render.Render) {
		username := params["username"]
		userstate.Logout(username)
		if userstate.IsLoggedIn(username) {
			r.JSON(http.StatusInternalServerError, map[string]interface{}{"error": "user " + username + " is still logged in!"})
			return
		}
		r.JSON(http.StatusOK, map[string]interface{}{"logout": true})
	})

	// For login status
	m.Any(API+"status/:username", func(params martini.Params, r render.Render) {
		username := params["username"]
		if userstate.IsLoggedIn(username) {
			r.JSON(http.StatusOK, map[string]interface{}{"login": true})
		} else {
			r.JSON(http.StatusOK, map[string]interface{}{"login": "false"})
		}
	})

	// Score POST og GET + timestamp
	m.Post(API+"score/:username/:score", func(params martini.Params, r render.Render) {
		username := params["username"]
		score := params["score"]

		if !userstate.HasUser(username) {
			r.JSON(http.StatusNotFound, map[string]interface{}{"error": "no such user " + username})
			return
		}

		users := userstate.GetUsers()
		users.Set(username, "score", score)

		r.JSON(http.StatusOK, map[string]interface{}{"score set": true})
	})
	m.Get(API+"score/:username", func(params martini.Params, r render.Render) {
		username := params["username"]

		if !userstate.HasUser(username) {
			r.JSON(http.StatusNotFound, map[string]interface{}{"error": "no such user " + username})
			return
		}

		users := userstate.GetUsers()
		score, err := users.Get(username, "score")
		if err != nil {
			r.JSON(http.StatusInternalServerError, map[string]interface{}{"error": "could not get score for " + username})
			return
		}

		r.JSON(http.StatusOK, map[string]interface{}{"score": score})
	})

	// Share the files in static
	m.Use(martini.Static("static"))

	// Only enable HTTP Basic Auth for paths that starts with "/api"
	m.Use(auth.BasicFunc(func(username, password string) bool {
		// Check if the admin user has the correct password, as registered for the admin user
		return auth.SecureCompare(AdminUsername, username) && userstate.CorrectPassword(AdminUsername, password)
	}, "/api"))

	// port 3000 by default, uses PORT and HOST environment variables
	m.Run()
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/tls"
	"encoding/base64"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"github.com/mpdroog/beanstalkd" //"github.com/maxid/beanstalkd"
	"gopkg.in/gomail.v1"
	"log"
	"os"
	"github.com/mpdroog/smtpw/config"
	"strings"
	"time"
	"github.com/coreos/go-systemd/daemon"
)

const ERR_WAIT_SEC = 5

var errTimedOut = errors.New("timed out")

var verbose bool
var readonly bool
var debug bool
var hostname string
var L *log.Logger

func proc(m config.Email, skipOne bool) error {
	conf, ok := config.C.From[m.From]
	if !ok {
		return errors.New("From does not exist: " + m.From)
	}

	host := conf.Hostname
	if host == "" {
		host = hostname
	}

	msg := gomail.NewMessage()
	msg.SetHeader("Message-ID", fmt.Sprintf("<%s@%s>", RandText(32), host))
	msg.SetHeader("X-Mailer", "smtpw")
	msg.SetHeader("X-Priority", "3")
	if conf.Bounce == nil {
		msg.SetHeader("From", conf.Display+" <"+conf.From+">")
	} else {
		// Set bounce handling
		// From receives bounces
		// Human-clients send to Reply-To
		msg.SetHeader("From", fmt.Sprintf("%s <%s>", conf.Display, *conf.Bounce))
		msg.SetHeader("Reply-To", fmt.Sprintf("%s <%s>", conf.Display, conf.From))
	}
	msg.SetHeader("To", m.To...)
	msg.SetHeader("Bcc", conf.Bcc...)
	msg.SetHeader("Subject", m.Subject)
	msg.SetBody("text/plain", m.Text)
	if len(m.Html) > 0 {
		msg.AddAlternative("text/html", m.Html)
	}

	for name, embed := range m.HtmlEmbed {
		raw, e := base64.StdEncoding.DecodeString(embed)
		if e != nil {
			if skipOne {
				L.Printf("WARN: HtmlEmbed: " + name + " is not base64!\n")
				return nil
			}
			return errors.New("HtmlEmbed: " + name + " is not base64!")
		}
		if !strings.Contains(m.Html, fmt.Sprintf("cid:"+name)) {
			if skipOne {
				L.Printf("WARN: HtmlEmbed: " + name + " is not used in the HTML!")
				return nil
			}
			return errors.New("HtmlEmbed: " + name + " is not used in the HTML!")
		}
		msg.Embed(gomail.CreateFile(name, raw))
	}
	for name, attachment := range m.Attachments {
		raw, e := base64.StdEncoding.DecodeString(attachment)
		if e != nil {
			if skipOne {
				L.Printf("WARN: Attachment: " + name + " is not base64!")
				return nil
			}
			return errors.New("Attachment: " + name + " is not base64!")
		}
		msg.Attach(gomail.CreateFile(name, raw))
	}

	if readonly {
		L.Printf("From: %s <%s>\n", conf.Display, conf.From)
		L.Printf("To: %v\n", m.To)
		L.Printf("Bcc: %v\n", conf.Bcc)
		L.Printf("Subject: %s\n", m.Subject)
		L.Printf("\ntext/plain\n")
		L.Printf(m.Text)
		L.Printf("\ntext/html\n")
		L.Printf(m.Html)
		L.Printf("\n\n")
		return nil
	}

	cfg := gomail.SetTLSConfig(&tls.Config{ServerName: conf.Host})
	if conf.Insecure {
		cfg = gomail.SetTLSConfig(&tls.Config{InsecureSkipVerify : true})
	}
	auth := LoginAuth(conf.User, conf.Pass)
	mailer := gomail.NewCustomMailer(fmt.Sprintf("%s:%d", conf.Host, conf.Port), auth, cfg)
	return mailer.Send(msg)
}

func connect() (*beanstalkd.BeanstalkdClient, error) {
	queue, e := beanstalkd.Dial(config.C.Beanstalk)
	if e != nil {
		return nil, e
	}
	// Only listen to email queue.
	queue.Use("email")
	if _, e := queue.Watch("email"); e != nil {
		return nil, e
	}
	queue.Ignore("default")
	return queue, nil
}

func main() {
	var (
		configPath string
		skipOne    bool
	)
	L = log.New(os.Stdout, "", log.LstdFlags)

	flag.BoolVar(&verbose, "v", false, "Verbose-mode")
        flag.BoolVar(&debug, "d", false, "Debug-mode")
	flag.BoolVar(&skipOne, "s", false, "Delete e-mail on deverr")
	flag.BoolVar(&readonly, "r", false, "Don't email but flush to stdout")
	flag.StringVar(&configPath, "c", "./config.json", "Path to config.json")
	flag.Parse()

	if e := config.Init(configPath); e != nil {
		panic(e)
	}
	if verbose {
		L.Printf("%+v\n", config.C)
	}
	// TODO: Test config before starting?

	queue, e := connect()
	if e != nil {
		panic(e)
	}
	hostname, e = os.Hostname()
	if e != nil {
		panic(e)
	}

	if verbose {
		L.Printf("SMTPw(%s) email-tube (ignoring default)\n", hostname)
	}
	if readonly {
		L.Printf("!! ReadOnly mode !!\n")
	}

	sent, e := daemon.SdNotify(false, "READY=1")
	if e != nil {
		panic(e)
	}
	if !sent {
		L.Printf("SystemD notify NOT sent\n")
        }

	for {
		job, e := queue.Reserve(15 * 60) //15min timeout
		if e != nil {
			if e.Error() == errTimedOut.Error() {
				// Reserve timeout
				continue
			}

			L.Printf("Beanstalkd err: %s\n", e.Error())
			time.Sleep(time.Second * ERR_WAIT_SEC)
			if strings.HasSuffix(e.Error(), "broken pipe") {
				// Beanstalkd down, reconnect!
				q, e := connect()
				if e != nil {
					L.Printf("Reconnect err: %s\n", e.Error())
				}
				if q != nil {
					queue = q
				}
			}
			continue
		}
		if debug {
			L.Printf("JSON:\n%s\n", string(job.Data))
		}
		// Parse
		var m config.Email
		if e := json.Unmarshal(job.Data, &m); e != nil {
			// Broken JSON
			if skipOne {
				L.Printf("WARN: Skip job as JSON is invalid\n")
				queue.Delete(job.Id)
				skipOne = false
				continue
			}

			// Ignore decode trouble
			L.Printf("CRIT: Invalid JSON received (msg=%s)\n", e.Error())
			continue
		}

                if verbose {
                        L.Printf("Email (job=%d email=%s subject=%s)\n", job.Id, m.To[0], m.Subject)
                }

		if e := proc(m, skipOne); e != nil {
			// 501 Syntax error in parameters or arguments
			// http://www.greenend.org.uk/rjk/tech/smtpreplies.html
			if strings.HasPrefix(e.Error(), "501 ") {
				L.Printf("WARN: Job buried, invalid email address(es)? (msg=%s)\n", e.Error())
				queue.Bury(job.Id, 1)
				continue
			}
			// TODO: Isolate deverr from senderr
			// Processing trouble?
			L.Printf("WARN: Failed sending, retry in 20sec (msg=%s)\n", e.Error())
			time.Sleep(time.Second * 20)
			continue
		}
		queue.Delete(job.Id)
		if verbose {
			L.Printf("Finished job %d", job.Id)
		}
	}
	queue.Quit()
}
<commit_msg>Bugfix. Loosen strictness to support empty array for attachments-field<commit_after>package main

import (
	"bytes"
	"crypto/tls"
	"encoding/base64"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"github.com/mpdroog/beanstalkd" //"github.com/maxid/beanstalkd"
	"gopkg.in/gomail.v1"
	"log"
	"os"
	"github.com/mpdroog/smtpw/config"
	"strings"
	"time"
	"github.com/coreos/go-systemd/daemon"
)

const ERR_WAIT_SEC = 5

var errTimedOut = errors.New("timed out")

var verbose bool
var readonly bool
var debug bool
var hostname string
var L *log.Logger

func proc(m config.Email, skipOne bool) error {
	conf, ok := config.C.From[m.From]
	if !ok {
		return errors.New("From does not exist: " + m.From)
	}

	host := conf.Hostname
	if host == "" {
		host = hostname
	}

	msg := gomail.NewMessage()
	msg.SetHeader("Message-ID", fmt.Sprintf("<%s@%s>", RandText(32), host))
	msg.SetHeader("X-Mailer", "smtpw")
	msg.SetHeader("X-Priority", "3")
	if conf.Bounce == nil {
		msg.SetHeader("From", conf.Display+" <"+conf.From+">")
	} else {
		// Set bounce handling
		// From receives bounces
		// Human-clients send to Reply-To
		msg.SetHeader("From", fmt.Sprintf("%s <%s>", conf.Display, *conf.Bounce))
		msg.SetHeader("Reply-To", fmt.Sprintf("%s <%s>", conf.Display, conf.From))
	}
	msg.SetHeader("To", m.To...)
	msg.SetHeader("Bcc", conf.Bcc...)
	msg.SetHeader("Subject", m.Subject)
	msg.SetBody("text/plain", m.Text)
	if len(m.Html) > 0 {
		msg.AddAlternative("text/html", m.Html)
	}

	for name, embed := range m.HtmlEmbed {
		raw, e := base64.StdEncoding.DecodeString(embed)
		if e != nil {
			if skipOne {
				L.Printf("WARN: HtmlEmbed: " + name + " is not base64!\n")
				return nil
			}
			return errors.New("HtmlEmbed: " + name + " is not base64!")
		}
		if !strings.Contains(m.Html, fmt.Sprintf("cid:"+name)) {
			if skipOne {
				L.Printf("WARN: HtmlEmbed: " + name + " is not used in the HTML!")
				return nil
			}
			return errors.New("HtmlEmbed: " + name + " is not used in the HTML!")
		}
		msg.Embed(gomail.CreateFile(name, raw))
	}
	for name, attachment := range m.Attachments {
		raw, e := base64.StdEncoding.DecodeString(attachment)
		if e != nil {
			if skipOne {
				L.Printf("WARN: Attachment: " + name + " is not base64!")
				return nil
			}
			return errors.New("Attachment: " + name + " is not base64!")
		}
		msg.Attach(gomail.CreateFile(name, raw))
	}

	if readonly {
		L.Printf("From: %s <%s>\n", conf.Display, conf.From)
		L.Printf("To: %v\n", m.To)
		L.Printf("Bcc: %v\n", conf.Bcc)
		L.Printf("Subject: %s\n", m.Subject)
		L.Printf("\ntext/plain\n")
		L.Printf(m.Text)
		L.Printf("\ntext/html\n")
		L.Printf(m.Html)
		L.Printf("\n\n")
		return nil
	}

	cfg := gomail.SetTLSConfig(&tls.Config{ServerName: conf.Host})
	if conf.Insecure {
		cfg = gomail.SetTLSConfig(&tls.Config{InsecureSkipVerify : true})
	}
	auth := LoginAuth(conf.User, conf.Pass)
	mailer := gomail.NewCustomMailer(fmt.Sprintf("%s:%d", conf.Host, conf.Port), auth, cfg)
	return mailer.Send(msg)
}

func connect() (*beanstalkd.BeanstalkdClient, error) {
	queue, e := beanstalkd.Dial(config.C.Beanstalk)
	if e != nil {
		return nil, e
	}
	// Only listen to email queue.
	queue.Use("email")
	if _, e := queue.Watch("email"); e != nil {
		return nil, e
	}
	queue.Ignore("default")
	return queue, nil
}

func main() {
	var (
		configPath string
		skipOne    bool
	)
	L = log.New(os.Stdout, "", log.LstdFlags)

	flag.BoolVar(&verbose, "v", false, "Verbose-mode")
        flag.BoolVar(&debug, "d", false, "Debug-mode")
	flag.BoolVar(&skipOne, "s", false, "Delete e-mail on deverr")
	flag.BoolVar(&readonly, "r", false, "Don't email but flush to stdout")
	flag.StringVar(&configPath, "c", "./config.json", "Path to config.json")
	flag.Parse()

	if e := config.Init(configPath); e != nil {
		panic(e)
	}
	if verbose {
		L.Printf("%+v\n", config.C)
	}
	// TODO: Test config before starting?

	queue, e := connect()
	if e != nil {
		panic(e)
	}
	hostname, e = os.Hostname()
	if e != nil {
		panic(e)
	}

	if verbose {
		L.Printf("SMTPw(%s) email-tube (ignoring default)\n", hostname)
	}
	if readonly {
		L.Printf("!! ReadOnly mode !!\n")
	}

	sent, e := daemon.SdNotify(false, "READY=1")
	if e != nil {
		panic(e)
	}
	if !sent {
		L.Printf("SystemD notify NOT sent\n")
        }

	for {
		job, e := queue.Reserve(15 * 60) //15min timeout
		if e != nil {
			if e.Error() == errTimedOut.Error() {
				// Reserve timeout
				continue
			}

			L.Printf("Beanstalkd err: %s\n", e.Error())
			time.Sleep(time.Second * ERR_WAIT_SEC)
			if strings.HasSuffix(e.Error(), "broken pipe") {
				// Beanstalkd down, reconnect!
				q, e := connect()
				if e != nil {
					L.Printf("Reconnect err: %s\n", e.Error())
				}
				if q != nil {
					queue = q
				}
			}
			continue
		}

		// Hackfix. Get rid of array when empty to prevent parsing issues here
		if bytes.Contains(job.Data, []byte(`,"attachments":[]`)) {
			job.Data = bytes.Replace(job.Data, []byte(`,"attachments":[]`), []byte(``), 1)
		}

		if debug {
			L.Printf("JSON:\n%s\n", string(job.Data))
		}
		// Parse
		var m config.Email
		if e := json.Unmarshal(job.Data, &m); e != nil {
			// Broken JSON
			if skipOne {
				L.Printf("WARN: Skip job as JSON is invalid\n")
				queue.Delete(job.Id)
				skipOne = false
				continue
			}

			// Ignore decode trouble
			L.Printf("CRIT: Invalid JSON received (msg=%s)\n", e.Error())
			continue
		}

                if verbose {
                        L.Printf("Email (job=%d email=%s subject=%s)\n", job.Id, m.To[0], m.Subject)
                }

		if e := proc(m, skipOne); e != nil {
			// 501 Syntax error in parameters or arguments
			// http://www.greenend.org.uk/rjk/tech/smtpreplies.html
			if strings.HasPrefix(e.Error(), "501 ") {
				L.Printf("WARN: Job buried, invalid email address(es)? (msg=%s)\n", e.Error())
				queue.Bury(job.Id, 1)
				continue
			}
			// TODO: Isolate deverr from senderr
			// Processing trouble?
			L.Printf("WARN: Failed sending, retry in 20sec (msg=%s)\n", e.Error())
			time.Sleep(time.Second * 20)
			continue
		}
		queue.Delete(job.Id)
		if verbose {
			L.Printf("Finished job %d", job.Id)
		}
	}
	queue.Quit()
}
<|endoftext|>


<commit_before>// Package main generates web project.
package main

import (
	"flag"
	"github.com/go-bootstrap/go-bootstrap/helpers"
	"log"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
	"strings"
)

func main() {
	dir := flag.String("dir", "", "Project directory relative to $GOPATH/src/")
	flag.Parse()

	if *dir == "" {
		log.Fatal("dir option is missing.")
	}

	// There can be more than one path, separated by colon.
	gopaths := strings.Split(os.ExpandEnv("$GOPATH"), ":")
	gopath := gopaths[0]

	trimmedPath := strings.Trim(*dir, "/")
	fullpath := filepath.Join(gopath, "src", trimmedPath)
	dirChunks := strings.Split(trimmedPath, "/")
	repoName := dirChunks[len(dirChunks)-3]
	repoUser := dirChunks[len(dirChunks)-2]
	projectName := dirChunks[len(dirChunks)-1]
	dbName := projectName
	testDbName := projectName + "-test"
	currentUser, _ := user.Current()

	// 1. Create target directory
	log.Print("Creating " + fullpath + "...")
	err := os.MkdirAll(fullpath, 0755)
	helpers.ExitOnError(err, "")

	// 2. Copy everything under blank directory to target directory.
	log.Print("Copying a blank project to " + fullpath + "...")
	blankDir := os.ExpandEnv(filepath.Join(gopath, "src", "github.com", "go-bootstrap", "go-bootstrap", "blank"))
	currDir, err := os.Getwd()
	helpers.ExitOnError(err, "Can't get current path!")

	err = os.Chdir(blankDir)
	helpers.ExitOnError(err, "")

	output, err := exec.Command("cp", "-rf", ".", fullpath).CombinedOutput()
	helpers.ExitOnError(err, string(output))

	err = os.Chdir(currDir)
	helpers.ExitOnError(err, "")

	// 3. Interpolate placeholder variables on the new project.
	log.Print("Replacing placeholder variables on " + repoUser + "/" + projectName + "...")

	replacers := make(map[string]string)
	replacers["$GO_BOOTSTRAP_REPO_NAME"] = repoName
	replacers["$GO_BOOTSTRAP_REPO_USER"] = repoUser
	replacers["$GO_BOOTSTRAP_PROJECT_NAME"] = projectName
	replacers["$GO_BOOTSTRAP_COOKIE_SECRET"] = helpers.RandString(16)
	replacers["$GO_BOOTSTRAP_CURRENT_USER"] = currentUser.Username
	replacers["$GO_BOOTSTRAP_PG_DSN"] = helpers.DefaultPGDSN(dbName)
	replacers["$GO_BOOTSTRAP_ESCAPED_PG_DSN"] = helpers.BashEscape(helpers.DefaultPGDSN(dbName))
	replacers["$GO_BOOTSTRAP_PG_TEST_DSN"] = helpers.DefaultPGDSN(testDbName)
	replacers["$GO_BOOTSTRAP_ESCAPED_PG_TEST_DSN"] = helpers.BashEscape(helpers.DefaultPGDSN(testDbName))

	err = helpers.RecursiveSearchReplaceFiles(fullpath, replacers)
	helpers.ExitOnError(err, "")

	// 4. go get github.com/mattes/migrate.
	log.Print("Installing github.com/mattes/migrate...")
	output, err = exec.Command("go", "get", "github.com/mattes/migrate").CombinedOutput()
	helpers.ExitOnError(err, string(output))

	// 5. Bootstrap databases.
	cmd := exec.Command("bash", "scripts/db-bootstrap")
	cmd.Dir = fullpath
	output, _ = cmd.CombinedOutput()
	log.Print(string(output))

	// 6. Get all application dependencies for the first time.
	log.Print("Running go get ./...")
	cmd = exec.Command("go", "get", "./...")
	cmd.Dir = fullpath
	output, err = cmd.CombinedOutput()
	helpers.ExitOnError(err, string(output))

	repoIsGit := strings.HasPrefix(repoName, "git")

	if repoIsGit {
		// Generate Godeps directory. Currently only works on git related repo.
		log.Print("Installing github.com/tools/godep...")
		output, err := exec.Command("go", "get", "github.com/tools/godep").CombinedOutput()
		helpers.ExitOnError(err, string(output))

		// git init.
		log.Print("Running git init")
		cmd := exec.Command("git", "init")
		cmd.Dir = fullpath
		output, err = cmd.CombinedOutput()
		helpers.ExitOnError(err, string(output))

		// godep save ./...
		log.Print("Running godep save ./...")
		cmd = exec.Command("godep", "save", "./...")
		cmd.Dir = fullpath
		output, err = cmd.CombinedOutput()
		helpers.ExitOnError(err, string(output))

		// Run tests on newly generated app.
		log.Print("Running godep go test ./...")
		cmd = exec.Command("godep", "go", "test", "./...")
		cmd.Dir = fullpath
		output, _ = cmd.CombinedOutput()
		log.Print(string(output))

	} else {
		// Run tests on newly generated app.
		log.Print("Running go test ./...")
		cmd = exec.Command("go", "test", "./...")
		cmd.Dir = fullpath
		output, _ = cmd.CombinedOutput()
		log.Print(string(output))
	}
}
<commit_msg>Godep directory is now automatically generated for bitbucket.org repos.<commit_after>// Package main generates web project.
package main

import (
	"flag"
	"github.com/go-bootstrap/go-bootstrap/helpers"
	"log"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
	"strings"
)

func main() {
	dir := flag.String("dir", "", "Project directory relative to $GOPATH/src/")
	flag.Parse()

	if *dir == "" {
		log.Fatal("dir option is missing.")
	}

	// There can be more than one path, separated by colon.
	gopaths := strings.Split(os.ExpandEnv("$GOPATH"), ":")
	gopath := gopaths[0]

	trimmedPath := strings.Trim(*dir, "/")
	fullpath := filepath.Join(gopath, "src", trimmedPath)
	dirChunks := strings.Split(trimmedPath, "/")
	repoName := dirChunks[len(dirChunks)-3]
	repoUser := dirChunks[len(dirChunks)-2]
	projectName := dirChunks[len(dirChunks)-1]
	dbName := projectName
	testDbName := projectName + "-test"
	currentUser, _ := user.Current()

	// 1. Create target directory
	log.Print("Creating " + fullpath + "...")
	err := os.MkdirAll(fullpath, 0755)
	helpers.ExitOnError(err, "")

	// 2. Copy everything under blank directory to target directory.
	log.Print("Copying a blank project to " + fullpath + "...")
	blankDir := os.ExpandEnv(filepath.Join(gopath, "src", "github.com", "go-bootstrap", "go-bootstrap", "blank"))
	currDir, err := os.Getwd()
	helpers.ExitOnError(err, "Can't get current path!")

	err = os.Chdir(blankDir)
	helpers.ExitOnError(err, "")

	output, err := exec.Command("cp", "-rf", ".", fullpath).CombinedOutput()
	helpers.ExitOnError(err, string(output))

	err = os.Chdir(currDir)
	helpers.ExitOnError(err, "")

	// 3. Interpolate placeholder variables on the new project.
	log.Print("Replacing placeholder variables on " + repoUser + "/" + projectName + "...")

	replacers := make(map[string]string)
	replacers["$GO_BOOTSTRAP_REPO_NAME"] = repoName
	replacers["$GO_BOOTSTRAP_REPO_USER"] = repoUser
	replacers["$GO_BOOTSTRAP_PROJECT_NAME"] = projectName
	replacers["$GO_BOOTSTRAP_COOKIE_SECRET"] = helpers.RandString(16)
	replacers["$GO_BOOTSTRAP_CURRENT_USER"] = currentUser.Username
	replacers["$GO_BOOTSTRAP_PG_DSN"] = helpers.DefaultPGDSN(dbName)
	replacers["$GO_BOOTSTRAP_ESCAPED_PG_DSN"] = helpers.BashEscape(helpers.DefaultPGDSN(dbName))
	replacers["$GO_BOOTSTRAP_PG_TEST_DSN"] = helpers.DefaultPGDSN(testDbName)
	replacers["$GO_BOOTSTRAP_ESCAPED_PG_TEST_DSN"] = helpers.BashEscape(helpers.DefaultPGDSN(testDbName))

	err = helpers.RecursiveSearchReplaceFiles(fullpath, replacers)
	helpers.ExitOnError(err, "")

	// 4. go get github.com/mattes/migrate.
	log.Print("Running go get github.com/mattes/migrate...")
	output, err = exec.Command("go", "get", "github.com/mattes/migrate").CombinedOutput()
	helpers.ExitOnError(err, string(output))

	// 5. Bootstrap databases.
	cmd := exec.Command("bash", "scripts/db-bootstrap")
	cmd.Dir = fullpath
	output, _ = cmd.CombinedOutput()
	log.Print(string(output))

	// 6. Get all application dependencies for the first time.
	log.Print("Running go get ./...")
	cmd = exec.Command("go", "get", "./...")
	cmd.Dir = fullpath
	output, err = cmd.CombinedOutput()
	helpers.ExitOnError(err, string(output))

	repoIsGit := strings.HasPrefix(repoName, "git")
	repoIsHg := strings.HasPrefix(repoName, "bitbucket")

	// Generate Godeps directory.
	// Works only on git repo or bitbucket repo.
	if repoIsGit || repoIsHg {
		log.Print("Installing github.com/tools/godep...")
		output, err := exec.Command("go", "get", "github.com/tools/godep").CombinedOutput()
		helpers.ExitOnError(err, string(output))

		if repoIsGit {
			log.Print("Running git init...")
			cmd := exec.Command("git", "init")
			cmd.Dir = fullpath
			output, err = cmd.CombinedOutput()
			helpers.ExitOnError(err, string(output))
		}
		if repoIsHg {
			log.Print("Running hg init...")
			cmd := exec.Command("hg", "init")
			cmd.Dir = fullpath
			output, _ = cmd.CombinedOutput()
			log.Print(string(output))
		}

		// godep save ./...
		log.Print("Running godep save ./...")
		cmd = exec.Command("godep", "save", "./...")
		cmd.Dir = fullpath
		output, err = cmd.CombinedOutput()
		helpers.ExitOnError(err, string(output))

		// Run tests on newly generated app.
		log.Print("Running godep go test ./...")
		cmd = exec.Command("godep", "go", "test", "./...")
		cmd.Dir = fullpath
		output, _ = cmd.CombinedOutput()
		log.Print(string(output))

	} else {
		// Run tests on newly generated app.
		log.Print("Running go test ./...")
		cmd = exec.Command("go", "test", "./...")
		cmd.Dir = fullpath
		output, _ = cmd.CombinedOutput()
		log.Print(string(output))
	}
}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"runtime"

	"github.com/Solher/zest/infrastructure"
	"github.com/Solher/zest/middlewares"
	"github.com/Solher/zest/ressources"
	"github.com/Solher/zest/usecases"
	"github.com/codegangsta/negroni"
	"github.com/dimfeld/httptreemux"
	"github.com/rs/cors"

	_ "github.com/clipperhouse/typewriter" // Forced to allow vendoring.
)

var Environment, Port, DatabaseURL string

func init() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	env := os.Getenv("GOENV")
	if env == "development" || env == "production" {
		Environment = env
	} else {
		Environment = "development"
	}

	port := os.Getenv("PORT")
	if port != "" {
		Port = ":" + os.Getenv("PORT")
	} else {
		Port = ":3000"
	}

	DatabaseURL = os.Getenv("DATABASE_URL")
}

func main() {
	mustExit := handleOsArgs()
	if mustExit {
		return
	}

	app := negroni.New()
	router := httptreemux.New()
	router.RedirectBehavior = httptreemux.UseHandler
	render := infrastructure.NewRender()
	store := infrastructure.NewGormStore()
	sessionCache := infrastructure.NewLRUCacheStore(1024)
	roleCache := infrastructure.NewCacheStore()
	aclCache := infrastructure.NewCacheStore()

	initApp(app, router, render, store, sessionCache, roleCache, aclCache)
	defer closeApp(store)

	app.Run(Port)
}

func handleOsArgs() bool {
	if len(os.Args) > 1 {
		switch os.Args[1] {
		case "resetDB":
			resetDatabase()
			return true
		case "migrateDB":
			migrateDatabase()
			return true
		}
	}

	return false
}

func initApp(app *negroni.Negroni, router *httptreemux.TreeMux, render *infrastructure.Render,
	store *infrastructure.GormStore, lruCacheStore *infrastructure.LRUCacheStore, roleCacheStore, aclCacheStore *infrastructure.CacheStore) {

	err := connectDB(store)
	if err != nil {
		panic("Could not connect to database.")
	}

	userRepo := ressources.NewUserRepo(store)
	sessionRepo := ressources.NewSessionRepo(store)
	accountRepo := ressources.NewAccountRepo(store)
	roleMappingRepo := ressources.NewRoleMappingRepo(store)
	roleRepo := ressources.NewRoleRepo(store)
	aclMappingRepo := ressources.NewAclMappingRepo(store)
	aclRepo := ressources.NewAclRepo(store)

	sessionCacheInter := usecases.NewSessionCacheInter(sessionRepo, lruCacheStore)
	permissionCacheInter := usecases.NewPermissionCacheInter(accountRepo, aclRepo, roleCacheStore, aclCacheStore)
	sessionCacheInter.Refresh()
	permissionCacheInter.Refresh()

	userInter := ressources.NewUserInter(userRepo)
	sessionInter := ressources.NewSessionInter(sessionRepo)
	accountInter := ressources.NewAccountInter(accountRepo, userInter, sessionInter, sessionCacheInter, permissionCacheInter)
	roleMappingInter := ressources.NewRoleMappingInter(roleMappingRepo)
	roleInter := ressources.NewRoleInter(roleRepo)
	aclMappingInter := ressources.NewAclMappingInter(aclMappingRepo)
	aclInter := ressources.NewAclInter(aclRepo)

	routeDir := usecases.NewRouteDirectory(accountInter, render)

	ressources.NewUserCtrl(userInter, render, routeDir)
	ressources.NewSessionCtrl(sessionInter, render, routeDir)
	ressources.NewAccountCtrl(accountInter, render, routeDir)
	ressources.NewRoleMappingCtrl(roleMappingInter, render, routeDir)
	ressources.NewRoleCtrl(roleInter, render, routeDir)
	ressources.NewAclMappingCtrl(aclMappingInter, render, routeDir)
	ressources.NewAclCtrl(aclInter, render, routeDir)

	routeDir.Register(router)
	aclInter.RefreshFromRoutes(routeDir.Routes())

	app.Use(negroni.NewLogger())
	app.Use(negroni.NewRecovery())
	app.Use(middlewares.NewSessions(accountInter))
	app.Use(cors.Default())

	app.UseHandler(router)
}

func closeApp(store *infrastructure.GormStore) {
	store.Close()
}

func connectDB(store *infrastructure.GormStore) error {
	var err error

	if DatabaseURL != "" {
		err = store.Connect("postgres", DatabaseURL)
	} else {
		err = store.Connect("sqlite3", "database.db")
	}

	return err
}
<commit_msg>CORS middleware fixed.<commit_after>package main

import (
	"os"
	"runtime"

	"github.com/Solher/zest/infrastructure"
	"github.com/Solher/zest/middlewares"
	"github.com/Solher/zest/ressources"
	"github.com/Solher/zest/usecases"
	"github.com/codegangsta/negroni"
	"github.com/dimfeld/httptreemux"
	"github.com/rs/cors"

	_ "github.com/clipperhouse/typewriter" // Forced to allow vendoring.
)

var Environment, Port, DatabaseURL string

func init() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	env := os.Getenv("GOENV")
	if env == "development" || env == "production" {
		Environment = env
	} else {
		Environment = "development"
	}

	port := os.Getenv("PORT")
	if port != "" {
		Port = ":" + os.Getenv("PORT")
	} else {
		Port = ":3000"
	}

	DatabaseURL = os.Getenv("DATABASE_URL")
}

func main() {
	mustExit := handleOsArgs()
	if mustExit {
		return
	}

	app := negroni.New()
	router := httptreemux.New()
	router.RedirectBehavior = httptreemux.UseHandler
	render := infrastructure.NewRender()
	store := infrastructure.NewGormStore()
	sessionCache := infrastructure.NewLRUCacheStore(1024)
	roleCache := infrastructure.NewCacheStore()
	aclCache := infrastructure.NewCacheStore()

	initApp(app, router, render, store, sessionCache, roleCache, aclCache)
	defer closeApp(store)

	app.Run(Port)
}

func handleOsArgs() bool {
	if len(os.Args) > 1 {
		switch os.Args[1] {
		case "resetDB":
			resetDatabase()
			return true
		case "migrateDB":
			migrateDatabase()
			return true
		}
	}

	return false
}

func initApp(app *negroni.Negroni, router *httptreemux.TreeMux, render *infrastructure.Render,
	store *infrastructure.GormStore, lruCacheStore *infrastructure.LRUCacheStore, roleCacheStore, aclCacheStore *infrastructure.CacheStore) {

	err := connectDB(store)
	if err != nil {
		panic("Could not connect to database.")
	}

	userRepo := ressources.NewUserRepo(store)
	sessionRepo := ressources.NewSessionRepo(store)
	accountRepo := ressources.NewAccountRepo(store)
	roleMappingRepo := ressources.NewRoleMappingRepo(store)
	roleRepo := ressources.NewRoleRepo(store)
	aclMappingRepo := ressources.NewAclMappingRepo(store)
	aclRepo := ressources.NewAclRepo(store)

	sessionCacheInter := usecases.NewSessionCacheInter(sessionRepo, lruCacheStore)
	permissionCacheInter := usecases.NewPermissionCacheInter(accountRepo, aclRepo, roleCacheStore, aclCacheStore)
	sessionCacheInter.Refresh()
	permissionCacheInter.Refresh()

	userInter := ressources.NewUserInter(userRepo)
	sessionInter := ressources.NewSessionInter(sessionRepo)
	accountInter := ressources.NewAccountInter(accountRepo, userInter, sessionInter, sessionCacheInter, permissionCacheInter)
	roleMappingInter := ressources.NewRoleMappingInter(roleMappingRepo)
	roleInter := ressources.NewRoleInter(roleRepo)
	aclMappingInter := ressources.NewAclMappingInter(aclMappingRepo)
	aclInter := ressources.NewAclInter(aclRepo)

	routeDir := usecases.NewRouteDirectory(accountInter, render)

	ressources.NewUserCtrl(userInter, render, routeDir)
	ressources.NewSessionCtrl(sessionInter, render, routeDir)
	ressources.NewAccountCtrl(accountInter, render, routeDir)
	ressources.NewRoleMappingCtrl(roleMappingInter, render, routeDir)
	ressources.NewRoleCtrl(roleInter, render, routeDir)
	ressources.NewAclMappingCtrl(aclMappingInter, render, routeDir)
	ressources.NewAclCtrl(aclInter, render, routeDir)

	routeDir.Register(router)
	aclInter.RefreshFromRoutes(routeDir.Routes())

	app.Use(negroni.NewLogger())
	app.Use(negroni.NewRecovery())
	app.Use(cors.Default())
	app.Use(middlewares.NewSessions(accountInter))

	app.UseHandler(router)
}

func closeApp(store *infrastructure.GormStore) {
	store.Close()
}

func connectDB(store *infrastructure.GormStore) error {
	var err error

	if DatabaseURL != "" {
		err = store.Connect("postgres", DatabaseURL)
	} else {
		err = store.Connect("sqlite3", "database.db")
	}

	return err
}
<|endoftext|>


<commit_before>// Command gogo is a web server for hosting multiplayer Go games.
package main

import (
	"crypto/tls"
	"flag"
	"log"
	"net/http"

	"github.com/waits/gogo/handler"
	"github.com/waits/gogo/model"
	"golang.org/x/crypto/acme/autocert"
	"golang.org/x/net/websocket"
)

var (
	host    = flag.String("host", "", "server hostname")
	dir     = flag.String("certs", "", "directory to store certificates in")
	devMode = flag.Bool("development", false, "run in development mode")
)

func main() {
	flag.Parse()
	t := handler.LoadTemplates("template/")
	if *devMode {
		t = nil
	}
	env := &handler.Env{Templates: t, TemplatePath: "template/"}
	model.InitPool(0)

	http.Handle("/", handler.Handler{Env: env, Fn: handler.Static})
	http.Handle("/game/", handler.Handler{Env: env, Fn: handler.Game})
	http.Handle("/static/", http.FileServer(http.Dir("./")))
	http.Handle("/live/game/", websocket.Handler(handler.Live))

	if *devMode {
		log.Printf("Starting server at http://0.0.0.0:8080\n")
		http.ListenAndServe("0.0.0.0:8080", nil)
		return
	}

	redir := "https://" + *host
	log.Printf("Starting server at http://" + *host)
	go http.ListenAndServe("0.0.0.0:80", http.RedirectHandler(redir, 301))

	m := autocert.Manager{
		Prompt:     autocert.AcceptTOS,
		HostPolicy: autocert.HostWhitelist(*host),
		Cache:      autocert.DirCache(*dir),
	}
	s := &http.Server{
		TLSConfig: &tls.Config{GetCertificate: m.GetCertificate},
	}
	log.Printf("Starting server at https://" + *host)
	log.Fatal(s.ListenAndServeTLS("", ""))
}
<commit_msg>Shorten dev mode flag<commit_after>// Command gogo is a web server for hosting multiplayer Go games.
package main

import (
	"crypto/tls"
	"flag"
	"log"
	"net/http"

	"github.com/waits/gogo/handler"
	"github.com/waits/gogo/model"
	"golang.org/x/crypto/acme/autocert"
	"golang.org/x/net/websocket"
)

var (
	host    = flag.String("host", "", "server hostname")
	dir     = flag.String("certs", "", "directory to store certificates in")
	devMode = flag.Bool("dev", false, "run in development mode")
)

func main() {
	flag.Parse()
	t := handler.LoadTemplates("template/")
	if *devMode {
		t = nil
	}
	env := &handler.Env{Templates: t, TemplatePath: "template/"}
	model.InitPool(0)

	http.Handle("/", handler.Handler{Env: env, Fn: handler.Static})
	http.Handle("/game/", handler.Handler{Env: env, Fn: handler.Game})
	http.Handle("/static/", http.FileServer(http.Dir("./")))
	http.Handle("/live/game/", websocket.Handler(handler.Live))

	if *devMode {
		log.Printf("Starting server at http://0.0.0.0:8080\n")
		http.ListenAndServe("0.0.0.0:8080", nil)
		return
	}

	redir := "https://" + *host
	log.Printf("Starting server at http://" + *host)
	go http.ListenAndServe("0.0.0.0:80", http.RedirectHandler(redir, 301))

	m := autocert.Manager{
		Prompt:     autocert.AcceptTOS,
		HostPolicy: autocert.HostWhitelist(*host),
		Cache:      autocert.DirCache(*dir),
	}
	s := &http.Server{
		TLSConfig: &tls.Config{GetCertificate: m.GetCertificate},
	}
	log.Printf("Starting server at https://" + *host)
	log.Fatal(s.ListenAndServeTLS("", ""))
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net"
	"net/http"
	"net/http/fcgi"
	"path"
	"strings"
	"time"

	"github.com/gin-gonic/gin"
)

// Logger is a simple log handler, out puts in the standard of apache access log common.
// See http://httpd.apache.org/docs/2.2/logs.html#accesslog
func Logger() gin.HandlerFunc {
	return func(c *gin.Context) {
		t := time.Now()
		ip, err := net.ResolveTCPAddr("tcp", c.Req.RemoteAddr)
		if err != nil {
			c.Abort(500)
		}

		// before request
		c.Next()
		// after request

		user := "-"
		if c.Req.URL.User != nil {
			user = c.Req.URL.User.Username()
		}

		latency := time.Since(t)

		// This is the format of Apache Log Common, with an additional field of latency
		fmt.Printf("%v - %v [%v] \"%v %v %v\" %v %v %v\n",
			ip.IP, user, t.Format(time.RFC3339), c.Req.Method, c.Req.URL.Path,
			c.Req.Proto, c.Writer.Status(), c.Req.ContentLength, latency)
	}
}

func mainHandler(c *gin.Context) {
	fields := strings.Split(c.Params.ByName("field"), ".")
	ip, err := net.ResolveTCPAddr("tcp", c.Req.RemoteAddr)
	if err != nil {
		c.Abort(500)
	}
	c.Set("ip", ip.IP.String())
	c.Set("port", ip.Port)
	c.Set("ua", c.Req.UserAgent())
	c.Set("lang", c.Req.Header.Get("Accept-Language"))
	c.Set("encoding", c.Req.Header.Get("Accept-Encoding"))
	c.Set("method",  c.Req.Method)
	c.Set("mime", c.Req.Header.Get("Accept"))
	c.Set("referer", c.Req.Header.Get("Referer"))
	c.Set("forwarded", c.Req.Header.Get("X-Forwarded-For"))

	hostnames, err := net.LookupAddr(ip.IP.String())
	if err != nil {
		c.Set("host", "")
	} else {
		c.Set("host", hostnames[0])
	}

	wantsJSON := false
	if len(fields) >= 2 && fields[1] == "json" {
		wantsJSON = true
	}

	ua := strings.Split(c.Req.UserAgent(), "/")
	switch fields[0] {
	case "":
		//If the user is using curl, then we should just return the IP, else we show the home page.
		if ua[0] == "curl" {
			c.String(200, fmt.Sprintln(ip.IP))
		} else {
			c.HTML(200, "index.html", c.Keys)
		}
		return
	case "request":
		c.JSON(200, c.Req)
		return
	case "all":
		if wantsJSON {
			c.JSON(200, c.Keys)
		} else {
			c.String(200, "%v", c.Keys)
		}
		return
	}

	fieldResult, err := c.Get(fields[0])
	if err != nil {
		c.String(404, "Not Found")
	}
	c.String(200, fmt.Sprintln(fieldResult))

}

// FileServer is a basic file serve handler, this is just here as an example.
// gin.Static() should be used instead
func FileServer(root string) gin.HandlerFunc {
	return func(c *gin.Context) {
		file := c.Params.ByName("file")
		if !strings.HasPrefix(file, "/") {
			file = "/" + file
		}
		http.ServeFile(c.Writer, c.Req, path.Join(root, path.Clean(file)))
	}
}

func main() {
	r := gin.New()
	r.Use(gin.Recovery())
	r.Use(Logger())
	r.LoadHTMLTemplates("templates/*")

	r.GET("/:field", mainHandler)
	r.GET("/", mainHandler)

	// Create a listener for FCGI
	fcgi_listen, err := net.Listen("tcp", "127.0.0.1:4000")
	if err != nil {
		panic(err)
	}
	err = fcgi.Serve(fcgi_listen, r)
	if err != nil {
		panic(err)
	}

	//port := os.Getenv("PORT")
	//host := os.Getenv("HOST")
	//if port == "" {
	//	port = "8080"
	//}
	//r.Run(host + ":" + port)
}
<commit_msg>Only lookup hostname if the results are going to need it.<commit_after>package main

import (
	"fmt"
	"net"
	"net/http"
	"net/http/fcgi"
	"path"
	"strings"
	"time"

	"github.com/gin-gonic/gin"
)

// Logger is a simple log handler, out puts in the standard of apache access log common.
// See http://httpd.apache.org/docs/2.2/logs.html#accesslog
func Logger() gin.HandlerFunc {
	return func(c *gin.Context) {
		t := time.Now()
		ip, err := net.ResolveTCPAddr("tcp", c.Req.RemoteAddr)
		if err != nil {
			c.Abort(500)
		}

		// before request
		c.Next()
		// after request

		user := "-"
		if c.Req.URL.User != nil {
			user = c.Req.URL.User.Username()
		}

		latency := time.Since(t)

		// This is the format of Apache Log Common, with an additional field of latency
		fmt.Printf("%v - %v [%v] \"%v %v %v\" %v %v %v\n",
			ip.IP, user, t.Format(time.RFC3339), c.Req.Method, c.Req.URL.Path,
			c.Req.Proto, c.Writer.Status(), c.Req.ContentLength, latency)
	}
}

func stringInSlice(a string, list []string) bool {
	for _, b := range list {
		if b == a {
			return true
		}
	}
	return false
}

func mainHandler(c *gin.Context) {
	fields := strings.Split(c.Params.ByName("field"), ".")
	ip, err := net.ResolveTCPAddr("tcp", c.Req.RemoteAddr)
	if err != nil {
		c.Abort(500)
	}
	c.Set("ip", ip.IP.String())
	c.Set("port", ip.Port)
	c.Set("ua", c.Req.UserAgent())
	c.Set("lang", c.Req.Header.Get("Accept-Language"))
	c.Set("encoding", c.Req.Header.Get("Accept-Encoding"))
	c.Set("method", c.Req.Method)
	c.Set("mime", c.Req.Header.Get("Accept"))
	c.Set("referer", c.Req.Header.Get("Referer"))
	c.Set("forwarded", c.Req.Header.Get("X-Forwarded-For"))

	// Only lookup hostname if the results are going to need it.
	if stringInSlice(fields[0], []string{"", "all", "host"}) {
		hostnames, err := net.LookupAddr(ip.IP.String())
		if err != nil {
			c.Set("host", "")
		} else {
			c.Set("host", hostnames[0])
		}
	}

	wantsJSON := false
	if len(fields) >= 2 && fields[1] == "json" {
		wantsJSON = true
	}

	ua := strings.Split(c.Req.UserAgent(), "/")
	switch fields[0] {
	case "":
		//If the user is using curl, then we should just return the IP, else we show the home page.
		if ua[0] == "curl" {
			c.String(200, fmt.Sprintln(ip.IP))
		} else {
			c.HTML(200, "index.html", c.Keys)
		}
		return
	case "request":
		c.JSON(200, c.Req)
		return
	case "all":
		if wantsJSON {
			c.JSON(200, c.Keys)
		} else {
			c.String(200, "%v", c.Keys)
		}
		return
	}

	fieldResult, err := c.Get(fields[0])
	if err != nil {
		c.String(404, "Not Found")
	}
	c.String(200, fmt.Sprintln(fieldResult))

}

// FileServer is a basic file serve handler, this is just here as an example.
// gin.Static() should be used instead
func FileServer(root string) gin.HandlerFunc {
	return func(c *gin.Context) {
		file := c.Params.ByName("file")
		if !strings.HasPrefix(file, "/") {
			file = "/" + file
		}
		http.ServeFile(c.Writer, c.Req, path.Join(root, path.Clean(file)))
	}
}

func main() {
	r := gin.New()
	r.Use(gin.Recovery())
	r.Use(Logger())
	r.LoadHTMLTemplates("templates/*")

	r.GET("/:field", mainHandler)
	r.GET("/", mainHandler)

	// Create a listener for FCGI
	fcgi_listen, err := net.Listen("tcp", "127.0.0.1:4000")
	if err != nil {
		panic(err)
	}
	err = fcgi.Serve(fcgi_listen, r)
	if err != nil {
		panic(err)
	}

	//port := os.Getenv("PORT")
	//host := os.Getenv("HOST")
	//if port == "" {
	//	port = "8080"
	//}
	//r.Run(host + ":" + port)
}
<|endoftext|>


<commit_before>/* Run tiedot HTTP API server, benchmarks, or embedded usage example. */
package main

import (
	"flag"
	"github.com/HouzuoGuo/tiedot/httpapi"
	"github.com/HouzuoGuo/tiedot/tdlog"
	"os"
	"os/signal"
	"runtime"
	"runtime/pprof"
	"strconv"
)

func main() {
	var err error
	var defaultMaxprocs int
	if defaultMaxprocs, err = strconv.Atoi(os.Getenv("GOMAXPROCS")); err != nil {
		defaultMaxprocs = runtime.NumCPU()
	}

	// Parse CLI parameters

	// General params
	var mode string
	var maxprocs int
	flag.StringVar(&mode, "mode", "", "[httpd|bench|bench2|example]")
	flag.IntVar(&maxprocs, "gomaxprocs", defaultMaxprocs, "GOMAXPROCS")
	// Debug params
	var profile, debug bool
	flag.BoolVar(&tdlog.VerboseLog, "verbose", false, "Turn verbose logging on/off")
	flag.BoolVar(&profile, "profile", false, "Write profiler results to prof.out")
	flag.BoolVar(&debug, "debug", false, "Dump goroutine stack traces upon receiving interrupt signal")
	// HTTP mode params
	var dir string
	var port int
	var tlsCrt, tlsKey string
	flag.StringVar(&dir, "dir", "", "(HTTP server) database directory")
	flag.IntVar(&port, "port", 8080, "(HTTP server) port number")
	flag.StringVar(&tlsCrt, "tlscrt", "", "(HTTP server) TLS certificate (empty to disable TLS).")
	flag.StringVar(&tlsKey, "tlskey", "", "(HTTP server) TLS certificate key (empty to disable TLS).")

	// HTTP + JWT params
	var jwtPubKey, jwtPrivateKey string
	flag.StringVar(&jwtPubKey, "jwtpubkey", "", "(HTTP JWT server) Public key for signing tokens (empty to disable JWT)")
	flag.StringVar(&jwtPrivateKey, "jwtprivatekey", "", "(HTTP JWT server) Private key for decoding tokens (empty to disable JWT)")

	// Benchmark mode params
	flag.IntVar(&benchSize, "benchsize", 400000, "Benchmark sample size")
	flag.BoolVar(&benchCleanup, "benchcleanup", true, "Whether to clean up (delete benchmark DB) after benchmark")
	flag.Parse()

	// User must specify a mode to run
	if mode == "" {
		flag.PrintDefaults()
		os.Exit(1)
	}

	// Set appropriate GOMAXPROCS
	runtime.GOMAXPROCS(maxprocs)
	tdlog.Noticef("GOMAXPROCS is set to %d", maxprocs)
	if maxprocs < runtime.NumCPU() {
		tdlog.Noticef("GOMAXPROCS (%d) is less than number of CPUs (%d), this may reduce performance. You can change it via environment variable GOMAXPROCS or by passing CLI parameter -gomaxprocs", maxprocs, runtime.NumCPU())
	}

	// Start profiler if enabled
	if profile {
		resultFile, err := os.Create("perf.out")
		if err != nil {
			tdlog.Noticef("Cannot create profiler result file %s", resultFile)
			os.Exit(1)
		}
		pprof.StartCPUProfile(resultFile)
		defer pprof.StopCPUProfile()
	}

	// Dump goroutine stacktraces upon receiving interrupt signal
	if debug {
		c := make(chan os.Signal, 1)
		signal.Notify(c, os.Interrupt)
		go func() {
			for _ = range c {
				pprof.Lookup("goroutine").WriteTo(os.Stderr, 1)
			}
		}()
	}

	switch mode {
	case "httpd":
		// Run HTTP API server
		if dir == "" {
			tdlog.Notice("Please specify database directory, for example -dir=/tmp/db")
			os.Exit(1)
		}
		if port == 0 {
			tdlog.Notice("Please specify port number, for example -port=8080")
			os.Exit(1)
		}
		if tlsCrt != "" && tlsKey == "" {
			tdlog.Notice("To enable HTTPS, please specify both RSA certificate and key file.")
			os.Exit(1)
		}
		if jwtPrivateKey != "" && jwtPubKey == "" {
			tdlog.Notice("To enable JWT, please specify RSA private and public key.")
			os.Exit(1)
		}
		httpapi.Start(dir, port, tlsCrt, tlsKey, jwtPubKey, jwtPrivateKey)
	case "example":
		// Run embedded usage examples
		embeddedExample()
	case "bench":
		// Benchmark scenarios
		benchmark()
	case "bench2":
		benchmark2()
	default:
		flag.PrintDefaults()
		return
	}
}
<commit_msg>offer performance advice based on system VM settings<commit_after>/* Run tiedot HTTP API server, benchmarks, or embedded usage example. */
package main

import (
	"flag"
	"github.com/HouzuoGuo/tiedot/httpapi"
	"github.com/HouzuoGuo/tiedot/tdlog"
	"io/ioutil"
	"os"
	"os/signal"
	"runtime"
	"runtime/pprof"
	"strconv"
	"strings"
)

// Read Linux system VM parameters and print performance configuration advice when necessary.
func linuxPerfAdvice() {
	readFileIntContent := func(filePath string) (contentInt int, err error) {
		content, err := ioutil.ReadFile(filePath)
		if err != nil {
			return
		}
		contentInt, err = strconv.Atoi(strings.TrimSpace(string(content)))
		return
	}
	swappiness, err := readFileIntContent("/proc/sys/vm/swappiness")
	if err != nil {
		tdlog.Notice("Non-fatal - unable to offer performance advice based on vm.swappiness.")
	} else if swappiness > 50 {
		tdlog.Noticef("System vm.swappiness is very high (%d), for optimium performance please lower it to below 50.", swappiness)
	}
	dirtyRatio, err := readFileIntContent("/proc/sys/vm/dirty_ratio")
	if err != nil {
		tdlog.Notice("Non-fatal - unable to offer performance advice based on vm.dirty_ratio.")
	} else if dirtyRatio < 50 {
		tdlog.Noticef("System vm.dirty_ratio is very low (%d), for optimium performance please increase it to above 50.", dirtyRatio)
	}
	dirtyBGRatio, err := readFileIntContent("/proc/sys/vm/dirty_background_ratio")
	if err != nil {
		tdlog.Notice("Non-fatal - unable to offer performance advice based on vm.dirty_background_ratio.")
	} else if dirtyBGRatio < 50 {
		tdlog.Noticef("System vm.dirty_background_ratio is very low (%d), for optimium performance please increase it to above 50.", dirtyBGRatio)
	}
}

func main() {
	var err error
	var defaultMaxprocs int
	if defaultMaxprocs, err = strconv.Atoi(os.Getenv("GOMAXPROCS")); err != nil {
		defaultMaxprocs = runtime.NumCPU()
	}

	// Parse CLI parameters

	// General params
	var mode string
	var maxprocs int
	flag.StringVar(&mode, "mode", "", "[httpd|bench|bench2|example]")
	flag.IntVar(&maxprocs, "gomaxprocs", defaultMaxprocs, "GOMAXPROCS")
	// Debug params
	var profile, debug bool
	flag.BoolVar(&tdlog.VerboseLog, "verbose", false, "Turn verbose logging on/off")
	flag.BoolVar(&profile, "profile", false, "Write profiler results to prof.out")
	flag.BoolVar(&debug, "debug", false, "Dump goroutine stack traces upon receiving interrupt signal")
	// HTTP mode params
	var dir string
	var port int
	var tlsCrt, tlsKey string
	flag.StringVar(&dir, "dir", "", "(HTTP server) database directory")
	flag.IntVar(&port, "port", 8080, "(HTTP server) port number")
	flag.StringVar(&tlsCrt, "tlscrt", "", "(HTTP server) TLS certificate (empty to disable TLS).")
	flag.StringVar(&tlsKey, "tlskey", "", "(HTTP server) TLS certificate key (empty to disable TLS).")

	// HTTP + JWT params
	var jwtPubKey, jwtPrivateKey string
	flag.StringVar(&jwtPubKey, "jwtpubkey", "", "(HTTP JWT server) Public key for signing tokens (empty to disable JWT)")
	flag.StringVar(&jwtPrivateKey, "jwtprivatekey", "", "(HTTP JWT server) Private key for decoding tokens (empty to disable JWT)")

	// Benchmark mode params
	flag.IntVar(&benchSize, "benchsize", 400000, "Benchmark sample size")
	flag.BoolVar(&benchCleanup, "benchcleanup", true, "Whether to clean up (delete benchmark DB) after benchmark")
	flag.Parse()

	// User must specify a mode to run
	if mode == "" {
		flag.PrintDefaults()
		os.Exit(1)
	}

	// Set appropriate GOMAXPROCS
	runtime.GOMAXPROCS(maxprocs)
	tdlog.Noticef("GOMAXPROCS is set to %d", maxprocs)

	// Performance advices
	if maxprocs < runtime.NumCPU() {
		tdlog.Noticef("GOMAXPROCS (%d) is less than number of CPUs (%d), this may reduce performance. You can change it via environment variable GOMAXPROCS or by passing CLI parameter -gomaxprocs", maxprocs, runtime.NumCPU())
	}
	linuxPerfAdvice()

	// Start profiler if enabled
	if profile {
		resultFile, err := os.Create("perf.out")
		if err != nil {
			tdlog.Noticef("Cannot create profiler result file %s", resultFile)
			os.Exit(1)
		}
		pprof.StartCPUProfile(resultFile)
		defer pprof.StopCPUProfile()
	}

	// Dump goroutine stacktraces upon receiving interrupt signal
	if debug {
		c := make(chan os.Signal, 1)
		signal.Notify(c, os.Interrupt)
		go func() {
			for _ = range c {
				pprof.Lookup("goroutine").WriteTo(os.Stderr, 1)
			}
		}()
	}

	switch mode {
	case "httpd":
		// Run HTTP API server
		if dir == "" {
			tdlog.Notice("Please specify database directory, for example -dir=/tmp/db")
			os.Exit(1)
		}
		if port == 0 {
			tdlog.Notice("Please specify port number, for example -port=8080")
			os.Exit(1)
		}
		if tlsCrt != "" && tlsKey == "" {
			tdlog.Notice("To enable HTTPS, please specify both RSA certificate and key file.")
			os.Exit(1)
		}
		if jwtPrivateKey != "" && jwtPubKey == "" {
			tdlog.Notice("To enable JWT, please specify RSA private and public key.")
			os.Exit(1)
		}
		httpapi.Start(dir, port, tlsCrt, tlsKey, jwtPubKey, jwtPrivateKey)
	case "example":
		// Run embedded usage examples
		embeddedExample()
	case "bench":
		// Benchmark scenarios
		benchmark()
	case "bench2":
		benchmark2()
	default:
		flag.PrintDefaults()
		return
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"

	"github.com/FiloSottile/zcash-mini/zcash"
)

var logo = `

             :88SX@888@@X8:
          %%Xt%tt%SSSSS:XXXt@@
        @S;;tt%%%t    ;;::XXXXSX
      .t:::;;%8888    88888tXXXX8;
     .%...:::8             8::XXX%;
     8888...:t888888X     8t;;::XX8
    %888888...:::;:8    :Xttt;;;::X@
    888888888...:St    8:%%tttt;;;:X
    88888888888S8    :%;ttt%%tttt;;X
    %888888888%t    8S:;;;tt%%%ttt;8
     8t8888888     S8888888Stt%%%t@
     .@tt888@              8;;ttt@;
      .8ttt8@SSSSS    SXXXX%:;;;X;
        X8ttt8888%    %88...::X8
          %8@tt88;8888%8888%8X
             :@888@XXX@888:

                    _       _
          _ __ ___ (_)_ __ (_)
         | '_ \ _ \| | '_ \| |
         | | | | | | | | | | |
         |_| |_| |_|_|_| |_|_|

`

var template = `%s

###############################################################
#
#  Here is your new z-address
#
#      %s
#
#  and here is the secret key
#
#      %s
#
#  KEEP IT SAFE, IT HAS NOT BEEN SAVED ANYWHERE
#
#  To spend, import it with
#
#      zcash-cli z_importkey KEY rescan
#
###############################################################

`

func main() {
	rawKey := zcash.GenerateKey()
	rawAddr, err := zcash.KeyToAddress(rawKey)
	if err != nil {
		log.Fatal(err)
	}

	key := zcash.Base58Encode(rawKey, zcash.ProdSpendingKey)
	addr := zcash.Base58Encode(rawAddr, zcash.ProdAddress)

	fmt.Printf(template, logo, addr, key)
}
<commit_msg>Add simple mode<commit_after>package main

import (
	"flag"
	"fmt"
	"log"

	"github.com/FiloSottile/zcash-mini/zcash"
)

var logo = `

             :88SX@888@@X8:
          %%Xt%tt%SSSSS:XXXt@@
        @S;;tt%%%t    ;;::XXXXSX
      .t:::;;%8888    88888tXXXX8;
     .%...:::8             8::XXX%;
     8888...:t888888X     8t;;::XX8
    %888888...:::;:8    :Xttt;;;::X@
    888888888...:St    8:%%tttt;;;:X
    88888888888S8    :%;ttt%%tttt;;X
    %888888888%t    8S:;;;tt%%%ttt;8
     8t8888888     S8888888Stt%%%t@
     .@tt888@              8;;ttt@;
      .8ttt8@SSSSS    SXXXX%:;;;X;
        X8ttt8888%    %88...::X8
          %8@tt88;8888%8888%8X
             :@888@XXX@888:

                    _       _
          _ __ ___ (_)_ __ (_)
         | '_ \ _ \| | '_ \| |
         | | | | | | | | | | |
         |_| |_| |_|_|_| |_|_|

`

var template = `%s

###############################################################
#
#  Here is your new z-address
#
#      %s
#
#  and here is the secret key
#
#      %s
#
#  KEEP IT SAFE, IT HAS NOT BEEN SAVED ANYWHERE
#
#  To spend, import it with
#
#      zcash-cli z_importkey KEY rescan
#
###############################################################

`

func main() {
	simpleMode := flag.Bool("simple", false, "output only address and key")
	flag.Parse()

	rawKey := zcash.GenerateKey()
	rawAddr, err := zcash.KeyToAddress(rawKey)
	if err != nil {
		log.Fatal(err)
	}

	key := zcash.Base58Encode(rawKey, zcash.ProdSpendingKey)
	addr := zcash.Base58Encode(rawAddr, zcash.ProdAddress)

	if *simpleMode {
		fmt.Printf("%s\n%s\n", addr, key)
	} else {
		fmt.Printf(template, logo, addr, key)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"github.com/Financial-Times/go-fthealth/v1a"
	"github.com/cyberdelia/go-metrics-graphite"
	"github.com/gorilla/handlers"
	"github.com/gorilla/mux"
	"github.com/jawher/mow.cli"
	"github.com/jmcvetta/neoism"
	"github.com/rcrowley/go-metrics"
	"io"
	"log"
	"net"
	"net/http"
	_ "net/http/pprof"
	"os"
	"time"
)

var peopleDriver PeopleDriver

func main() {
	fmt.Println(os.Args)
	app := cli.App("people-rw-neo4j", "A RESTful API for managing People in neo4j")
	neoURL := app.StringOpt("neo-url", "http://localhost:7474/db/data", "neo4j endpoint URL")
	port := app.StringOpt("port", "8080", "Port to listen on")
	batchSize := app.IntOpt("batchSize", 1024, "Maximum number of statements to execute per batch")
	timeoutMs := app.IntOpt("timeoutMs", 20,
		"Number of milliseconds to wait before executing a batch of statements regardless of its size")
	graphiteTCPAddress := app.StringOpt("graphiteTCPAddress", "",
		"Graphite TCP address, e.g. graphite.ft.com:2003. Leave as default if you do NOT want to output to graphite (e.g. if running locally)")
	graphitePrefix := app.StringOpt("graphitePrefix", "",
		"Prefix to use. Should start with content, include the environment, and the host name. e.g. content.test.people.rw.neo4j.ftaps58938-law1a-eu-t")
	logMetrics := app.BoolOpt("logMetrics", false, "Whether to log metrics. Set to true if running locally and you want metrics output")

	app.Action = func() {
		runServer(*neoURL, *port, *batchSize, *timeoutMs, *graphiteTCPAddress, *graphitePrefix, *logMetrics)
	}

	app.Run(os.Args)
}

func runServer(neoURL string, port string, batchSize int, timeoutMs int, graphiteTCPAddress string,
	graphitePrefix string, logMetrics bool) {

	flags := log.Ldate | log.Ltime | log.Lshortfile

	log.SetFlags(flags)

	db, err := neoism.Connect(neoURL)
	if err != nil {
		log.Println("ERROR Could not connect to neo4j, error=[%s]", err)
	}

	personIndexes, err := db.Indexes("Person")

	if err != nil {
		log.Println("ERROR Error on creating index=%v", err)
	}

	var indexFound bool

	for _, index := range personIndexes {
		if len(index.PropertyKeys) == 1 && index.PropertyKeys[0] == "uuid" {
			indexFound = true
			break
		}
	}
	if !indexFound {
		log.Println("INFO Creating index for person for neo4j instance at %s", neoURL)
		db.CreateIndex("Person", "uuid")
	}

	peopleDriver = NewPeopleCypherDriver(NewBatchCypherRunner(db, batchSize, time.Millisecond*time.Duration(timeoutMs)))

	if graphiteTCPAddress != "" {
		addr, _ := net.ResolveTCPAddr("tcp", graphiteTCPAddress)
		go graphite.Graphite(metrics.DefaultRegistry, 1*time.Minute, graphitePrefix, addr)
	}
	if logMetrics { //useful locally
		go metrics.Log(metrics.DefaultRegistry, 60*time.Second, log.New(os.Stderr, "metrics: ", log.Lmicroseconds))
	}

	r := mux.NewRouter()
	r.HandleFunc("/people/{uuid}", peopleWrite).Methods("PUT")
	r.HandleFunc("/people/{uuid}", peopleRead).Methods("GET")
	r.HandleFunc("/__health", v1a.Handler("PeopleReadWriteNeo4j Healthchecks",
		"Checks for accessing neo4j", setUpHealthCheck(db)))
	r.HandleFunc("/ping", ping)
	http.ListenAndServe(":"+port, HttpMetricsHandler(handlers.CombinedLoggingHandler(os.Stdout, r)))
}

func ping(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "pong")
}

func peopleWrite(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	uuid := vars["uuid"]
	p, err := parsePerson(r.Body)
	if err != nil || p.UUID != uuid {
		log.Println("ERROR Error on parse=%v", err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	err = peopleDriver.Write(p)
	if err != nil {
		log.Println("ERROR Error on write=%v", err)
		w.WriteHeader(http.StatusServiceUnavailable)
		return
	}
	//Not necessary for a 200 to be returned, but for PUT requests, if don't specify, don't see 200 status logged in request logs
	w.WriteHeader(http.StatusOK)
}

func peopleRead(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	uuid := vars["uuid"]

	p, found, err := peopleDriver.Read(uuid)

	w.Header().Add("Content-Type", "application/json")

	if err != nil {
		log.Println("ERROR Error on read=%v", err)
		w.WriteHeader(http.StatusServiceUnavailable)
		return
	}

	if !found {
		w.WriteHeader(http.StatusNotFound)
		return
	}

	enc := json.NewEncoder(w)

	if err := enc.Encode(p); err != nil {
		log.Println("ERROR Error on json encoding=%v", err)
		w.WriteHeader(http.StatusServiceUnavailable)
		return
	}
}

func parsePerson(jsonInput io.Reader) (person, error) {
	dec := json.NewDecoder(jsonInput)
	var p person
	err := dec.Decode(&p)
	return p, err
}

func HttpMetricsHandler(h http.Handler) http.Handler {
	return httpMetricsHandler{h}
}

type httpMetricsHandler struct {
	handler http.Handler
}

func (h httpMetricsHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	t := metrics.GetOrRegisterTimer(req.Method, metrics.DefaultRegistry)
	t.Time(func() { h.handler.ServeHTTP(w, req) })
}
<commit_msg>move index creation into function.<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"github.com/Financial-Times/go-fthealth/v1a"
	"github.com/cyberdelia/go-metrics-graphite"
	"github.com/gorilla/handlers"
	"github.com/gorilla/mux"
	"github.com/jawher/mow.cli"
	"github.com/jmcvetta/neoism"
	"github.com/rcrowley/go-metrics"
	"io"
	"log"
	"net"
	"net/http"
	_ "net/http/pprof"
	"os"
	"time"
)

var peopleDriver PeopleDriver

func main() {
	fmt.Println(os.Args)
	app := cli.App("people-rw-neo4j", "A RESTful API for managing People in neo4j")
	neoURL := app.StringOpt("neo-url", "http://localhost:7474/db/data", "neo4j endpoint URL")
	port := app.StringOpt("port", "8080", "Port to listen on")
	batchSize := app.IntOpt("batchSize", 1024, "Maximum number of statements to execute per batch")
	timeoutMs := app.IntOpt("timeoutMs", 20,
		"Number of milliseconds to wait before executing a batch of statements regardless of its size")
	graphiteTCPAddress := app.StringOpt("graphiteTCPAddress", "",
		"Graphite TCP address, e.g. graphite.ft.com:2003. Leave as default if you do NOT want to output to graphite (e.g. if running locally)")
	graphitePrefix := app.StringOpt("graphitePrefix", "",
		"Prefix to use. Should start with content, include the environment, and the host name. e.g. content.test.people.rw.neo4j.ftaps58938-law1a-eu-t")
	logMetrics := app.BoolOpt("logMetrics", false, "Whether to log metrics. Set to true if running locally and you want metrics output")

	app.Action = func() {
		runServer(*neoURL, *port, *batchSize, *timeoutMs, *graphiteTCPAddress, *graphitePrefix, *logMetrics)
	}

	app.Run(os.Args)
}

func runServer(neoURL string, port string, batchSize int, timeoutMs int, graphiteTCPAddress string,
	graphitePrefix string, logMetrics bool) {

	flags := log.Ldate | log.Ltime | log.Lshortfile

	log.SetFlags(flags)

	db, err := neoism.Connect(neoURL)
	if err != nil {
		log.Println("ERROR Could not connect to neo4j, error=[%s]", err)
	}

	ensureIndex(db, "Person", "uuid")

	peopleDriver = NewPeopleCypherDriver(NewBatchCypherRunner(db, batchSize, time.Millisecond*time.Duration(timeoutMs)))

	if graphiteTCPAddress != "" {
		addr, _ := net.ResolveTCPAddr("tcp", graphiteTCPAddress)
		go graphite.Graphite(metrics.DefaultRegistry, 1*time.Minute, graphitePrefix, addr)
	}
	if logMetrics { //useful locally
		go metrics.Log(metrics.DefaultRegistry, 60*time.Second, log.New(os.Stderr, "metrics: ", log.Lmicroseconds))
	}

	r := mux.NewRouter()
	r.HandleFunc("/people/{uuid}", peopleWrite).Methods("PUT")
	r.HandleFunc("/people/{uuid}", peopleRead).Methods("GET")
	r.HandleFunc("/__health", v1a.Handler("PeopleReadWriteNeo4j Healthchecks",
		"Checks for accessing neo4j", setUpHealthCheck(db)))
	r.HandleFunc("/ping", ping)
	http.ListenAndServe(":"+port, HttpMetricsHandler(handlers.CombinedLoggingHandler(os.Stdout, r)))
}

func ensureIndex(db *neoism.Database, label string, property string) {

	personIndexes, err := db.Indexes(label)

	if err != nil {
		log.Println("ERROR Error on creating index=%v", err)
	}

	var indexFound bool

	for _, index := range personIndexes {
		if len(index.PropertyKeys) == 1 && index.PropertyKeys[0] == property {
			indexFound = true
			break
		}
	}
	if !indexFound {
		log.Println("INFO Creating index for person for neo4j instance at %s", db.Url)
		db.CreateIndex(label, property)
	}

}

func ping(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "pong")
}

func peopleWrite(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	uuid := vars["uuid"]
	p, err := parsePerson(r.Body)
	if err != nil || p.UUID != uuid {
		log.Println("ERROR Error on parse=%v", err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	err = peopleDriver.Write(p)
	if err != nil {
		log.Println("ERROR Error on write=%v", err)
		w.WriteHeader(http.StatusServiceUnavailable)
		return
	}
	//Not necessary for a 200 to be returned, but for PUT requests, if don't specify, don't see 200 status logged in request logs
	w.WriteHeader(http.StatusOK)
}

func peopleRead(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	uuid := vars["uuid"]

	p, found, err := peopleDriver.Read(uuid)

	w.Header().Add("Content-Type", "application/json")

	if err != nil {
		log.Println("ERROR Error on read=%v", err)
		w.WriteHeader(http.StatusServiceUnavailable)
		return
	}

	if !found {
		w.WriteHeader(http.StatusNotFound)
		return
	}

	enc := json.NewEncoder(w)

	if err := enc.Encode(p); err != nil {
		log.Println("ERROR Error on json encoding=%v", err)
		w.WriteHeader(http.StatusServiceUnavailable)
		return
	}
}

func parsePerson(jsonInput io.Reader) (person, error) {
	dec := json.NewDecoder(jsonInput)
	var p person
	err := dec.Decode(&p)
	return p, err
}

func HttpMetricsHandler(h http.Handler) http.Handler {
	return httpMetricsHandler{h}
}

type httpMetricsHandler struct {
	handler http.Handler
}

func (h httpMetricsHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	t := metrics.GetOrRegisterTimer(req.Method, metrics.DefaultRegistry)
	t.Time(func() { h.handler.ServeHTTP(w, req) })
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"io/ioutil"
	"log"
	"net/http"
	"strconv"
	"strings"
)

var (
	warpAddr      = flag.String("warp.addr", "127.0.0.1:4242", "Address of sensision")
	listenAddress = flag.String("web.listen-address", ":9121", "Address to listen on for web interface and telemetry.")
	metricPath    = flag.String("web.telemetry-path", "/metrics", "Path under which to expose metrics.")
	debug         = flag.Bool("debug", false, "Debug mode displays what is fetch.")
)

type warp struct {
	warpAddr string
	metrics  map[string]warpMetric
}

type warpMetric struct {
	desc    *prometheus.Desc
	valType prometheus.ValueType
}

func parseFloatOrZero(s string) float64 {
	res, err := strconv.ParseFloat(s, 64)
	if err != nil {
		log.Printf("Cannot parse %s\n", s)
		return 0.0
	}
	return res
}

func NewWarpExporter(addr string) *warp {
	e := warp{
		warpAddr: addr,
		metrics: map[string]warpMetric{
			"warp_ingress_update_requests": {
				desc:    prometheus.NewDesc("warp_ingress_update_requests", "Number of request", nil, nil),
				valType: prometheus.CounterValue,
			},
			"warp_ingress_update_parseerrors": {
				desc:    prometheus.NewDesc("warp_ingress_update_parseerrors", "Number of parse error", nil, nil),
				valType: prometheus.CounterValue,
			},
			"warp_ingress_update_invalidtoken": {
				desc:    prometheus.NewDesc("warp_ingress_update_invalidtoken", "Number of invalid token", nil, nil),
				valType: prometheus.CounterValue,
			},
		},
	}
	return &e
}

func (w *warp) scrapeSensisionMetrics(ch chan<- prometheus.Metric) {
	resp, err := http.Get("http://" + w.warpAddr + "/metrics")
	if err == nil {
		defer resp.Body.Close()
		body, _ := ioutil.ReadAll(resp.Body)
		for _, line := range strings.Split(string(body), "\n") {
			tokens := strings.Split(line, " ")
			if len(tokens) == 3 {
				metric := strings.Replace(tokens[1], ".", "_", -1)
				metric = metric[:strings.IndexRune(metric, '{')]
				if val, ok := w.metrics[metric]; ok {
					value := parseFloatOrZero(tokens[2])
					if *debug {
						log.Printf("Sensision output %s\n", line)
						log.Printf("Metric name %s, value %f\n", metric, value)
					}
					ch <- prometheus.MustNewConstMetric(val.desc, val.valType, value)
				}
			}
		}
	}
}

func (w *warp) Describe(ch chan<- *prometheus.Desc) {
	for _, i := range w.metrics {
		ch <- i.desc
	}
}

func (w *warp) Collect(ch chan<- prometheus.Metric) {
	if *debug {
		log.Printf("Start collecting")
	}
	w.scrapeSensisionMetrics(ch)
	if *debug {
		log.Printf("Stop collecting")
	}
}

func main() {
	flag.Parse()
	e := NewWarpExporter(*warpAddr)
	prometheus.MustRegister(e)
	http.Handle(*metricPath, promhttp.Handler())
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(`
		       <title>Warp10 exporter</title>

Warp10 exporter

<a href='` + *metricPath + `'>Metrics

		       `))
	})
	log.Printf("providing metrics at %s%s", *listenAddress, *metricPath)
	log.Fatal(http.ListenAndServe(*listenAddress, nil))
}
<commit_msg>Trim sensision output<commit_after>package main

import (
	"flag"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"io/ioutil"
	"log"
	"net/http"
	"strconv"
	"strings"
)

var (
	warpAddr      = flag.String("warp.addr", "127.0.0.1:4242", "Address of sensision")
	listenAddress = flag.String("web.listen-address", ":9121", "Address to listen on for web interface and telemetry.")
	metricPath    = flag.String("web.telemetry-path", "/metrics", "Path under which to expose metrics.")
	debug         = flag.Bool("debug", false, "Debug mode displays what is fetch.")
)

type warp struct {
	warpAddr string
	metrics  map[string]warpMetric
}

type warpMetric struct {
	desc    *prometheus.Desc
	valType prometheus.ValueType
}

func parseFloatOrZero(s string) float64 {
	res, err := strconv.ParseFloat(s, 64)
	if err != nil {
		log.Printf("Cannot parse %s\n", s)
		return 0.0
	}
	return res
}

func NewWarpExporter(addr string) *warp {
	e := warp{
		warpAddr: addr,
		metrics: map[string]warpMetric{
			"warp_ingress_update_requests": {
				desc:    prometheus.NewDesc("warp_ingress_update_requests", "Number of request", nil, nil),
				valType: prometheus.CounterValue,
			},
			"warp_ingress_update_parseerrors": {
				desc:    prometheus.NewDesc("warp_ingress_update_parseerrors", "Number of parse error", nil, nil),
				valType: prometheus.CounterValue,
			},
			"warp_ingress_update_invalidtoken": {
				desc:    prometheus.NewDesc("warp_ingress_update_invalidtoken", "Number of invalid token", nil, nil),
				valType: prometheus.CounterValue,
			},
		},
	}
	return &e
}

func (w *warp) scrapeSensisionMetrics(ch chan<- prometheus.Metric) {
	resp, err := http.Get("http://" + w.warpAddr + "/metrics")
	if err == nil {
		defer resp.Body.Close()
		body, _ := ioutil.ReadAll(resp.Body)
		for _, line := range strings.Split(string(body), "\n") {
			tokens := strings.Split(strings.Trim(line, "\t\r\n"), " ")
			if len(tokens) == 3 {
				metric := strings.Replace(tokens[1], ".", "_", -1)
				metric = metric[:strings.IndexRune(metric, '{')]
				if val, ok := w.metrics[metric]; ok {
					value := parseFloatOrZero(tokens[2])
					if *debug {
						log.Printf("Sensision output %s\n", line)
						log.Printf("Metric name %s, value %f\n", metric, value)
					}
					ch <- prometheus.MustNewConstMetric(val.desc, val.valType, value)
				}
			}
		}
	} else {
		log.Printf("Cannot fetch sensision output\n")
	}
}

func (w *warp) Describe(ch chan<- *prometheus.Desc) {
	for _, i := range w.metrics {
		ch <- i.desc
	}
}

func (w *warp) Collect(ch chan<- prometheus.Metric) {
	if *debug {
		log.Printf("Start collecting")
	}
	w.scrapeSensisionMetrics(ch)
	if *debug {
		log.Printf("Stop collecting")
	}
}

func main() {
	flag.Parse()
	e := NewWarpExporter(*warpAddr)
	prometheus.MustRegister(e)
	http.Handle(*metricPath, promhttp.Handler())
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(`
		       <title>Warp10 exporter</title>

Warp10 exporter

<a href='` + *metricPath + `'>Metrics

		       `))
	})
	log.Printf("providing metrics at %s%s", *listenAddress, *metricPath)
	log.Fatal(http.ListenAndServe(*listenAddress, nil))
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"strings"

	"github.com/gorilla/websocket"
)

var url = flag.String("url", "", "-url ws://127.0.0.1")

func main() {
	flag.Parse()
	if *url == "" {
		log.Fatal("URL can't be empty.")
	}
	done := make(chan bool)
	c := NewClient(*url)

	go func() {
		for {
			_, r, err := c.ReadMessage()
			if err != nil {
				log.Fatal(err)
				return
			}
			fmt.Println("\n", color(string(r)))
		}
	}()

	r := bufio.NewReader(os.Stdin)
	go func() {
		for {
			line, err := r.ReadString('\n')
			if err != nil && err.Error() != "unexpected newline" {
				fmt.Println(err)
				return
			}
			line = strings.TrimSpace(line)
			if err = c.WriteMessage(1, []byte(line)); err != nil {
				fmt.Println(err.Error())
			}
		}
	}()

	<-done

}

func NewClient(url string) *websocket.Conn {
	r, _ := http.NewRequest("GET", url, nil)
	r.Header.Add("Content-Type", "application/json")
	c, _, err := websocket.DefaultDialer.Dial(url, r.Header)
	if err != nil {
		log.Fatal("errrr ", err)
	} else {
		fmt.Println("Connected!")
	}

	return c
}

func color(msg string) string {
	return ("\033[36m" + msg + "\033[0m")
}
<commit_msg>Added 'headers' parameter for pass custom headers to websocket connection (#4)<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"strings"

	"github.com/gorilla/websocket"
)

var url = flag.String("url", "", "-url ws://127.0.0.1")
var headers = flag.String("headers", "", "-headers name1=value1,name2=value2")

func main() {
	flag.Parse()
	if *url == "" {
		log.Fatal("URL can't be empty.")
	}
	headersMap := make(map[string]string)
	if *headers != "" {
		headersSeq := strings.Split(*headers, ",")
		for _, headerString := range headersSeq {
			splittedHeader := strings.Split(headerString, "=")
			headersMap[splittedHeader[0]] = splittedHeader[1]
		}
	}
	done := make(chan bool)
	c := NewClient(*url, headersMap)

	go func() {
		for {
			_, r, err := c.ReadMessage()
			if err != nil {
				log.Fatal(err)
				return
			}
			fmt.Println("\n", color(string(r)))
		}
	}()

	r := bufio.NewReader(os.Stdin)
	go func() {
		for {
			line, err := r.ReadString('\n')
			if err != nil && err.Error() != "unexpected newline" {
				fmt.Println(err)
				return
			}
			line = strings.TrimSpace(line)
			if err = c.WriteMessage(1, []byte(line)); err != nil {
				fmt.Println(err.Error())
			}
		}
	}()

	<-done

}

func NewClient(url string, headers map[string]string) *websocket.Conn {
	r, _ := http.NewRequest("GET", url, nil)
	r.Header.Add("Content-Type", "application/json")
	for key, value := range headers {
		r.Header.Add(key, value)
	}
	c, _, err := websocket.DefaultDialer.Dial(url, r.Header)
	if err != nil {
		log.Fatal("errrr ", err)
	} else {
		fmt.Println("Connected!")
	}

	return c
}

func color(msg string) string {
	return ("\033[36m" + msg + "\033[0m")
}
<|endoftext|>