<commit_before>package api

import (
	"encoding/json"
	"log"
	"net/http"
	"strings"
	"time"

	"github.com/stretchr/graceful"
)

const (
	apiTimeout = 5 * time.Second
)

// handleHTTPRequest is a wrapper function that logs and then handles all
// incoming calls to the API.
func handleHTTPRequest(mux *http.ServeMux, url string, handler http.HandlerFunc) {
	mux.HandleFunc(url, func(w http.ResponseWriter, req *http.Request) {
		log.Printf("%s %s", req.Method, req.URL)
		handler(w, req)
	})
}

// initAPI determines which functions handle each API call.
func (srv *Server) initAPI(addr string) {
	mux := http.NewServeMux()

	// Consensus API Calls
	handleHTTPRequest(mux, "/consensus/status", srv.consensusStatusHandler)

	// Server API Calls
	handleHTTPRequest(mux, "/daemon/stop", srv.daemonStopHandler)
	handleHTTPRequest(mux, "/daemon/update/apply", srv.daemonUpdateApplyHandler)
	handleHTTPRequest(mux, "/daemon/update/check", srv.daemonUpdateCheckHandler)

	// Debugging API Calls
	handleHTTPRequest(mux, "/debug/constants", srv.debugConstantsHandler)
	handleHTTPRequest(mux, "/debug/mutextest", srv.mutexTestHandler)

	// Gateway API Calls
	handleHTTPRequest(mux, "/gateway/status", srv.gatewayStatusHandler)
	handleHTTPRequest(mux, "/gateway/synchronize", srv.gatewaySynchronizeHandler)
	handleHTTPRequest(mux, "/gateway/peer/add", srv.gatewayPeerAddHandler)
	handleHTTPRequest(mux, "/gateway/peer/remove", srv.gatewayPeerRemoveHandler)

	// Host API Calls
	handleHTTPRequest(mux, "/host/announce", srv.hostAnnounceHandler)
	handleHTTPRequest(mux, "/host/config", srv.hostConfigHandler)
	handleHTTPRequest(mux, "/host/status", srv.hostStatusHandler)

	// HostDB API Calls

	// Miner API Calls
	handleHTTPRequest(mux, "/miner/start", srv.minerStartHandler)
	handleHTTPRequest(mux, "/miner/status", srv.minerStatusHandler)
	handleHTTPRequest(mux, "/miner/stop", srv.minerStopHandler)

	// Renter API Calls
	handleHTTPRequest(mux, "/renter/download", srv.renterDownloadHandler)
	handleHTTPRequest(mux, "/renter/downloadqueue", srv.renterDownloadqueueHandler)
	handleHTTPRequest(mux, "/renter/files", srv.renterFilesHandler)
	handleHTTPRequest(mux, "/renter/status", srv.renterStatusHandler)
	handleHTTPRequest(mux, "/renter/upload", srv.renterUploadHandler)

	// TransactionPool API Calls
	handleHTTPRequest(mux, "/transactionpool/transactions", srv.transactionpoolTransactionsHandler)

	// Wallet API Calls
	handleHTTPRequest(mux, "/wallet/address", srv.walletAddressHandler)
	handleHTTPRequest(mux, "/wallet/send", srv.walletSendHandler)
	handleHTTPRequest(mux, "/wallet/status", srv.walletStatusHandler)

	// create graceful HTTP server
	srv.apiServer = &graceful.Server{
		Timeout: apiTimeout,
		Server:  &http.Server{Addr: addr, Handler: mux},
	}
}

// Serve listens for and handles API calls. It a blocking function.
func (srv *Server) Serve() error {
	// graceful will run until it catches a signal.
	// It can also be stopped manually by stopHandler.
	err := srv.apiServer.ListenAndServe()
	// despite its name, graceful still propogates this benign error
	if err != nil && strings.HasSuffix(err.Error(), "use of closed network connection") {
		err = nil
	}
	return err
}

// writeError logs an writes an error to the API caller.
func writeError(w http.ResponseWriter, msg string, err int) {
	log.Printf("%d HTTP ERROR: %s", err, msg)
	http.Error(w, msg, err)
}

// writeJSON writes the object to the ResponseWriter. If the encoding fails, an
// error is written instead.
func writeJSON(w http.ResponseWriter, obj interface{}) {
	if json.NewEncoder(w).Encode(obj) != nil {
		http.Error(w, "Failed to encode response", http.StatusInternalServerError)
	}
}

// writeSuccess writes the success json object ({"Success":true}) to the
// ResponseWriter
func writeSuccess(w http.ResponseWriter) {
	writeJSON(w, struct{ Success bool }{true})
}
<commit_msg>indicate that siad caught the stop signal<commit_after>package api

import (
	"encoding/json"
	"log"
	"net/http"
	"strings"
	"time"

	"github.com/stretchr/graceful"
)

const (
	apiTimeout = 5 * time.Second
)

// handleHTTPRequest is a wrapper function that logs and then handles all
// incoming calls to the API.
func handleHTTPRequest(mux *http.ServeMux, url string, handler http.HandlerFunc) {
	mux.HandleFunc(url, func(w http.ResponseWriter, req *http.Request) {
		log.Printf("%s %s", req.Method, req.URL)
		handler(w, req)
	})
}

// initAPI determines which functions handle each API call.
func (srv *Server) initAPI(addr string) {
	mux := http.NewServeMux()

	// Consensus API Calls
	handleHTTPRequest(mux, "/consensus/status", srv.consensusStatusHandler)

	// Server API Calls
	handleHTTPRequest(mux, "/daemon/stop", srv.daemonStopHandler)
	handleHTTPRequest(mux, "/daemon/update/apply", srv.daemonUpdateApplyHandler)
	handleHTTPRequest(mux, "/daemon/update/check", srv.daemonUpdateCheckHandler)

	// Debugging API Calls
	handleHTTPRequest(mux, "/debug/constants", srv.debugConstantsHandler)
	handleHTTPRequest(mux, "/debug/mutextest", srv.mutexTestHandler)

	// Gateway API Calls
	handleHTTPRequest(mux, "/gateway/status", srv.gatewayStatusHandler)
	handleHTTPRequest(mux, "/gateway/synchronize", srv.gatewaySynchronizeHandler)
	handleHTTPRequest(mux, "/gateway/peer/add", srv.gatewayPeerAddHandler)
	handleHTTPRequest(mux, "/gateway/peer/remove", srv.gatewayPeerRemoveHandler)

	// Host API Calls
	handleHTTPRequest(mux, "/host/announce", srv.hostAnnounceHandler)
	handleHTTPRequest(mux, "/host/config", srv.hostConfigHandler)
	handleHTTPRequest(mux, "/host/status", srv.hostStatusHandler)

	// HostDB API Calls

	// Miner API Calls
	handleHTTPRequest(mux, "/miner/start", srv.minerStartHandler)
	handleHTTPRequest(mux, "/miner/status", srv.minerStatusHandler)
	handleHTTPRequest(mux, "/miner/stop", srv.minerStopHandler)

	// Renter API Calls
	handleHTTPRequest(mux, "/renter/download", srv.renterDownloadHandler)
	handleHTTPRequest(mux, "/renter/downloadqueue", srv.renterDownloadqueueHandler)
	handleHTTPRequest(mux, "/renter/files", srv.renterFilesHandler)
	handleHTTPRequest(mux, "/renter/status", srv.renterStatusHandler)
	handleHTTPRequest(mux, "/renter/upload", srv.renterUploadHandler)

	// TransactionPool API Calls
	handleHTTPRequest(mux, "/transactionpool/transactions", srv.transactionpoolTransactionsHandler)

	// Wallet API Calls
	handleHTTPRequest(mux, "/wallet/address", srv.walletAddressHandler)
	handleHTTPRequest(mux, "/wallet/send", srv.walletSendHandler)
	handleHTTPRequest(mux, "/wallet/status", srv.walletStatusHandler)

	// create graceful HTTP server
	srv.apiServer = &graceful.Server{
		Timeout: apiTimeout,
		Server:  &http.Server{Addr: addr, Handler: mux},
	}
}

// Serve listens for and handles API calls. It a blocking function.
func (srv *Server) Serve() error {
	// graceful will run until it catches a signal.
	// It can also be stopped manually by stopHandler.
	err := srv.apiServer.ListenAndServe()
	// despite its name, graceful still propogates this benign error
	if err != nil && strings.HasSuffix(err.Error(), "use of closed network connection") {
		err = nil
	}
	println("\rCaught stop signal, quitting.")
	return err
}

// writeError logs an writes an error to the API caller.
func writeError(w http.ResponseWriter, msg string, err int) {
	log.Printf("%d HTTP ERROR: %s", err, msg)
	http.Error(w, msg, err)
}

// writeJSON writes the object to the ResponseWriter. If the encoding fails, an
// error is written instead.
func writeJSON(w http.ResponseWriter, obj interface{}) {
	if json.NewEncoder(w).Encode(obj) != nil {
		http.Error(w, "Failed to encode response", http.StatusInternalServerError)
	}
}

// writeSuccess writes the success json object ({"Success":true}) to the
// ResponseWriter
func writeSuccess(w http.ResponseWriter) {
	writeJSON(w, struct{ Success bool }{true})
}
<|endoftext|>


<commit_before>package apk

type Instrumentation struct {
	Name            string `xml:"name,attr"`
	Target          string `xml:"targetPackage,attr"`
	HandleProfiling bool   `xml:"handleProfiling,attr"`
	FunctionalTest  bool   `xml:"functionalTest,attr"`
}

type Application struct {
	AllowTaskReparenting  bool   `xml:"allowTaskReparenting,attr"`
	AllowBackup           bool   `xml:"allowBackup,attr"`
	BackupAgent           string `xml:"backupAgent,attr"`
	Debuggable            bool   `xml:"debuggable,attr"`
	Description           string `xml:"description,attr"`
	Enabled               bool   `xml:"enabled,attr"`
	HasCode               bool   `xml:"hasCode,attr"`
	HardwareAccelerated   bool   `xml:"hardwareAccelerated,attr"`
	Icon                  string `xml:"icon,attr"`
	KillAfterRestore      bool   `xml:"killAfterRestore,attr"`
	LargeHeap             bool   `xml:"largeHeap,attr"`
	Label                 string `xml:"label,attr"`
	Logo                  int    `xml:"logo,attr"`
	ManageSpaceActivity   string `xml:"manageSpaceActivity,attr"`
	Name                  string `xml:"name,attr"`
	Permission            string `xml:"permission,attr"`
	Persistent            bool   `xml:"persistent,attr"`
	Process               string `xml:"process,attr"`
	RestoreAnyVersion     bool   `xml:"restoreAnyVersion,attr"`
	RequiredAccountType   string `xml:"requiredAccountType,attr"`
	RestrictedAccountType string `xml:"restrictedAccountType,attr"`
	SupportsRtl           bool   `xml:"supportsRtl,attr"`
	TaskAffinity          string `xml:"taskAffinity,attr"`
	TestOnly              bool   `xml:"testOnly,attr"`
	Theme                 int    `xml:"theme,attr"`
	UiOptions             string `xml:"uiOptions,attr"`
	VmSafeMode            bool   `xml:"vmSafeMode,attr"`
}

type Manifest struct {
	Package     string          `xml:"package,attr"`
	VersionCode int             `xml:"versionCode,attr"`
	VersionName string          `xml:"versionName,attr"`
	App         Application     `xml:"application"`
	Instrument  Instrumentation `xml:"instrumentation"`
}
<commit_msg>Added Uses Sdk<commit_after>package apk

import "github.com/wmbest2/android/adb"

type Instrumentation struct {
	Name            string `xml:"name,attr"`
	Target          string `xml:"targetPackage,attr"`
	HandleProfiling bool   `xml:"handleProfiling,attr"`
	FunctionalTest  bool   `xml:"functionalTest,attr"`
}

type Application struct {
	AllowTaskReparenting  bool   `xml:"allowTaskReparenting,attr"`
	AllowBackup           bool   `xml:"allowBackup,attr"`
	BackupAgent           string `xml:"backupAgent,attr"`
	Debuggable            bool   `xml:"debuggable,attr"`
	Description           string `xml:"description,attr"`
	Enabled               bool   `xml:"enabled,attr"`
	HasCode               bool   `xml:"hasCode,attr"`
	HardwareAccelerated   bool   `xml:"hardwareAccelerated,attr"`
	Icon                  string `xml:"icon,attr"`
	KillAfterRestore      bool   `xml:"killAfterRestore,attr"`
	LargeHeap             bool   `xml:"largeHeap,attr"`
	Label                 string `xml:"label,attr"`
	Logo                  int    `xml:"logo,attr"`
	ManageSpaceActivity   string `xml:"manageSpaceActivity,attr"`
	Name                  string `xml:"name,attr"`
	Permission            string `xml:"permission,attr"`
	Persistent            bool   `xml:"persistent,attr"`
	Process               string `xml:"process,attr"`
	RestoreAnyVersion     bool   `xml:"restoreAnyVersion,attr"`
	RequiredAccountType   string `xml:"requiredAccountType,attr"`
	RestrictedAccountType string `xml:"restrictedAccountType,attr"`
	SupportsRtl           bool   `xml:"supportsRtl,attr"`
	TaskAffinity          string `xml:"taskAffinity,attr"`
	TestOnly              bool   `xml:"testOnly,attr"`
	Theme                 int    `xml:"theme,attr"`
	UiOptions             string `xml:"uiOptions,attr"`
	VmSafeMode            bool   `xml:"vmSafeMode,attr"`
}

type UsesSdk struct {
	Min    adb.SdkVersion `xml:"minSdkVersion,attr"`
	Target adb.SdkVersion `xml:"targetSdkVersion,attr"`
	Max    adb.SdkVersion `xml:"maxSdkVersion,attr"`
}

type Manifest struct {
	Package     string          `xml:"package,attr"`
	VersionCode int             `xml:"versionCode,attr"`
	VersionName string          `xml:"versionName,attr"`
	App         Application     `xml:"application"`
	Instrument  Instrumentation `xml:"instrumentation"`
	Sdk         UsesSdk         `xml:"uses-sdk"`
}
<|endoftext|>


<commit_before>package app

import (
	"errors"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"unicode"

	"github.com/PuerkitoBio/goquery"
	"github.com/gorilla/context"
	"github.com/gorilla/mux"
	"github.com/gorilla/sessions"
	"github.com/justinas/alice"
	"github.com/kyleterry/sufr/config"
	"github.com/kyleterry/sufr/db"
)

var (
	router   = mux.NewRouter()
	store    = sessions.NewCookieStore([]byte("I gotta glock in my rari")) // TODO(kt): generate secret key instead of using Fetty Wap lyrics
	database *db.SufrDB
)

const (
	VisibilityPrivate = "private"
	VisibilityPublic  = "public"
	SUFRUserAgent     = "Linux:SUFR:v1.0 (by /u/found_dead)"
)

type templatecontext int

const TemplateContext templatecontext = 0

type errorHandler func(http.ResponseWriter, *http.Request) error

func (fn errorHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	err := fn(w, r)
	if err != nil {
		log.Printf("Got error while processing the request: %s\n", err)
		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
	}
}

// Sufr is the main application struct. It also implements http.Handler so it can
// be passed directly into ListenAndServe
type Sufr struct {
}

// New created a new pointer to Sufr
func New() *Sufr {
	log.Println("Creating new Sufr instance")
	app := &Sufr{}

	database = db.New(config.DatabaseFile)
	if config.Debug {
		go database.Statsdumper()
	}
	err := database.Open()
	// Panic if we can't open the database
	if err != nil {
		log.Panic(err)
	}

	// This route is used to initially configure the instance
	router.Handle("/config", errorHandler(registrationHandler)).Methods("POST", "GET").Name("config")
	router.Handle("/login", errorHandler(loginHandler)).Methods("POST", "GET").Name("login")
	router.Handle("/logout", errorHandler(logoutHandler)).Methods("POST", "GET").Name("logout")

	all := alice.New(SetSettingsHandler, SetLoggedInHandler, SetActiveTabHandler, LoggingHandler)
	auth := alice.New(AuthHandler)
	auth = auth.Extend(all)

	router.Handle("/", all.Then(errorHandler(urlIndexHandler))).Name("url-index")
	router.Handle("/url/new", auth.Then(errorHandler(urlNewHandler))).Name("url-new")
	router.Handle("/url/submit", auth.Then(errorHandler(urlSubmitHandler))).Methods("POST").Name("url-submit")
	router.Handle("/url/{id:[0-9]+}", all.Then(errorHandler(urlViewHandler))).Name("url-view")
	router.Handle("/url/{id:[0-9]+}/edit", auth.Then(errorHandler(urlEditHandler))).Name("url-edit")
	router.Handle("/url/{id:[0-9]+}/save", auth.Then(errorHandler(urlSaveHandler))).Methods("POST").Name("url-save")
	router.Handle("/url/{id:[0-9]+}/delete", auth.Then(errorHandler(urlDeleteHandler))).Name("url-delete")
	router.Handle("/url/{id:[0-9]+}/toggle-fav", auth.Then(errorHandler(urlFavHandler))).Methods("POST").Name("url-fav-toggle")
	router.Handle("/tag", all.Then(errorHandler(tagIndexHandler))).Name("tag-index")
	router.Handle("/tag/{id:[0-9]+}", all.Then(errorHandler(tagViewHandler))).Name("tag-view")
	router.Handle("/import/shitbucket", auth.Then(errorHandler(shitbucketImportHandler))).Methods("POST", "GET").Name("shitbucket-import")
	router.Handle("/settings", auth.Then(errorHandler(settingsHandler))).Methods("POST", "GET").Name("settings")
	router.Handle("/database-backup", auth.Then(errorHandler(database.BackupHandler))).Methods("GET").Name("database-backup")
	router.PathPrefix("/static").Handler(staticHandler)

	router.NotFoundHandler = errorHandler(func(w http.ResponseWriter, r *http.Request) error {
		w.WriteHeader(http.StatusNotFound)
		return renderTemplate(w, "404", nil)
	})

	return app
}

func (s Sufr) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	ctx := make(map[string]interface{})
	flashes := make(map[string][]interface{})
	session, err := store.Get(r, "flashes")
	if err != nil {
		log.Println(err)
	}
	flashes["danger"] = session.Flashes("danger")
	flashes["success"] = session.Flashes("success")
	flashes["warning"] = session.Flashes("warning")
	ctx["Flashes"] = flashes
	session.Save(r, w)

	context.Set(r, TemplateContext, ctx)

	// Have we configured?
	if !applicationConfigured() {
		if r.RequestURI != "/config" &&
			!strings.HasPrefix(r.RequestURI, "/static") {
			http.Redirect(w, r, reverse("config"), http.StatusSeeOther)
			return
		}
		router.ServeHTTP(w, r)
		return
	}

	// Is it a private only instance?
	if r.RequestURI != "/login" && instancePrivate() && !loggedIn(r) && !strings.HasPrefix(r.RequestURI, "/static") {
		http.Redirect(w, r, reverse("login"), http.StatusSeeOther)
		return
	}

	router.ServeHTTP(w, r)
}

// reverse Uses gorilla mux to give us a uri path by name. This is attached to template Funcs
func reverse(name string, params ...interface{}) string {
	s := make([]string, len(params))
	for _, param := range params {
		s = append(s, fmt.Sprint(param))
	}
	url, err := router.GetRoute(name).URL(s...)
	if err != nil {
		log.Println(err)
	}
	return url.Path
}

// Returns the page title or an error. If there is an error, the url is returned as well.
func getPageTitle(url string) (string, error) {
	client := &http.Client{}
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return url, err
	}

	req.Header.Set("User-Agent", SUFRUserAgent)

	res, err := client.Do(req)
	if err != nil {
		return url, err
	}

	defer res.Body.Close()

	doc, err := goquery.NewDocumentFromReader(res.Body)

	if err != nil {
		return url, err
	}

	title := doc.Find("title").Text()
	return title, nil
}

func parseTags(tagsstring string) []string {
	return strings.FieldsFunc(tagsstring, func(c rune) bool {
		return !unicode.IsLetter(c) && !unicode.IsNumber(c) && !unicode.IsPunct(c)
	})
}

func parseTagsMap(tagsstring string) map[string]struct{} {
	var m = make(map[string]struct{})
	for _, t := range parseTags(tagsstring) {
		m[t] = struct{}{}
	}
	return m
}

func ui64toa(v uint64) string {
	return strconv.FormatUint(v, 10)
}

func applicationConfigured() bool {
	settings, err := database.Get(uint64(1), config.BucketNameRoot)
	if err != nil {
		panic(err)
	}
	if settings != nil {
		return true
	}
	return false
}

func instancePrivate() bool {
	settingsbytes, err := database.Get(uint64(1), config.BucketNameRoot)
	if err != nil {
		panic(err)
	}
	settings := DeserializeSettings(settingsbytes)
	if settings.Visibility == VisibilityPrivate {
		return true
	}
	return false
}

func loggedIn(r *http.Request) bool {
	session, err := store.Get(r, "auth")
	if err != nil {
		panic(err)
	}

	val := session.Values["userID"]
	userID, ok := val.(uint64)

	if !ok || userID <= 0 {
		return false
	}

	user, err := database.Get(userID, config.BucketNameUser)
	if err != nil {
		panic(err)
	}

	if user == nil {
		return false
	}

	return true
}

func isYoutube(url string) bool {
	return strings.Contains(url, "youtube.com/watch")
}

func youtubevid(video string) string {
	u, _ := url.Parse(video)
	return u.Query()["v"][0]
}

func newcontext(values ...interface{}) (map[string]interface{}, error) {
	if len(values)%2 != 0 {
		return nil, errors.New("invalid dict call")
	}
	dict := make(map[string]interface{}, len(values)/2)
	for i := 0; i < len(values); i += 2 {
		key, ok := values[i].(string)
		if !ok {
			return nil, errors.New("dict keys must be strings")
		}
		dict[key] = values[i+1]
	}
	return dict, nil
}
<commit_msg>Removing reddit user from user-agent<commit_after>package app

import (
	"errors"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"unicode"

	"github.com/PuerkitoBio/goquery"
	"github.com/gorilla/context"
	"github.com/gorilla/mux"
	"github.com/gorilla/sessions"
	"github.com/justinas/alice"
	"github.com/kyleterry/sufr/config"
	"github.com/kyleterry/sufr/db"
)

var (
	router   = mux.NewRouter()
	store    = sessions.NewCookieStore([]byte("I gotta glock in my rari")) // TODO(kt): generate secret key instead of using Fetty Wap lyrics
	database *db.SufrDB
)

const (
	VisibilityPrivate = "private"
	VisibilityPublic  = "public"
	SUFRUserAgent     = "Linux:SUFR:v1.0"
)

type templatecontext int

const TemplateContext templatecontext = 0

type errorHandler func(http.ResponseWriter, *http.Request) error

func (fn errorHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	err := fn(w, r)
	if err != nil {
		log.Printf("Got error while processing the request: %s\n", err)
		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
	}
}

// Sufr is the main application struct. It also implements http.Handler so it can
// be passed directly into ListenAndServe
type Sufr struct {
}

// New created a new pointer to Sufr
func New() *Sufr {
	log.Println("Creating new Sufr instance")
	app := &Sufr{}

	database = db.New(config.DatabaseFile)
	if config.Debug {
		go database.Statsdumper()
	}
	err := database.Open()
	// Panic if we can't open the database
	if err != nil {
		log.Panic(err)
	}

	// This route is used to initially configure the instance
	router.Handle("/config", errorHandler(registrationHandler)).Methods("POST", "GET").Name("config")
	router.Handle("/login", errorHandler(loginHandler)).Methods("POST", "GET").Name("login")
	router.Handle("/logout", errorHandler(logoutHandler)).Methods("POST", "GET").Name("logout")

	all := alice.New(SetSettingsHandler, SetLoggedInHandler, SetActiveTabHandler, LoggingHandler)
	auth := alice.New(AuthHandler)
	auth = auth.Extend(all)

	router.Handle("/", all.Then(errorHandler(urlIndexHandler))).Name("url-index")
	router.Handle("/url/new", auth.Then(errorHandler(urlNewHandler))).Name("url-new")
	router.Handle("/url/submit", auth.Then(errorHandler(urlSubmitHandler))).Methods("POST").Name("url-submit")
	router.Handle("/url/{id:[0-9]+}", all.Then(errorHandler(urlViewHandler))).Name("url-view")
	router.Handle("/url/{id:[0-9]+}/edit", auth.Then(errorHandler(urlEditHandler))).Name("url-edit")
	router.Handle("/url/{id:[0-9]+}/save", auth.Then(errorHandler(urlSaveHandler))).Methods("POST").Name("url-save")
	router.Handle("/url/{id:[0-9]+}/delete", auth.Then(errorHandler(urlDeleteHandler))).Name("url-delete")
	router.Handle("/url/{id:[0-9]+}/toggle-fav", auth.Then(errorHandler(urlFavHandler))).Methods("POST").Name("url-fav-toggle")
	router.Handle("/tag", all.Then(errorHandler(tagIndexHandler))).Name("tag-index")
	router.Handle("/tag/{id:[0-9]+}", all.Then(errorHandler(tagViewHandler))).Name("tag-view")
	router.Handle("/import/shitbucket", auth.Then(errorHandler(shitbucketImportHandler))).Methods("POST", "GET").Name("shitbucket-import")
	router.Handle("/settings", auth.Then(errorHandler(settingsHandler))).Methods("POST", "GET").Name("settings")
	router.Handle("/database-backup", auth.Then(errorHandler(database.BackupHandler))).Methods("GET").Name("database-backup")
	router.PathPrefix("/static").Handler(staticHandler)

	router.NotFoundHandler = errorHandler(func(w http.ResponseWriter, r *http.Request) error {
		w.WriteHeader(http.StatusNotFound)
		return renderTemplate(w, "404", nil)
	})

	return app
}

func (s Sufr) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	ctx := make(map[string]interface{})
	flashes := make(map[string][]interface{})
	session, err := store.Get(r, "flashes")
	if err != nil {
		log.Println(err)
	}
	flashes["danger"] = session.Flashes("danger")
	flashes["success"] = session.Flashes("success")
	flashes["warning"] = session.Flashes("warning")
	ctx["Flashes"] = flashes
	session.Save(r, w)

	context.Set(r, TemplateContext, ctx)

	// Have we configured?
	if !applicationConfigured() {
		if r.RequestURI != "/config" &&
			!strings.HasPrefix(r.RequestURI, "/static") {
			http.Redirect(w, r, reverse("config"), http.StatusSeeOther)
			return
		}
		router.ServeHTTP(w, r)
		return
	}

	// Is it a private only instance?
	if r.RequestURI != "/login" && instancePrivate() && !loggedIn(r) && !strings.HasPrefix(r.RequestURI, "/static") {
		http.Redirect(w, r, reverse("login"), http.StatusSeeOther)
		return
	}

	router.ServeHTTP(w, r)
}

// reverse Uses gorilla mux to give us a uri path by name. This is attached to template Funcs
func reverse(name string, params ...interface{}) string {
	s := make([]string, len(params))
	for _, param := range params {
		s = append(s, fmt.Sprint(param))
	}
	url, err := router.GetRoute(name).URL(s...)
	if err != nil {
		log.Println(err)
	}
	return url.Path
}

// Returns the page title or an error. If there is an error, the url is returned as well.
func getPageTitle(url string) (string, error) {
	client := &http.Client{}
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return url, err
	}

	req.Header.Set("User-Agent", SUFRUserAgent)

	res, err := client.Do(req)
	if err != nil {
		return url, err
	}

	defer res.Body.Close()

	doc, err := goquery.NewDocumentFromReader(res.Body)

	if err != nil {
		return url, err
	}

	title := doc.Find("title").Text()
	return title, nil
}

func parseTags(tagsstring string) []string {
	return strings.FieldsFunc(tagsstring, func(c rune) bool {
		return !unicode.IsLetter(c) && !unicode.IsNumber(c) && !unicode.IsPunct(c)
	})
}

func parseTagsMap(tagsstring string) map[string]struct{} {
	var m = make(map[string]struct{})
	for _, t := range parseTags(tagsstring) {
		m[t] = struct{}{}
	}
	return m
}

func ui64toa(v uint64) string {
	return strconv.FormatUint(v, 10)
}

func applicationConfigured() bool {
	settings, err := database.Get(uint64(1), config.BucketNameRoot)
	if err != nil {
		panic(err)
	}
	if settings != nil {
		return true
	}
	return false
}

func instancePrivate() bool {
	settingsbytes, err := database.Get(uint64(1), config.BucketNameRoot)
	if err != nil {
		panic(err)
	}
	settings := DeserializeSettings(settingsbytes)
	if settings.Visibility == VisibilityPrivate {
		return true
	}
	return false
}

func loggedIn(r *http.Request) bool {
	session, err := store.Get(r, "auth")
	if err != nil {
		panic(err)
	}

	val := session.Values["userID"]
	userID, ok := val.(uint64)

	if !ok || userID <= 0 {
		return false
	}

	user, err := database.Get(userID, config.BucketNameUser)
	if err != nil {
		panic(err)
	}

	if user == nil {
		return false
	}

	return true
}

func isYoutube(url string) bool {
	return strings.Contains(url, "youtube.com/watch")
}

func youtubevid(video string) string {
	u, _ := url.Parse(video)
	return u.Query()["v"][0]
}

func newcontext(values ...interface{}) (map[string]interface{}, error) {
	if len(values)%2 != 0 {
		return nil, errors.New("invalid dict call")
	}
	dict := make(map[string]interface{}, len(values)/2)
	for i := 0; i < len(values); i += 2 {
		key, ok := values[i].(string)
		if !ok {
			return nil, errors.New("dict keys must be strings")
		}
		dict[key] = values[i+1]
	}
	return dict, nil
}
<|endoftext|>


<commit_before>// Package app is the "runtime" for the ps-top / ps-stats application packages
//
// This file contains the library routines related to running the app.
package app

import (
	"database/sql"
	"fmt"
	"log"
	"os"
	"os/signal"
	"syscall"
	"time"

	"github.com/sjmudd/anonymiser"
	"github.com/sjmudd/ps-top/connector"
	"github.com/sjmudd/ps-top/context"
	"github.com/sjmudd/ps-top/display"
	"github.com/sjmudd/ps-top/event"
	"github.com/sjmudd/ps-top/global"
	fsbi "github.com/sjmudd/ps-top/file_io_latency"
	"github.com/sjmudd/ps-top/lib"
	"github.com/sjmudd/ps-top/logger"
	"github.com/sjmudd/ps-top/memory_usage"
	ewsgben "github.com/sjmudd/ps-top/mutex_latency"
	"github.com/sjmudd/ps-top/p_s/ps_table"
	"github.com/sjmudd/ps-top/setup_instruments"
	essgben "github.com/sjmudd/ps-top/stages_latency"
	tiwsbt "github.com/sjmudd/ps-top/table_io_latency"
	tlwsbt "github.com/sjmudd/ps-top/table_lock_latency"
	"github.com/sjmudd/ps-top/user_latency"
	"github.com/sjmudd/ps-top/view"
	"github.com/sjmudd/ps-top/wait_info"
)

const (
	hostname = "HOSTNAME"
	uptime   = "UPTIME"
	version  = "VERSION"
)

// Flags for initialising the app
type Flags struct {
	Anonymise bool
	Conn      *connector.Connector
	Interval  int
	Count     int
	Stdout    bool
	View      string
	Disp      display.Display
}

// App holds the data needed by an application
type App struct {
	ctx                *context.Context
	count              int
	display            display.Display
	done               chan struct{}
	sigChan            chan os.Signal
	wi                 wait_info.WaitInfo
	finished           bool
	stdout             bool
	dbh                *sql.DB
	help               bool
	fsbi               ps_table.Tabler // ufsbi.File_summary_by_instance
	tiwsbt             tiwsbt.Object
	tlwsbt             ps_table.Tabler // tlwsbt.Table_lock_waits_summary_by_table
	ewsgben            ps_table.Tabler // ewsgben.Events_waits_summary_global_by_event_name
	essgben            ps_table.Tabler // essgben.Events_stages_summary_global_by_event_name
	memory             memory_usage.Object
	users              user_latency.Object
	view               view.View
	wait_info.WaitInfo // embedded
	setupInstruments   setup_instruments.SetupInstruments
	wantRelativeStats  bool
}

// Ignore any errors. Perhaps not ideal but makes the code easier to read
// further down.
func selectGlobalVariableByVariableName(dbh *sql.DB, name string) string {
	result, err := global.SelectVariableByName(dbh, name)
	if err != nil {
		logger.Fatal("selectGlobalVariableByName(" + name + ") failed:", err)
	}
	return result
}

// Ignore any errors. Perhaps not ideal but makes the code easier to read
// further down.
func selectGlobalStatusByVariableName(dbh *sql.DB, name string) int {
	result, err := global.SelectStatusByName(dbh, name)
	if err != nil {
		logger.Fatal("selectGlobalStatusByName(" + name + ") failed:", err)
	}
	return result
}

// ensure performance_schema is enabled
// - if not will not return and will exit
func ensurePerformanceSchemaEnabled(dbh *sql.DB, variables map[string]string) {
	if dbh == nil {
		log.Fatal("ensurePerformanceSchemaEnabled() dbh is nil")
	}
	if variables == nil {
		log.Fatal("ensurePerformanceSchemaEnabled() variables is nil")
	}

	// check that performance_schema = ON
	if value, ok := variables["performance_schema"]; ok {
		if value != "ON" {
			log.Fatal(fmt.Sprintf("ensurePerformanceSchemaEnabled(): performance_schema = '%s'. Please configure performance_schema = 1 in /etc/my.cnf (or equivalent) and restart mysqld to use %s.",
				value, lib.MyName()))
		} else {
			logger.Println("performance_schema = ON check succeeds")
		}
	} else {
		log.Fatal("ensurePerformanceSchemaEnabled() performance_schema variable not found!")
	}
}

// NewApp sets up the application given various parameters.
func NewApp(flags Flags) *App {
	logger.Println("app.NewApp()")
	app := new(App)

	anonymiser.Enable(flags.Anonymise) // not dynamic at the moment
	app.ctx = new(context.Context)
	app.count = flags.Count
	app.dbh = flags.Conn.Handle()
	app.finished = false

	// Prior to setting up screen check that performance_schema is enabled.
	// On MariaDB this is not the default setting so it will confuse people.
	variables, _ := global.SelectAllVariables(app.dbh)
	ensurePerformanceSchemaEnabled(app.dbh, variables)

	app.stdout = flags.Stdout
	app.display = flags.Disp
	app.display.SetContext(app.ctx)
	app.SetHelp(false)

	if err := view.ValidateViews(app.dbh); err != nil {
		log.Fatal(err)
	}

	logger.Println("app.Setup() Setting the default view to:", flags.View)
	app.view.SetByName(flags.View) // if empty will use the default

	app.setupInstruments = setup_instruments.NewSetupInstruments(app.dbh)
	app.setupInstruments.EnableMonitoring()

	app.wi.SetWaitInterval(time.Second * time.Duration(flags.Interval))

	// setup to their initial types/values
	app.fsbi = fsbi.NewFileSummaryByInstance(variables)
	app.tlwsbt = new(tlwsbt.Object)
	app.ewsgben = new(ewsgben.Object)
	app.essgben = new(essgben.Object)

	app.SetWantRelativeStats(true)
	app.fixLatencySetting() // adjust to see ops/latency

	app.resetDBStatistics()

	app.ctx.SetHostname(anonymiser.Anonymise("host", selectGlobalVariableByVariableName(app.dbh, hostname)))
	app.ctx.SetMySQLVersion(selectGlobalVariableByVariableName(app.dbh, version))

	return app
}

// Finished tells us if we have finished
func (app App) Finished() bool {
	return app.finished
}

// CollectAll collects all the stats together in one go
func (app *App) collectAll() {
	app.fsbi.Collect(app.dbh)
	app.tlwsbt.Collect(app.dbh)
	app.tiwsbt.Collect(app.dbh)
	app.users.Collect(app.dbh)
	app.essgben.Collect(app.dbh)
	app.ewsgben.Collect(app.dbh)
	app.memory.Collect(app.dbh)
}

func (app *App) WantRelativeStats() bool {
	return app.wantRelativeStats
}

// do a fresh collection of data and then update the initial values based on that.
func (app *App) resetDBStatistics() {
	app.collectAll()
	app.setInitialFromCurrent()
}

func (app *App) setInitialFromCurrent() {
	start := time.Now()
	app.fsbi.SetInitialFromCurrent()
	app.tlwsbt.SetInitialFromCurrent()
	app.tiwsbt.SetInitialFromCurrent()
	app.essgben.SetInitialFromCurrent()
	app.ewsgben.SetInitialFromCurrent()
	app.memory.SetInitialFromCurrent()
	logger.Println("app.setInitialFromCurrent() took", time.Duration(time.Since(start)).String())
}

// Collect the data we are looking at.
func (app *App) Collect() {
	start := time.Now()

	switch app.view.Get() {
	case view.ViewLatency, view.ViewOps:
		app.tiwsbt.Collect(app.dbh)
	case view.ViewIO:
		app.fsbi.Collect(app.dbh)
	case view.ViewLocks:
		app.tlwsbt.Collect(app.dbh)
	case view.ViewUsers:
		app.users.Collect(app.dbh)
	case view.ViewMutex:
		app.ewsgben.Collect(app.dbh)
	case view.ViewStages:
		app.essgben.Collect(app.dbh)
	case view.ViewMemory:
		app.memory.Collect(app.dbh)
	}
	app.wi.CollectedNow()
	logger.Println("app.Collect() took", time.Duration(time.Since(start)).String())
}

// SetHelp determines if we need to display help
func (app *App) SetHelp(newHelp bool) {
	app.help = newHelp

	app.display.ClearScreen()
}

// Help returns the internal help variable
func (app App) Help() bool {
	return app.help
}

// Display shows the output appropriate to the corresponding view and device
func (app *App) Display() {
	if app.help {
		app.display.DisplayHelp() // shouldn't get here if in --stdout mode
	} else {
		app.ctx.SetUptime(selectGlobalStatusByVariableName(app.dbh, uptime))

		switch app.view.Get() {
		case view.ViewLatency, view.ViewOps:
			app.display.Display(app.tiwsbt)
		case view.ViewIO:
			app.display.Display(app.fsbi)
		case view.ViewLocks:
			app.display.Display(app.tlwsbt)
		case view.ViewUsers:
			app.display.Display(app.users)
		case view.ViewMutex:
			app.display.Display(app.ewsgben)
		case view.ViewStages:
			app.display.Display(app.essgben)
		case view.ViewMemory:
			app.display.Display(app.memory)
		}
	}
}

// fixLatencySetting() ensures the SetWantsLatency() value is
// correct. This needs to be done more cleanly.
func (app *App) fixLatencySetting() {
	if app.view.Get() == view.ViewLatency {
		app.tiwsbt.SetWantsLatency(true)
	}
	if app.view.Get() == view.ViewOps {
		app.tiwsbt.SetWantsLatency(false)
	}
}

// change to the previous display mode
func (app *App) displayPrevious() {
	app.view.SetPrev()
	app.fixLatencySetting()
	app.display.ClearScreen()
	app.Display()
}

// change to the next display mode
func (app *App) displayNext() {
	app.view.SetNext()
	app.fixLatencySetting()
	app.display.ClearScreen()
	app.Display()
}

// SetWantRelativeStats sets whether we want to see data that's relative or absolute
func (app *App) SetWantRelativeStats(want bool) {
	app.wantRelativeStats = want

	app.fsbi.SetWantRelativeStats(want)
	app.tlwsbt.SetWantRelativeStats(want)
	app.tiwsbt.SetWantRelativeStats(want)
	app.users.SetWantRelativeStats(want) // ignored
	app.essgben.SetWantRelativeStats(want)
	app.ewsgben.SetWantRelativeStats(want) // ignored
	app.memory.SetWantRelativeStats(want)
}

// Cleanup prepares  the application prior to shutting down
func (app *App) Cleanup() {
	app.display.Close()
	if app.dbh != nil {
		app.setupInstruments.RestoreConfiguration()
		_ = app.dbh.Close()
	}
	logger.Println("App.Cleanup completed")
}

// Run runs the application in a loop until we're ready to finish
func (app *App) Run() {
	logger.Println("app.Run()")

	app.sigChan = make(chan os.Signal, 10) // 10 entries
	signal.Notify(app.sigChan, syscall.SIGINT, syscall.SIGTERM)

	eventChan := app.display.EventChan()

	for !app.Finished() {
		select {
		case sig := <-app.sigChan:
			fmt.Println("Caught signal: ", sig)
			app.finished = true
		case <-app.wi.WaitNextPeriod():
			app.Collect()
			app.Display()
			if app.stdout {
				app.setInitialFromCurrent()
			}
		case inputEvent := <-eventChan:
			switch inputEvent.Type {
			case event.EventAnonymise:
				anonymiser.Enable(!anonymiser.Enabled()) // toggle current behaviour
			case event.EventFinished:
				app.finished = true
			case event.EventViewNext:
				app.displayNext()
			case event.EventViewPrev:
				app.displayPrevious()
			case event.EventDecreasePollTime:
				if app.wi.WaitInterval() > time.Second {
					app.wi.SetWaitInterval(app.wi.WaitInterval() - time.Second)
				}
			case event.EventIncreasePollTime:
				app.wi.SetWaitInterval(app.wi.WaitInterval() + time.Second)
			case event.EventHelp:
				app.SetHelp(!app.Help())
			case event.EventToggleWantRelative:
				app.SetWantRelativeStats(!app.WantRelativeStats())
				app.Display()
			case event.EventResetStatistics:
				app.resetDBStatistics()
				app.Display()
			case event.EventResizeScreen:
				width, height := inputEvent.Width, inputEvent.Height
				app.display.Resize(width, height)
				app.Display()
			case event.EventError:
				log.Fatalf("Quitting because of EventError error")
			}
		}
		// provide a hook to stop the application if the counter goes down to zero
		if app.stdout && app.count > 0 {
			app.count--
			if app.count == 0 {
				app.finished = true
			}
		}
	}
}
<commit_msg>Check return from global.SelectAllVariables()<commit_after>// Package app is the "runtime" for the ps-top / ps-stats application packages
//
// This file contains the library routines related to running the app.
package app

import (
	"database/sql"
	"fmt"
	"log"
	"os"
	"os/signal"
	"syscall"
	"time"

	"github.com/sjmudd/anonymiser"
	"github.com/sjmudd/ps-top/connector"
	"github.com/sjmudd/ps-top/context"
	"github.com/sjmudd/ps-top/display"
	"github.com/sjmudd/ps-top/event"
	fsbi "github.com/sjmudd/ps-top/file_io_latency"
	"github.com/sjmudd/ps-top/global"
	"github.com/sjmudd/ps-top/lib"
	"github.com/sjmudd/ps-top/logger"
	"github.com/sjmudd/ps-top/memory_usage"
	ewsgben "github.com/sjmudd/ps-top/mutex_latency"
	"github.com/sjmudd/ps-top/p_s/ps_table"
	"github.com/sjmudd/ps-top/setup_instruments"
	essgben "github.com/sjmudd/ps-top/stages_latency"
	tiwsbt "github.com/sjmudd/ps-top/table_io_latency"
	tlwsbt "github.com/sjmudd/ps-top/table_lock_latency"
	"github.com/sjmudd/ps-top/user_latency"
	"github.com/sjmudd/ps-top/view"
	"github.com/sjmudd/ps-top/wait_info"
)

const (
	hostname = "HOSTNAME"
	uptime   = "UPTIME"
	version  = "VERSION"
)

// Flags for initialising the app
type Flags struct {
	Anonymise bool
	Conn      *connector.Connector
	Interval  int
	Count     int
	Stdout    bool
	View      string
	Disp      display.Display
}

// App holds the data needed by an application
type App struct {
	ctx                *context.Context
	count              int
	display            display.Display
	done               chan struct{}
	sigChan            chan os.Signal
	wi                 wait_info.WaitInfo
	finished           bool
	stdout             bool
	dbh                *sql.DB
	help               bool
	fsbi               ps_table.Tabler // ufsbi.File_summary_by_instance
	tiwsbt             tiwsbt.Object
	tlwsbt             ps_table.Tabler // tlwsbt.Table_lock_waits_summary_by_table
	ewsgben            ps_table.Tabler // ewsgben.Events_waits_summary_global_by_event_name
	essgben            ps_table.Tabler // essgben.Events_stages_summary_global_by_event_name
	memory             memory_usage.Object
	users              user_latency.Object
	view               view.View
	wait_info.WaitInfo // embedded
	setupInstruments   setup_instruments.SetupInstruments
	wantRelativeStats  bool
}

// Ignore any errors. Perhaps not ideal but makes the code easier to read
// further down.
func selectGlobalVariableByVariableName(dbh *sql.DB, name string) string {
	result, err := global.SelectVariableByName(dbh, name)
	if err != nil {
		logger.Fatal("selectGlobalVariableByName("+name+") failed:", err)
	}
	return result
}

// Ignore any errors. Perhaps not ideal but makes the code easier to read
// further down.
func selectGlobalStatusByVariableName(dbh *sql.DB, name string) int {
	result, err := global.SelectStatusByName(dbh, name)
	if err != nil {
		logger.Fatal("selectGlobalStatusByName("+name+") failed:", err)
	}
	return result
}

// ensure performance_schema is enabled
// - if not will not return and will exit
func ensurePerformanceSchemaEnabled(dbh *sql.DB, variables map[string]string) {
	if dbh == nil {
		log.Fatal("ensurePerformanceSchemaEnabled() dbh is nil")
	}
	if variables == nil {
		log.Fatal("ensurePerformanceSchemaEnabled() variables is nil")
	}

	// check that performance_schema = ON
	if value, ok := variables["performance_schema"]; ok {
		if value != "ON" {
			log.Fatal(fmt.Sprintf("ensurePerformanceSchemaEnabled(): performance_schema = '%s'. Please configure performance_schema = 1 in /etc/my.cnf (or equivalent) and restart mysqld to use %s.",
				value, lib.MyName()))
		} else {
			logger.Println("performance_schema = ON check succeeds")
		}
	} else {
		log.Fatal("ensurePerformanceSchemaEnabled() performance_schema variable not found!")
	}
}

// NewApp sets up the application given various parameters.
func NewApp(flags Flags) *App {
	logger.Println("app.NewApp()")
	app := new(App)

	anonymiser.Enable(flags.Anonymise) // not dynamic at the moment
	app.ctx = new(context.Context)
	app.count = flags.Count
	app.dbh = flags.Conn.Handle()
	app.finished = false

	// Prior to setting up screen check that performance_schema is enabled.
	// On MariaDB this is not the default setting so it will confuse people.
	variables, err := global.SelectAllVariables(app.dbh)
	if err != nil {
		log.Fatal("Fatal error collecting global variables", err)
	}
	ensurePerformanceSchemaEnabled(app.dbh, variables)

	app.stdout = flags.Stdout
	app.display = flags.Disp
	app.display.SetContext(app.ctx)
	app.SetHelp(false)

	if err := view.ValidateViews(app.dbh); err != nil {
		log.Fatal(err)
	}

	logger.Println("app.Setup() Setting the default view to:", flags.View)
	app.view.SetByName(flags.View) // if empty will use the default

	app.setupInstruments = setup_instruments.NewSetupInstruments(app.dbh)
	app.setupInstruments.EnableMonitoring()

	app.wi.SetWaitInterval(time.Second * time.Duration(flags.Interval))

	// setup to their initial types/values
	app.fsbi = fsbi.NewFileSummaryByInstance(variables)
	app.tlwsbt = new(tlwsbt.Object)
	app.ewsgben = new(ewsgben.Object)
	app.essgben = new(essgben.Object)

	app.SetWantRelativeStats(true)
	app.fixLatencySetting() // adjust to see ops/latency

	app.resetDBStatistics()

	app.ctx.SetHostname(anonymiser.Anonymise("host", selectGlobalVariableByVariableName(app.dbh, hostname)))
	app.ctx.SetMySQLVersion(selectGlobalVariableByVariableName(app.dbh, version))

	return app
}

// Finished tells us if we have finished
func (app App) Finished() bool {
	return app.finished
}

// CollectAll collects all the stats together in one go
func (app *App) collectAll() {
	app.fsbi.Collect(app.dbh)
	app.tlwsbt.Collect(app.dbh)
	app.tiwsbt.Collect(app.dbh)
	app.users.Collect(app.dbh)
	app.essgben.Collect(app.dbh)
	app.ewsgben.Collect(app.dbh)
	app.memory.Collect(app.dbh)
}

func (app *App) WantRelativeStats() bool {
	return app.wantRelativeStats
}

// do a fresh collection of data and then update the initial values based on that.
func (app *App) resetDBStatistics() {
	app.collectAll()
	app.setInitialFromCurrent()
}

func (app *App) setInitialFromCurrent() {
	start := time.Now()
	app.fsbi.SetInitialFromCurrent()
	app.tlwsbt.SetInitialFromCurrent()
	app.tiwsbt.SetInitialFromCurrent()
	app.essgben.SetInitialFromCurrent()
	app.ewsgben.SetInitialFromCurrent()
	app.memory.SetInitialFromCurrent()
	logger.Println("app.setInitialFromCurrent() took", time.Duration(time.Since(start)).String())
}

// Collect the data we are looking at.
func (app *App) Collect() {
	start := time.Now()

	switch app.view.Get() {
	case view.ViewLatency, view.ViewOps:
		app.tiwsbt.Collect(app.dbh)
	case view.ViewIO:
		app.fsbi.Collect(app.dbh)
	case view.ViewLocks:
		app.tlwsbt.Collect(app.dbh)
	case view.ViewUsers:
		app.users.Collect(app.dbh)
	case view.ViewMutex:
		app.ewsgben.Collect(app.dbh)
	case view.ViewStages:
		app.essgben.Collect(app.dbh)
	case view.ViewMemory:
		app.memory.Collect(app.dbh)
	}
	app.wi.CollectedNow()
	logger.Println("app.Collect() took", time.Duration(time.Since(start)).String())
}

// SetHelp determines if we need to display help
func (app *App) SetHelp(newHelp bool) {
	app.help = newHelp

	app.display.ClearScreen()
}

// Help returns the internal help variable
func (app App) Help() bool {
	return app.help
}

// Display shows the output appropriate to the corresponding view and device
func (app *App) Display() {
	if app.help {
		app.display.DisplayHelp() // shouldn't get here if in --stdout mode
	} else {
		app.ctx.SetUptime(selectGlobalStatusByVariableName(app.dbh, uptime))

		switch app.view.Get() {
		case view.ViewLatency, view.ViewOps:
			app.display.Display(app.tiwsbt)
		case view.ViewIO:
			app.display.Display(app.fsbi)
		case view.ViewLocks:
			app.display.Display(app.tlwsbt)
		case view.ViewUsers:
			app.display.Display(app.users)
		case view.ViewMutex:
			app.display.Display(app.ewsgben)
		case view.ViewStages:
			app.display.Display(app.essgben)
		case view.ViewMemory:
			app.display.Display(app.memory)
		}
	}
}

// fixLatencySetting() ensures the SetWantsLatency() value is
// correct. This needs to be done more cleanly.
func (app *App) fixLatencySetting() {
	if app.view.Get() == view.ViewLatency {
		app.tiwsbt.SetWantsLatency(true)
	}
	if app.view.Get() == view.ViewOps {
		app.tiwsbt.SetWantsLatency(false)
	}
}

// change to the previous display mode
func (app *App) displayPrevious() {
	app.view.SetPrev()
	app.fixLatencySetting()
	app.display.ClearScreen()
	app.Display()
}

// change to the next display mode
func (app *App) displayNext() {
	app.view.SetNext()
	app.fixLatencySetting()
	app.display.ClearScreen()
	app.Display()
}

// SetWantRelativeStats sets whether we want to see data that's relative or absolute
func (app *App) SetWantRelativeStats(want bool) {
	app.wantRelativeStats = want

	app.fsbi.SetWantRelativeStats(want)
	app.tlwsbt.SetWantRelativeStats(want)
	app.tiwsbt.SetWantRelativeStats(want)
	app.users.SetWantRelativeStats(want) // ignored
	app.essgben.SetWantRelativeStats(want)
	app.ewsgben.SetWantRelativeStats(want) // ignored
	app.memory.SetWantRelativeStats(want)
}

// Cleanup prepares  the application prior to shutting down
func (app *App) Cleanup() {
	app.display.Close()
	if app.dbh != nil {
		app.setupInstruments.RestoreConfiguration()
		_ = app.dbh.Close()
	}
	logger.Println("App.Cleanup completed")
}

// Run runs the application in a loop until we're ready to finish
func (app *App) Run() {
	logger.Println("app.Run()")

	app.sigChan = make(chan os.Signal, 10) // 10 entries
	signal.Notify(app.sigChan, syscall.SIGINT, syscall.SIGTERM)

	eventChan := app.display.EventChan()

	for !app.Finished() {
		select {
		case sig := <-app.sigChan:
			fmt.Println("Caught signal: ", sig)
			app.finished = true
		case <-app.wi.WaitNextPeriod():
			app.Collect()
			app.Display()
			if app.stdout {
				app.setInitialFromCurrent()
			}
		case inputEvent := <-eventChan:
			switch inputEvent.Type {
			case event.EventAnonymise:
				anonymiser.Enable(!anonymiser.Enabled()) // toggle current behaviour
			case event.EventFinished:
				app.finished = true
			case event.EventViewNext:
				app.displayNext()
			case event.EventViewPrev:
				app.displayPrevious()
			case event.EventDecreasePollTime:
				if app.wi.WaitInterval() > time.Second {
					app.wi.SetWaitInterval(app.wi.WaitInterval() - time.Second)
				}
			case event.EventIncreasePollTime:
				app.wi.SetWaitInterval(app.wi.WaitInterval() + time.Second)
			case event.EventHelp:
				app.SetHelp(!app.Help())
			case event.EventToggleWantRelative:
				app.SetWantRelativeStats(!app.WantRelativeStats())
				app.Display()
			case event.EventResetStatistics:
				app.resetDBStatistics()
				app.Display()
			case event.EventResizeScreen:
				width, height := inputEvent.Width, inputEvent.Height
				app.display.Resize(width, height)
				app.Display()
			case event.EventError:
				log.Fatalf("Quitting because of EventError error")
			}
		}
		// provide a hook to stop the application if the counter goes down to zero
		if app.stdout && app.count > 0 {
			app.count--
			if app.count == 0 {
				app.finished = true
			}
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package app

import (
	"fmt"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/pkg/errors"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/tsuru/tsuru/api/shutdown"
	"github.com/tsuru/tsuru/db"
	"github.com/tsuru/tsuru/log"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

var (
	bulkMaxWaitMongoTime = 1 * time.Second
	bulkMaxNumberMsgs    = 1000
	bulkQueueMaxSize     = 10000

	buckets = append([]float64{0.1, 0.5}, prometheus.ExponentialBuckets(1, 1.6, 15)...)

	logsInQueue = prometheus.NewGauge(prometheus.GaugeOpts{
		Name: "tsuru_logs_queue_current",
		Help: "The current number of log entries in dispatcher queue.",
	})

	logsInAppQueues = prometheus.NewGauge(prometheus.GaugeOpts{
		Name: "tsuru_logs_app_queues_current",
		Help: "The current number of log entries in app queues.",
	})

	logsQueueBlockedTotal = prometheus.NewCounter(prometheus.CounterOpts{
		Name: "tsuru_logs_queue_blocked_seconds_total",
		Help: "The total time spent blocked trying to add log to queue.",
	})

	logsQueueSize = prometheus.NewGauge(prometheus.GaugeOpts{
		Name: "tsuru_logs_dispatcher_queue_size",
		Help: "The max number of log entries in a dispatcher queue.",
	})

	logsEnqueued = prometheus.NewCounter(prometheus.CounterOpts{
		Name: "tsuru_logs_enqueued_total",
		Help: "The number of log entries enqueued for processing.",
	})

	logsWritten = prometheus.NewCounter(prometheus.CounterOpts{
		Name: "tsuru_logs_write_total",
		Help: "The number of log entries written to mongo.",
	})

	logsDropped = prometheus.NewCounter(prometheus.CounterOpts{
		Name: "tsuru_logs_dropped_total",
		Help: "The number of log entries dropped due to full buffers.",
	})

	logsMongoFullLatency = prometheus.NewHistogram(prometheus.HistogramOpts{
		Name:    "tsuru_logs_mongo_full_duration_seconds",
		Help:    "The latency distributions for log messages to be stored in database.",
		Buckets: buckets,
	})

	logsMongoLatency = prometheus.NewHistogram(prometheus.HistogramOpts{
		Name:    "tsuru_logs_mongo_duration_seconds",
		Help:    "The latency distributions for log messages to be stored in database.",
		Buckets: buckets,
	})
)

func init() {
	prometheus.MustRegister(logsInQueue)
	prometheus.MustRegister(logsInAppQueues)
	prometheus.MustRegister(logsQueueSize)
	prometheus.MustRegister(logsEnqueued)
	prometheus.MustRegister(logsWritten)
	prometheus.MustRegister(logsDropped)
	prometheus.MustRegister(logsQueueBlockedTotal)
	prometheus.MustRegister(logsMongoFullLatency)
	prometheus.MustRegister(logsMongoLatency)
}

type LogListener struct {
	c       <-chan Applog
	logConn *db.LogStorage
	quit    chan struct{}
}

func isCappedPositionLost(err error) bool {
	if err == nil {
		return false
	}
	return strings.Contains(err.Error(), "CappedPositionLost")
}

func isSessionClosed(r interface{}) bool {
	return fmt.Sprintf("%v", r) == "Session already closed"
}

func NewLogListener(a *App, filterLog Applog) (*LogListener, error) {
	conn, err := db.LogConn()
	if err != nil {
		return nil, err
	}
	c := make(chan Applog, 10)
	quit := make(chan struct{})
	coll := conn.Logs(a.Name)
	var lastLog Applog
	err = coll.Find(nil).Sort("-_id").Limit(1).One(&lastLog)
	if err == mgo.ErrNotFound {
		// Tail cursors do not work correctly if the collection is empty (the
		// Next() call wouldn't block). So if the collection is empty we insert
		// the very first log line in it. This is quite rare in the real world
		// though so the impact of this extra log message is really small.
		err = a.Log("Logs initialization", "tsuru", "")
		if err != nil {
			return nil, err
		}
		err = coll.Find(nil).Sort("-_id").Limit(1).One(&lastLog)
	}
	if err != nil {
		return nil, err
	}
	lastId := lastLog.MongoID
	mkQuery := func() bson.M {
		m := bson.M{
			"_id": bson.M{"$gt": lastId},
		}
		if filterLog.Source != "" {
			m["source"] = filterLog.Source
		}
		if filterLog.Unit != "" {
			m["unit"] = filterLog.Unit
		}
		return m
	}
	query := coll.Find(mkQuery())
	tailTimeout := 10 * time.Second
	iter := query.Sort("$natural").Tail(tailTimeout)
	go func() {
		defer close(c)
		defer func() {
			if r := recover(); r != nil {
				if isSessionClosed(r) {
					return
				}
				panic(err)
			}
		}()
		for {
			var applog Applog
			for iter.Next(&applog) {
				lastId = applog.MongoID
				select {
				case c <- applog:
				case <-quit:
					iter.Close()
					return
				}
			}
			if iter.Timeout() {
				continue
			}
			if err := iter.Err(); err != nil {
				if !isCappedPositionLost(err) {
					log.Errorf("error tailing logs: %v", err)
					iter.Close()
					return
				}
			}
			iter.Close()
			query = coll.Find(mkQuery())
			iter = query.Sort("$natural").Tail(tailTimeout)
		}
	}()
	l := LogListener{c: c, logConn: conn, quit: quit}
	return &l, nil
}

func (l *LogListener) ListenChan() <-chan Applog {
	return l.c
}

func (l *LogListener) Close() {
	l.logConn.Close()
	if l.quit != nil {
		close(l.quit)
		l.quit = nil
	}
	return
}

type LogDispatcher struct {
	mu             sync.RWMutex
	dispatchers    map[string]*appLogDispatcher
	msgCh          chan *msgWithTS
	shuttingDown   int32
	doneProcessing chan struct{}
}

type msgWithTS struct {
	msg        *Applog
	arriveTime time.Time
}

func NewlogDispatcher(chanSize int) *LogDispatcher {
	d := &LogDispatcher{
		dispatchers:    make(map[string]*appLogDispatcher),
		msgCh:          make(chan *msgWithTS, chanSize),
		doneProcessing: make(chan struct{}),
	}
	go d.runWriter()
	shutdown.Register(d)
	logsQueueSize.Set(float64(chanSize))
	return d
}

func (d *LogDispatcher) getMessageDispatcher(msg *Applog) *appLogDispatcher {
	appName := msg.AppName
	d.mu.RLock()
	appD, ok := d.dispatchers[appName]
	if !ok {
		d.mu.RUnlock()
		d.mu.Lock()
		appD, ok = d.dispatchers[appName]
		if !ok {
			appD = newAppLogDispatcher(appName)
			d.dispatchers[appName] = appD
		}
		d.mu.Unlock()
	} else {
		d.mu.RUnlock()
	}
	return appD
}

func (d *LogDispatcher) runWriter() {
	defer close(d.doneProcessing)
	for msgExtra := range d.msgCh {
		if msgExtra == nil {
			break
		}
		logsInQueue.Dec()
		appD := d.getMessageDispatcher(msgExtra.msg)
		appD.send(msgExtra)
	}
}

func (d *LogDispatcher) Send(msg *Applog) error {
	if atomic.LoadInt32(&d.shuttingDown) == 1 {
		return errors.New("log dispatcher is shutting down")
	}
	logsInQueue.Inc()
	logsEnqueued.Inc()
	msgExtra := &msgWithTS{msg: msg, arriveTime: time.Now()}
	select {
	case d.msgCh <- msgExtra:
	default:
		t0 := time.Now()
		d.msgCh <- msgExtra
		logsQueueBlockedTotal.Add(time.Since(t0).Seconds())
	}
	return nil
}

func (a *LogDispatcher) String() string {
	return "log dispatcher"
}

func (d *LogDispatcher) Shutdown() {
	atomic.StoreInt32(&d.shuttingDown, 1)
	d.msgCh <- nil
	<-d.doneProcessing
	logsInQueue.Set(0)
	for _, appD := range d.dispatchers {
		appD.stopWait()
	}
}

type appLogDispatcher struct {
	appName string
	*bulkProcessor
}

func newAppLogDispatcher(appName string) *appLogDispatcher {
	d := &appLogDispatcher{
		bulkProcessor: initBulkProcessor(bulkMaxWaitMongoTime, bulkMaxNumberMsgs),
		appName:       appName,
	}
	d.flushable = d
	go d.run()
	return d
}

func (d *appLogDispatcher) flush(msgs []interface{}, lastMessage *msgWithTS) bool {
	conn, err := db.LogConn()
	if err != nil {
		log.Errorf("[log flusher] unable to connect to mongodb: %s", err)
		return false
	}
	coll := conn.Logs(d.appName)
	err = coll.Insert(msgs...)
	coll.Close()
	if err != nil {
		log.Errorf("[log flusher] unable to insert logs: %s", err)
		return false
	}
	if lastMessage != nil {
		logsMongoLatency.Observe(time.Since(lastMessage.arriveTime).Seconds())
		logsMongoFullLatency.Observe(time.Since(lastMessage.msg.Date).Seconds())
	}
	logsWritten.Add(float64(len(msgs)))
	return true
}

type bulkProcessor struct {
	maxWaitTime time.Duration
	bulkSize    int
	finished    chan struct{}
	ch          chan *msgWithTS
	nextNotify  *time.Timer
	flushable   interface {
		flush([]interface{}, *msgWithTS) bool
	}
}

func initBulkProcessor(maxWait time.Duration, bulkSize int) *bulkProcessor {
	return &bulkProcessor{
		maxWaitTime: maxWait,
		bulkSize:    bulkSize,
		finished:    make(chan struct{}),
		ch:          make(chan *msgWithTS, bulkQueueMaxSize),
		nextNotify:  time.NewTimer(0),
	}
}

func (p *bulkProcessor) send(msg *msgWithTS) {
	select {
	case p.ch <- msg:
		logsInAppQueues.Set(float64(len(p.ch)))
	default:
		logsDropped.Inc()
		select {
		case <-p.nextNotify.C:
			log.Errorf("dropping log messages to mongodb due to full channel buffer. app: %q, len: %d", msg.msg.AppName, len(p.ch))
			p.nextNotify.Reset(time.Minute)
		default:
		}
	}
}

func (p *bulkProcessor) stopWait() {
	p.ch <- nil
	<-p.finished
}

func (p *bulkProcessor) run() {
	defer close(p.finished)
	t := time.NewTimer(p.maxWaitTime)
	pos := 0
	bulkBuffer := make([]interface{}, p.bulkSize)
	shouldReturn := false
	var lastMessage *msgWithTS
	for {
		var flush bool
		select {
		case msgExtra := <-p.ch:
			logsInAppQueues.Set(float64(len(p.ch)))
			if msgExtra == nil {
				flush = true
				shouldReturn = true
				break
			}
			if pos == p.bulkSize {
				flush = true
				break
			}
			lastMessage = msgExtra
			bulkBuffer[pos] = msgExtra.msg
			pos++
			flush = p.bulkSize == pos
		case <-t.C:
			flush = pos > 0
			t.Reset(p.maxWaitTime)
		}
		if flush {
			if p.flushable.flush(bulkBuffer[:pos], lastMessage) {
				lastMessage = nil
				pos = 0
			}
		}
		if shouldReturn {
			return
		}
	}
}
<commit_msg>app: prevent empty log flush calls<commit_after>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package app

import (
	"fmt"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/pkg/errors"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/tsuru/tsuru/api/shutdown"
	"github.com/tsuru/tsuru/db"
	"github.com/tsuru/tsuru/log"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

var (
	bulkMaxWaitMongoTime = 1 * time.Second
	bulkMaxNumberMsgs    = 1000
	bulkQueueMaxSize     = 10000

	buckets = append([]float64{0.1, 0.5}, prometheus.ExponentialBuckets(1, 1.6, 15)...)

	logsInQueue = prometheus.NewGauge(prometheus.GaugeOpts{
		Name: "tsuru_logs_queue_current",
		Help: "The current number of log entries in dispatcher queue.",
	})

	logsInAppQueues = prometheus.NewGauge(prometheus.GaugeOpts{
		Name: "tsuru_logs_app_queues_current",
		Help: "The current number of log entries in app queues.",
	})

	logsQueueBlockedTotal = prometheus.NewCounter(prometheus.CounterOpts{
		Name: "tsuru_logs_queue_blocked_seconds_total",
		Help: "The total time spent blocked trying to add log to queue.",
	})

	logsQueueSize = prometheus.NewGauge(prometheus.GaugeOpts{
		Name: "tsuru_logs_dispatcher_queue_size",
		Help: "The max number of log entries in a dispatcher queue.",
	})

	logsEnqueued = prometheus.NewCounter(prometheus.CounterOpts{
		Name: "tsuru_logs_enqueued_total",
		Help: "The number of log entries enqueued for processing.",
	})

	logsWritten = prometheus.NewCounter(prometheus.CounterOpts{
		Name: "tsuru_logs_write_total",
		Help: "The number of log entries written to mongo.",
	})

	logsDropped = prometheus.NewCounter(prometheus.CounterOpts{
		Name: "tsuru_logs_dropped_total",
		Help: "The number of log entries dropped due to full buffers.",
	})

	logsMongoFullLatency = prometheus.NewHistogram(prometheus.HistogramOpts{
		Name:    "tsuru_logs_mongo_full_duration_seconds",
		Help:    "The latency distributions for log messages to be stored in database.",
		Buckets: buckets,
	})

	logsMongoLatency = prometheus.NewHistogram(prometheus.HistogramOpts{
		Name:    "tsuru_logs_mongo_duration_seconds",
		Help:    "The latency distributions for log messages to be stored in database.",
		Buckets: buckets,
	})
)

func init() {
	prometheus.MustRegister(logsInQueue)
	prometheus.MustRegister(logsInAppQueues)
	prometheus.MustRegister(logsQueueSize)
	prometheus.MustRegister(logsEnqueued)
	prometheus.MustRegister(logsWritten)
	prometheus.MustRegister(logsDropped)
	prometheus.MustRegister(logsQueueBlockedTotal)
	prometheus.MustRegister(logsMongoFullLatency)
	prometheus.MustRegister(logsMongoLatency)
}

type LogListener struct {
	c       <-chan Applog
	logConn *db.LogStorage
	quit    chan struct{}
}

func isCappedPositionLost(err error) bool {
	if err == nil {
		return false
	}
	return strings.Contains(err.Error(), "CappedPositionLost")
}

func isSessionClosed(r interface{}) bool {
	return fmt.Sprintf("%v", r) == "Session already closed"
}

func NewLogListener(a *App, filterLog Applog) (*LogListener, error) {
	conn, err := db.LogConn()
	if err != nil {
		return nil, err
	}
	c := make(chan Applog, 10)
	quit := make(chan struct{})
	coll := conn.Logs(a.Name)
	var lastLog Applog
	err = coll.Find(nil).Sort("-_id").Limit(1).One(&lastLog)
	if err == mgo.ErrNotFound {
		// Tail cursors do not work correctly if the collection is empty (the
		// Next() call wouldn't block). So if the collection is empty we insert
		// the very first log line in it. This is quite rare in the real world
		// though so the impact of this extra log message is really small.
		err = a.Log("Logs initialization", "tsuru", "")
		if err != nil {
			return nil, err
		}
		err = coll.Find(nil).Sort("-_id").Limit(1).One(&lastLog)
	}
	if err != nil {
		return nil, err
	}
	lastId := lastLog.MongoID
	mkQuery := func() bson.M {
		m := bson.M{
			"_id": bson.M{"$gt": lastId},
		}
		if filterLog.Source != "" {
			m["source"] = filterLog.Source
		}
		if filterLog.Unit != "" {
			m["unit"] = filterLog.Unit
		}
		return m
	}
	query := coll.Find(mkQuery())
	tailTimeout := 10 * time.Second
	iter := query.Sort("$natural").Tail(tailTimeout)
	go func() {
		defer close(c)
		defer func() {
			if r := recover(); r != nil {
				if isSessionClosed(r) {
					return
				}
				panic(err)
			}
		}()
		for {
			var applog Applog
			for iter.Next(&applog) {
				lastId = applog.MongoID
				select {
				case c <- applog:
				case <-quit:
					iter.Close()
					return
				}
			}
			if iter.Timeout() {
				continue
			}
			if err := iter.Err(); err != nil {
				if !isCappedPositionLost(err) {
					log.Errorf("error tailing logs: %v", err)
					iter.Close()
					return
				}
			}
			iter.Close()
			query = coll.Find(mkQuery())
			iter = query.Sort("$natural").Tail(tailTimeout)
		}
	}()
	l := LogListener{c: c, logConn: conn, quit: quit}
	return &l, nil
}

func (l *LogListener) ListenChan() <-chan Applog {
	return l.c
}

func (l *LogListener) Close() {
	l.logConn.Close()
	if l.quit != nil {
		close(l.quit)
		l.quit = nil
	}
	return
}

type LogDispatcher struct {
	mu             sync.RWMutex
	dispatchers    map[string]*appLogDispatcher
	msgCh          chan *msgWithTS
	shuttingDown   int32
	doneProcessing chan struct{}
}

type msgWithTS struct {
	msg        *Applog
	arriveTime time.Time
}

func NewlogDispatcher(chanSize int) *LogDispatcher {
	d := &LogDispatcher{
		dispatchers:    make(map[string]*appLogDispatcher),
		msgCh:          make(chan *msgWithTS, chanSize),
		doneProcessing: make(chan struct{}),
	}
	go d.runWriter()
	shutdown.Register(d)
	logsQueueSize.Set(float64(chanSize))
	return d
}

func (d *LogDispatcher) getMessageDispatcher(msg *Applog) *appLogDispatcher {
	appName := msg.AppName
	d.mu.RLock()
	appD, ok := d.dispatchers[appName]
	if !ok {
		d.mu.RUnlock()
		d.mu.Lock()
		appD, ok = d.dispatchers[appName]
		if !ok {
			appD = newAppLogDispatcher(appName)
			d.dispatchers[appName] = appD
		}
		d.mu.Unlock()
	} else {
		d.mu.RUnlock()
	}
	return appD
}

func (d *LogDispatcher) runWriter() {
	defer close(d.doneProcessing)
	for msgExtra := range d.msgCh {
		if msgExtra == nil {
			break
		}
		logsInQueue.Dec()
		appD := d.getMessageDispatcher(msgExtra.msg)
		appD.send(msgExtra)
	}
}

func (d *LogDispatcher) Send(msg *Applog) error {
	if atomic.LoadInt32(&d.shuttingDown) == 1 {
		return errors.New("log dispatcher is shutting down")
	}
	logsInQueue.Inc()
	logsEnqueued.Inc()
	msgExtra := &msgWithTS{msg: msg, arriveTime: time.Now()}
	select {
	case d.msgCh <- msgExtra:
	default:
		t0 := time.Now()
		d.msgCh <- msgExtra
		logsQueueBlockedTotal.Add(time.Since(t0).Seconds())
	}
	return nil
}

func (a *LogDispatcher) String() string {
	return "log dispatcher"
}

func (d *LogDispatcher) Shutdown() {
	atomic.StoreInt32(&d.shuttingDown, 1)
	d.msgCh <- nil
	<-d.doneProcessing
	logsInQueue.Set(0)
	for _, appD := range d.dispatchers {
		appD.stopWait()
	}
}

type appLogDispatcher struct {
	appName string
	*bulkProcessor
}

func newAppLogDispatcher(appName string) *appLogDispatcher {
	d := &appLogDispatcher{
		bulkProcessor: initBulkProcessor(bulkMaxWaitMongoTime, bulkMaxNumberMsgs),
		appName:       appName,
	}
	d.flushable = d
	go d.run()
	return d
}

func (d *appLogDispatcher) flush(msgs []interface{}, lastMessage *msgWithTS) bool {
	conn, err := db.LogConn()
	if err != nil {
		log.Errorf("[log flusher] unable to connect to mongodb: %s", err)
		return false
	}
	coll := conn.Logs(d.appName)
	err = coll.Insert(msgs...)
	coll.Close()
	if err != nil {
		log.Errorf("[log flusher] unable to insert logs: %s", err)
		return false
	}
	if lastMessage != nil {
		logsMongoLatency.Observe(time.Since(lastMessage.arriveTime).Seconds())
		logsMongoFullLatency.Observe(time.Since(lastMessage.msg.Date).Seconds())
	}
	logsWritten.Add(float64(len(msgs)))
	return true
}

type bulkProcessor struct {
	maxWaitTime time.Duration
	bulkSize    int
	finished    chan struct{}
	ch          chan *msgWithTS
	nextNotify  *time.Timer
	flushable   interface {
		flush([]interface{}, *msgWithTS) bool
	}
}

func initBulkProcessor(maxWait time.Duration, bulkSize int) *bulkProcessor {
	return &bulkProcessor{
		maxWaitTime: maxWait,
		bulkSize:    bulkSize,
		finished:    make(chan struct{}),
		ch:          make(chan *msgWithTS, bulkQueueMaxSize),
		nextNotify:  time.NewTimer(0),
	}
}

func (p *bulkProcessor) send(msg *msgWithTS) {
	select {
	case p.ch <- msg:
		logsInAppQueues.Set(float64(len(p.ch)))
	default:
		logsDropped.Inc()
		select {
		case <-p.nextNotify.C:
			log.Errorf("dropping log messages to mongodb due to full channel buffer. app: %q, len: %d", msg.msg.AppName, len(p.ch))
			p.nextNotify.Reset(time.Minute)
		default:
		}
	}
}

func (p *bulkProcessor) stopWait() {
	p.ch <- nil
	<-p.finished
}

func (p *bulkProcessor) run() {
	defer close(p.finished)
	t := time.NewTimer(p.maxWaitTime)
	pos := 0
	bulkBuffer := make([]interface{}, p.bulkSize)
	shouldReturn := false
	var lastMessage *msgWithTS
	for {
		var flush bool
		select {
		case msgExtra := <-p.ch:
			logsInAppQueues.Set(float64(len(p.ch)))
			if msgExtra == nil {
				flush = true
				shouldReturn = true
				break
			}
			if pos == p.bulkSize {
				flush = true
				break
			}
			lastMessage = msgExtra
			bulkBuffer[pos] = msgExtra.msg
			pos++
			flush = p.bulkSize == pos
		case <-t.C:
			flush = true
			t.Reset(p.maxWaitTime)
		}
		if flush && pos > 0 {
			if p.flushable.flush(bulkBuffer[:pos], lastMessage) {
				lastMessage = nil
				pos = 0
			}
		}
		if shouldReturn {
			return
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the LGPLv3, see LICENCE file for details.

package apt

import (
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"regexp"
	"strings"
	"syscall"
	"time"

	"github.com/juju/loggo"

	"github.com/juju/utils"
	"github.com/juju/utils/proxy"
)

var (
	logger  = loggo.GetLogger("juju.utils.apt")
	proxyRE = regexp.MustCompile(`(?im)^\s*Acquire::(?P<protocol>[a-z]+)::Proxy\s+"(?P<proxy>[^"]+)";\s*$`)

	// ConfFile is the full file path for the proxy settings that are
	// written by cloud-init and the machine environ worker.
	ConfFile = "/etc/apt/apt.conf.d/42-juju-proxy-settings"
)

// Some helpful functions for running apt in a sane way

// CommandOutput calls cmd.Output, this is used as an overloading point so we
// can test what *would* be run without actually executing another program
var CommandOutput = (*exec.Cmd).CombinedOutput

// getCommand is the default apt-get command used in cloud-init, the various settings
// mean that apt won't actually block waiting for a prompt from the user.
var getCommand = []string{
	"apt-get", "--option=Dpkg::Options::=--force-confold",
	"--option=Dpkg::options::=--force-unsafe-io", "--assume-yes", "--quiet",
}

// getEnvOptions are options we need to pass to apt-get to not have it prompt
// the user
var getEnvOptions = []string{"DEBIAN_FRONTEND=noninteractive"}

// cloudArchivePackages maintaines a list of packages that GetPreparePackages
// should reference when determining the --target-release for a given series.
// http://reqorts.qa.ubuntu.com/reports/ubuntu-server/cloud-archive/cloud-tools_versions.html
var cloudArchivePackages = map[string]bool{
	"cloud-image-utils":       true,
	"cloud-utils":             true,
	"curtin":                  true,
	"djorm-ext-pgarray":       true,
	"golang":                  true,
	"iproute2":                true,
	"isc-dhcp":                true,
	"juju-core":               true,
	"libseccomp":              true,
	"libv8-3.14":              true,
	"lxc":                     true,
	"maas":                    true,
	"mongodb":                 true,
	"mongodb-server":          true,
	"python-django":           true,
	"python-django-piston":    true,
	"python-jujuclient":       true,
	"python-tx-tftp":          true,
	"python-websocket-client": true,
	"raphael 2.1.0-1ubuntu1":  true,
	"simplestreams":           true,
	"txlongpoll":              true,
	"uvtool":                  true,
	"yui3":                    true,
}

// targetRelease returns a string base on the current series
// that is suitable for use with the apt-get --target-release option
func targetRelease(series string) string {
	switch series {
	case "precise":
		return "precise-updates/cloud-tools"
	default:
		return ""
	}
}

// GetPreparePackages returns a slice of installCommands. Each item
// in the slice is suitable for passing directly to Apt.
// It inspects the series passed to it
// and properly generates an installCommand entry with a --target-release
// should the series be an LTS release with cloud archive packages.
func GetPreparePackages(packages []string, series string) [][]string {
	var installCommands [][]string
	if target := targetRelease(series); target == "" {
		return append(installCommands, packages)
	} else {
		var pkgs []string
		pkgs_with_target := []string{"--target-release", target}
		for _, pkg := range packages {
			if cloudArchivePackages[pkg] {
				pkgs_with_target = append(pkgs_with_target, pkg)
			} else {
				pkgs = append(pkgs, pkg)
			}
		}

		// We check for >2 here so that we only append pkgs_with_target
		// if there was an actual package in the slice.
		if len(pkgs_with_target) > 2 {
			installCommands = append(installCommands, pkgs_with_target)
		}

		// Sometimes we may end up with all cloudArchivePackages
		// in that case we do not want to append an empty slice of pkgs
		if len(pkgs) > 0 {
			installCommands = append(installCommands, pkgs)
		}

		return installCommands
	}
}

// GetInstall runs 'apt-get install packages' for the packages listed
// here. apt-get install calls are retried for 30 times with a 1
// second sleep between attempts.
func GetInstall(packages ...string) error {
	cmdArgs := append([]string(nil), getCommand...)
	cmdArgs = append(cmdArgs, "install")
	cmdArgs = append(cmdArgs, packages...)
	logger.Infof("Running: %s", cmdArgs)
	cmd := exec.Command(cmdArgs[0], cmdArgs[1:]...)
	cmd.Env = append(os.Environ(), getEnvOptions...)

	var err error
	var out []byte
	// Retry APT operations for 30 times, sleeping 10 seconds
	// between attempts. This avoids failure in the case of
	// something else having the dpkg lock (e.g. a charm on the
	// machine we're deploying containers to).
	attempt := utils.AttemptStrategy{Delay: 10 * time.Second, Min: 30}
	for a := attempt.Start(); a.Next(); {
		out, err = CommandOutput(cmd)
		if err == nil {
			return nil
		}
		exitError, ok := err.(*exec.ExitError)
		if !ok {
			err = fmt.Errorf("unexpected error type %T", err)
			break
		}
		waitStatus, ok := exitError.ProcessState.Sys().(syscall.WaitStatus)
		if !ok {
			err = fmt.Errorf("unexpected process state type %T", exitError.ProcessState.Sys())
			break
		}
		// From apt-get(8) "apt-get returns zero on normal
		// operation, decimal 100 on error."
		if waitStatus.ExitStatus() != 100 {
			break
		}
	}
	if err != nil {
		logger.Errorf("apt-get command failed: %v\nargs: %#v\n%s",
			err, cmdArgs, string(out))
		return fmt.Errorf("apt-get failed: %v", err)
	}
	return nil
}

// ConfigProxy will consult apt-config about the configured proxy
// settings. If there are no proxy settings configured, an empty string is
// returned.
func ConfigProxy() (string, error) {
	cmdArgs := []string{
		"apt-config",
		"dump",
		"Acquire::http::Proxy",
		"Acquire::https::Proxy",
		"Acquire::ftp::Proxy",
	}
	cmd := exec.Command(cmdArgs[0], cmdArgs[1:]...)
	out, err := CommandOutput(cmd)
	if err != nil {
		logger.Errorf("apt-config command failed: %v\nargs: %#v\n%s",
			err, cmdArgs, string(out))
		return "", fmt.Errorf("apt-config failed: %v", err)
	}
	return string(bytes.Join(proxyRE.FindAll(out, -1), []byte("\n"))), nil
}

// DetectProxies will parse the results of ConfigProxy to return a
// ProxySettings instance.
func DetectProxies() (result proxy.Settings, err error) {
	output, err := ConfigProxy()
	if err != nil {
		return result, err
	}
	for _, match := range proxyRE.FindAllStringSubmatch(output, -1) {
		switch match[1] {
		case "http":
			result.Http = match[2]
		case "https":
			result.Https = match[2]
		case "ftp":
			result.Ftp = match[2]
		}
	}
	return result, nil
}

// ProxyContent produces the format expected by the apt config files
// from the ProxySettings struct.
func ProxyContent(proxySettings proxy.Settings) string {
	lines := []string{}
	addLine := func(proxySettings, value string) {
		if value != "" {
			lines = append(lines, fmt.Sprintf(
				"Acquire::%s::Proxy %q;", proxySettings, value))
		}
	}
	addLine("http", proxySettings.Http)
	addLine("https", proxySettings.Https)
	addLine("ftp", proxySettings.Ftp)
	return strings.Join(lines, "\n")
}

// IsPackageInstalled uses dpkg-query to determine if the `packageName`
// package is installed.
func IsPackageInstalled(packageName string) bool {
	_, err := utils.RunCommand("dpkg-query", "--status", packageName)
	return err == nil
}
<commit_msg>Off by 9 error (oops)<commit_after>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the LGPLv3, see LICENCE file for details.

package apt

import (
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"regexp"
	"strings"
	"syscall"
	"time"

	"github.com/juju/loggo"

	"github.com/juju/utils"
	"github.com/juju/utils/proxy"
)

var (
	logger  = loggo.GetLogger("juju.utils.apt")
	proxyRE = regexp.MustCompile(`(?im)^\s*Acquire::(?P<protocol>[a-z]+)::Proxy\s+"(?P<proxy>[^"]+)";\s*$`)

	// ConfFile is the full file path for the proxy settings that are
	// written by cloud-init and the machine environ worker.
	ConfFile = "/etc/apt/apt.conf.d/42-juju-proxy-settings"
)

// Some helpful functions for running apt in a sane way

// CommandOutput calls cmd.Output, this is used as an overloading point so we
// can test what *would* be run without actually executing another program
var CommandOutput = (*exec.Cmd).CombinedOutput

// getCommand is the default apt-get command used in cloud-init, the various settings
// mean that apt won't actually block waiting for a prompt from the user.
var getCommand = []string{
	"apt-get", "--option=Dpkg::Options::=--force-confold",
	"--option=Dpkg::options::=--force-unsafe-io", "--assume-yes", "--quiet",
}

// getEnvOptions are options we need to pass to apt-get to not have it prompt
// the user
var getEnvOptions = []string{"DEBIAN_FRONTEND=noninteractive"}

// cloudArchivePackages maintaines a list of packages that GetPreparePackages
// should reference when determining the --target-release for a given series.
// http://reqorts.qa.ubuntu.com/reports/ubuntu-server/cloud-archive/cloud-tools_versions.html
var cloudArchivePackages = map[string]bool{
	"cloud-image-utils":       true,
	"cloud-utils":             true,
	"curtin":                  true,
	"djorm-ext-pgarray":       true,
	"golang":                  true,
	"iproute2":                true,
	"isc-dhcp":                true,
	"juju-core":               true,
	"libseccomp":              true,
	"libv8-3.14":              true,
	"lxc":                     true,
	"maas":                    true,
	"mongodb":                 true,
	"mongodb-server":          true,
	"python-django":           true,
	"python-django-piston":    true,
	"python-jujuclient":       true,
	"python-tx-tftp":          true,
	"python-websocket-client": true,
	"raphael 2.1.0-1ubuntu1":  true,
	"simplestreams":           true,
	"txlongpoll":              true,
	"uvtool":                  true,
	"yui3":                    true,
}

// targetRelease returns a string base on the current series
// that is suitable for use with the apt-get --target-release option
func targetRelease(series string) string {
	switch series {
	case "precise":
		return "precise-updates/cloud-tools"
	default:
		return ""
	}
}

// GetPreparePackages returns a slice of installCommands. Each item
// in the slice is suitable for passing directly to Apt.
// It inspects the series passed to it
// and properly generates an installCommand entry with a --target-release
// should the series be an LTS release with cloud archive packages.
func GetPreparePackages(packages []string, series string) [][]string {
	var installCommands [][]string
	if target := targetRelease(series); target == "" {
		return append(installCommands, packages)
	} else {
		var pkgs []string
		pkgs_with_target := []string{"--target-release", target}
		for _, pkg := range packages {
			if cloudArchivePackages[pkg] {
				pkgs_with_target = append(pkgs_with_target, pkg)
			} else {
				pkgs = append(pkgs, pkg)
			}
		}

		// We check for >2 here so that we only append pkgs_with_target
		// if there was an actual package in the slice.
		if len(pkgs_with_target) > 2 {
			installCommands = append(installCommands, pkgs_with_target)
		}

		// Sometimes we may end up with all cloudArchivePackages
		// in that case we do not want to append an empty slice of pkgs
		if len(pkgs) > 0 {
			installCommands = append(installCommands, pkgs)
		}

		return installCommands
	}
}

// GetInstall runs 'apt-get install packages' for the packages listed
// here. apt-get install calls are retried for 30 times with a 10
// second sleep between attempts.
func GetInstall(packages ...string) error {
	cmdArgs := append([]string(nil), getCommand...)
	cmdArgs = append(cmdArgs, "install")
	cmdArgs = append(cmdArgs, packages...)
	logger.Infof("Running: %s", cmdArgs)
	cmd := exec.Command(cmdArgs[0], cmdArgs[1:]...)
	cmd.Env = append(os.Environ(), getEnvOptions...)

	var err error
	var out []byte
	// Retry APT operations for 30 times, sleeping 10 seconds
	// between attempts. This avoids failure in the case of
	// something else having the dpkg lock (e.g. a charm on the
	// machine we're deploying containers to).
	attempt := utils.AttemptStrategy{Delay: 10 * time.Second, Min: 30}
	for a := attempt.Start(); a.Next(); {
		out, err = CommandOutput(cmd)
		if err == nil {
			return nil
		}
		exitError, ok := err.(*exec.ExitError)
		if !ok {
			err = fmt.Errorf("unexpected error type %T", err)
			break
		}
		waitStatus, ok := exitError.ProcessState.Sys().(syscall.WaitStatus)
		if !ok {
			err = fmt.Errorf("unexpected process state type %T", exitError.ProcessState.Sys())
			break
		}
		// From apt-get(8) "apt-get returns zero on normal
		// operation, decimal 100 on error."
		if waitStatus.ExitStatus() != 100 {
			break
		}
	}
	if err != nil {
		logger.Errorf("apt-get command failed: %v\nargs: %#v\n%s",
			err, cmdArgs, string(out))
		return fmt.Errorf("apt-get failed: %v", err)
	}
	return nil
}

// ConfigProxy will consult apt-config about the configured proxy
// settings. If there are no proxy settings configured, an empty string is
// returned.
func ConfigProxy() (string, error) {
	cmdArgs := []string{
		"apt-config",
		"dump",
		"Acquire::http::Proxy",
		"Acquire::https::Proxy",
		"Acquire::ftp::Proxy",
	}
	cmd := exec.Command(cmdArgs[0], cmdArgs[1:]...)
	out, err := CommandOutput(cmd)
	if err != nil {
		logger.Errorf("apt-config command failed: %v\nargs: %#v\n%s",
			err, cmdArgs, string(out))
		return "", fmt.Errorf("apt-config failed: %v", err)
	}
	return string(bytes.Join(proxyRE.FindAll(out, -1), []byte("\n"))), nil
}

// DetectProxies will parse the results of ConfigProxy to return a
// ProxySettings instance.
func DetectProxies() (result proxy.Settings, err error) {
	output, err := ConfigProxy()
	if err != nil {
		return result, err
	}
	for _, match := range proxyRE.FindAllStringSubmatch(output, -1) {
		switch match[1] {
		case "http":
			result.Http = match[2]
		case "https":
			result.Https = match[2]
		case "ftp":
			result.Ftp = match[2]
		}
	}
	return result, nil
}

// ProxyContent produces the format expected by the apt config files
// from the ProxySettings struct.
func ProxyContent(proxySettings proxy.Settings) string {
	lines := []string{}
	addLine := func(proxySettings, value string) {
		if value != "" {
			lines = append(lines, fmt.Sprintf(
				"Acquire::%s::Proxy %q;", proxySettings, value))
		}
	}
	addLine("http", proxySettings.Http)
	addLine("https", proxySettings.Https)
	addLine("ftp", proxySettings.Ftp)
	return strings.Join(lines, "\n")
}

// IsPackageInstalled uses dpkg-query to determine if the `packageName`
// package is installed.
func IsPackageInstalled(packageName string) bool {
	_, err := utils.RunCommand("dpkg-query", "--status", packageName)
	return err == nil
}
<|endoftext|>


<commit_before>// Copyright 2010 The GoGo Authors. All rights reserved.
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

package main

//
// Set of recognized tokens
//
const TOKEN_IDENTIFIER uint64 = 1;  // Identifier
const TOKEN_STRING uint64 = 2;      // String using "..."
const TOKEN_EOS uint64 = 3;         // End of Scan
const TOKEN_LBRAC uint64 = 4;       // Left bracket '('
const TOKEN_RBRAC uint64 = 5;       // Right bracket ')'
const TOKEN_LSBRAC uint64 = 6;      // Left square bracket '['
const TOKEN_RSBRAC uint64 = 7;      // Right square bracket ']'
const TOKEN_INTEGER uint64 = 8;     // Integer number
const TOKEN_LCBRAC uint64 = 9;      // Left curly bracket '{'
const TOKEN_RCBRAC uint64 = 10;     // Right curly bracket '}'
const TOKEN_PT uint64 = 11;         // Point '.'
const TOKEN_NOT uint64 = 12;        // Boolean negation '!'
const TOKEN_NOTEQUAL uint64 = 13;   // Comparison, not equal '!='
const TOKEN_SEMICOLON uint64 = 14;  // Semi-colon ';'
const TOKEN_COLON uint64 = 15;      // Colon ','
const TOKEN_ASSIGN uint64 = 16;     // Assignment '='
const TOKEN_EQUALS uint64 = 17;     // Equal comparison '=='
const TOKEN_CHAR uint64 = 18;       // Single Quoted Character 'x'
const TOKEN_REL_AND uint64 = 19;    // AND Relation '&&'
const TOKEN_REL_OR uint64 = 20;     // OR Relation '||'
const TOKEN_REL_GTOE uint64 = 21;   // Greather-Than or Equal '>='
const TOKEN_REL_GT uint64 = 22;     // Greather-Than '>'
const TOKEN_REL_LTOE uint64 = 23;   // Less-Than or Equal '<='
const TOKEN_REL_LT uint64 = 24;     // Less-Than '<'
const TOKEN_ARITH_PLUS uint64 = 25; // Arith. Plus '+'
const TOKEN_ARITH_MINUS uint64 = 26;// Arith. Minus '-'
const TOKEN_ARITH_MUL uint64 = 27;  // Arith. Multiplication '*'
const TOKEN_ARITH_DIV uint64 = 28;  // Arith. Division '/'
const TOKEN_OP_ADR uint64 = 29;     // Address operator '&'

//
// Advanced tokens, that are generated in the 2nd step from identifiers
// The tokens represent the corresponding language keywords.
//
const TOKEN_FOR uint64 = 101;
const TOKEN_IF uint64 = 102;
const TOKEN_TYPE uint64 = 103;
const TOKEN_CONST uint64 = 104;
const TOKEN_VAR uint64 = 105;
const TOKEN_STRUCT uint64 = 106;
const TOKEN_RETURN uint64 = 107;
const TOKEN_FUNC uint64 = 108;
const TOKEN_PACKAGE uint64 = 109;
const TOKEN_IMPORT uint64 = 110;
<commit_msg>token.go: Add TokenToString()<commit_after>// Copyright 2010 The GoGo Authors. All rights reserved.
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

package main

//
// Set of recognized tokens
//
const TOKEN_IDENTIFIER uint64 = 1;  // Identifier
const TOKEN_STRING uint64 = 2;      // String using "..."
const TOKEN_EOS uint64 = 3;         // End of Scan
const TOKEN_LBRAC uint64 = 4;       // Left bracket '('
const TOKEN_RBRAC uint64 = 5;       // Right bracket ')'
const TOKEN_LSBRAC uint64 = 6;      // Left square bracket '['
const TOKEN_RSBRAC uint64 = 7;      // Right square bracket ']'
const TOKEN_INTEGER uint64 = 8;     // Integer number
const TOKEN_LCBRAC uint64 = 9;      // Left curly bracket '{'
const TOKEN_RCBRAC uint64 = 10;     // Right curly bracket '}'
const TOKEN_PT uint64 = 11;         // Point '.'
const TOKEN_NOT uint64 = 12;        // Boolean negation '!'
const TOKEN_NOTEQUAL uint64 = 13;   // Comparison, not equal '!='
const TOKEN_SEMICOLON uint64 = 14;  // Semi-colon ';'
const TOKEN_COLON uint64 = 15;      // Colon ','
const TOKEN_ASSIGN uint64 = 16;     // Assignment '='
const TOKEN_EQUALS uint64 = 17;     // Equal comparison '=='
const TOKEN_CHAR uint64 = 18;       // Single Quoted Character 'x'
const TOKEN_REL_AND uint64 = 19;    // AND Relation '&&'
const TOKEN_REL_OR uint64 = 20;     // OR Relation '||'
const TOKEN_REL_GTOE uint64 = 21;   // Greather-Than or Equal '>='
const TOKEN_REL_GT uint64 = 22;     // Greather-Than '>'
const TOKEN_REL_LTOE uint64 = 23;   // Less-Than or Equal '<='
const TOKEN_REL_LT uint64 = 24;     // Less-Than '<'
const TOKEN_ARITH_PLUS uint64 = 25; // Arith. Plus '+'
const TOKEN_ARITH_MINUS uint64 = 26;// Arith. Minus '-'
const TOKEN_ARITH_MUL uint64 = 27;  // Arith. Multiplication '*'
const TOKEN_ARITH_DIV uint64 = 28;  // Arith. Division '/'
const TOKEN_OP_ADR uint64 = 29;     // Address operator '&'

//
// Advanced tokens, that are generated in the 2nd step from identifiers
// The tokens represent the corresponding language keywords.
//
const TOKEN_FOR uint64 = 101;
const TOKEN_IF uint64 = 102;
const TOKEN_TYPE uint64 = 103;
const TOKEN_CONST uint64 = 104;
const TOKEN_VAR uint64 = 105;
const TOKEN_STRUCT uint64 = 106;
const TOKEN_RETURN uint64 = 107;
const TOKEN_FUNC uint64 = 108;
const TOKEN_PACKAGE uint64 = 109;
const TOKEN_IMPORT uint64 = 110;

//
// Helper functions
//

func TokenToString (id uint64) string {
    var retStr string;

    if id == TOKEN_IDENTIFIER {
        retStr = "<identifier>";
    }
    if id == TOKEN_STRING {
        retStr = "<string>";
    }
    if id == TOKEN_EOS {
        retStr = "<END-OF-SCAN>";
    }
    if id == TOKEN_LBRAC {
        retStr = "(";
    }
    if id == TOKEN_RBRAC {
        retStr = ")";
    }
    if id == TOKEN_LSBRAC {
        retStr = "[";
    }
    if id == TOKEN_RSBRAC {
        retStr = "]";
    }
    if id == TOKEN_INTEGER {
        retStr = "<integer>";
    }
    if id == TOKEN_LCBRAC {
        retStr = "{";
    }
    if id == TOKEN_RCBRAC {
        retStr = "}";
    }
    if id == TOKEN_PT {
        retStr = ".";
    }
    if id == TOKEN_NOT {
        retStr = "!";
    }
    if id == TOKEN_NOTEQUAL {
        retStr = "!=";
    }
    if id == TOKEN_SEMICOLON {
        retStr = ";";
    }
    if id == TOKEN_COLON {
        retStr = ",";
    }
    if id == TOKEN_ASSIGN {
        retStr = "=";
    }
    if id == TOKEN_EQUALS {
        retStr = "==";
    }
    if id == TOKEN_CHAR {
        retStr = "'<char>'";
    }
    if id == TOKEN_REL_AND {
        retStr = "&&";
    }
    if id == TOKEN_REL_OR {
        retStr = "||"
    }
    if id == TOKEN_REL_GTOE {
        retStr = ">=";
    }
    if id == TOKEN_REL_GT {
        retStr = ">";
    }
    if id == TOKEN_REL_LTOE {
        retStr = "<=";
    }
    if id == TOKEN_REL_LT {
        retStr = "<";
    }
    if id == TOKEN_ARITH_PLUS {
        retStr = "+";
    }
    if id == TOKEN_ARITH_MINUS {
        retStr = "-";
    }
    if id == TOKEN_ARITH_MUL {
        retStr = "*";
    }
    if id == TOKEN_ARITH_DIV {
        retStr = "/";
    }
    if id == TOKEN_OP_ADR {
        retStr = "&";
    }
    if id == TOKEN_FOR {
        retStr = "for";
    }
    if id == TOKEN_IF {
        retStr = "if";
    }
    if id == TOKEN_TYPE {
        retStr = "type";
    }
    if id == TOKEN_CONST {
        retStr = "const";
    }
    if id == TOKEN_VAR {
        retStr = "var";
    }
    if id == TOKEN_STRUCT {
        retStr = "struct";
    }
    if id == TOKEN_RETURN {
        retStr = "return";
    }
    if id == TOKEN_FUNC {
        retStr = "func";
    }
    if id == TOKEN_PACKAGE {
        retStr = "package";
    }
    if id == TOKEN_IMPORT {
        retStr = "import";
    }

    return retStr;
}
<|endoftext|>


<commit_before>/*

rtop-vis - ad hoc cluster monitoring over SSH

Copyright (c) 2015 RapidLoop

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/

package main

import (
	"bufio"
	"log"
	"os"
	"path"
	"strconv"
	"strings"
)

type Section struct {
	Hostname     string
	Port         int
	User         string
	IdentityFile string
}

func (s *Section) clear() {
	s.Hostname = ""
	s.Port = 0
	s.User = ""
	s.IdentityFile = ""
}

func (s *Section) getFull(name string, def Section) (host string, port int, user, keyfile string) {
	if len(s.Hostname) > 0 {
		host = s.Hostname
	} else if len(def.Hostname) > 0 {
		host = def.Hostname
	}
	if s.Port > 0 {
		port = s.Port
	} else if def.Port > 0 {
		port = def.Port
	}
	if len(s.User) > 0 {
		user = s.User
	} else if len(def.User) > 0 {
		user = def.User
	}
	if len(s.IdentityFile) > 0 {
		keyfile = s.IdentityFile
	} else if len(def.IdentityFile) > 0 {
		keyfile = def.IdentityFile
	}
	return
}

var HostInfo = make(map[string]Section)

func getSshEntry(name string) (host string, port int, user, keyfile string) {

	def := Section{Hostname: name}
	if defcfg, ok := HostInfo["*"]; ok {
		def = defcfg
	}

	if s, ok := HostInfo[name]; ok {
		return s.getFull(name, def)
	}
	for h, s := range HostInfo {
		if ok, err := path.Match(h, name); ok && err == nil {
			return s.getFull(name, def)
		}
	}
	return def.Hostname, def.Port, def.User, def.IdentityFile
}

func parseSshConfig(path string) bool {
	f, err := os.Open(path)
	if err != nil {
		log.Printf("warning: %v", err)
		return false
	}
	defer f.Close()
	update := func(cb func(s *Section)) {}
	s := bufio.NewScanner(f)
	for s.Scan() {
		line := strings.TrimSpace(s.Text())
		if len(line) == 0 || line[0] == '#' {
			continue
		}
		parts := strings.Fields(line)
		if len(parts) > 1 && parts[0] == "Host" {
			hosts := parts[1:]
			for _, h := range hosts {
				if _, ok := HostInfo[h]; !ok {
					HostInfo[h] = Section{}
				}
			}
			update = func(cb func(s *Section)) {
				for _, h := range hosts {
					s, _ := HostInfo[h]
					cb(&s)
					HostInfo[h] = s
				}
			}
		}
		if len(parts) == 2 {
			switch parts[0] {
			case "HostName":
				update(func(s *Section) {
					s.Hostname = parts[1]
				})
			case "Port":
				if p, err := strconv.Atoi(parts[1]); err == nil {
					update(func(s *Section) {
						s.Port = p
					})
				}
			case "User":
				update(func(s *Section) {
					s.User = parts[1]
				})
			case "IdentityFile":
				update(func(s *Section) {
					s.IdentityFile = parts[1]
				})
			}
		}
	}
	return true
}
<commit_msg>case insensitive recognition of ssh config values to prevent errors (e.g. "HostName" was not recognized)<commit_after>/*

rtop-vis - ad hoc cluster monitoring over SSH

Copyright (c) 2015 RapidLoop

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/

package main

import (
	"bufio"
	"log"
	"os"
	"path"
	"strconv"
	"strings"
)

type Section struct {
	Hostname     string
	Port         int
	User         string
	IdentityFile string
}

func (s *Section) clear() {
	s.Hostname = ""
	s.Port = 0
	s.User = ""
	s.IdentityFile = ""
}

func (s *Section) getFull(name string, def Section) (host string, port int, user, keyfile string) {
	if len(s.Hostname) > 0 {
		host = s.Hostname
	} else if len(def.Hostname) > 0 {
		host = def.Hostname
	}
	if s.Port > 0 {
		port = s.Port
	} else if def.Port > 0 {
		port = def.Port
	}
	if len(s.User) > 0 {
		user = s.User
	} else if len(def.User) > 0 {
		user = def.User
	}
	if len(s.IdentityFile) > 0 {
		keyfile = s.IdentityFile
	} else if len(def.IdentityFile) > 0 {
		keyfile = def.IdentityFile
	}
	return
}

var HostInfo = make(map[string]Section)

func getSshEntry(name string) (host string, port int, user, keyfile string) {

	def := Section{Hostname: name}
	if defcfg, ok := HostInfo["*"]; ok {
		def = defcfg
	}

	if s, ok := HostInfo[name]; ok {
		return s.getFull(name, def)
	}
	for h, s := range HostInfo {
		if ok, err := path.Match(h, name); ok && err == nil {
			return s.getFull(name, def)
		}
	}
	return def.Hostname, def.Port, def.User, def.IdentityFile
}

func parseSshConfig(path string) bool {
	f, err := os.Open(path)
	if err != nil {
		log.Printf("warning: %v", err)
		return false
	}
	defer f.Close()
	update := func(cb func(s *Section)) {}
	s := bufio.NewScanner(f)
	for s.Scan() {
		line := strings.TrimSpace(s.Text())
		if len(line) == 0 || line[0] == '#' {
			continue
		}
		parts := strings.Fields(line)
		if len(parts) > 1 && parts[0] == "Host" {
			hosts := parts[1:]
			for _, h := range hosts {
				if _, ok := HostInfo[h]; !ok {
					HostInfo[h] = Section{}
				}
			}
			update = func(cb func(s *Section)) {
				for _, h := range hosts {
					s, _ := HostInfo[h]
					cb(&s)
					HostInfo[h] = s
				}
			}
		}
		if len(parts) == 2 {
			switch strings.ToLower(parts[0]) {
			case "hostname":
				update(func(s *Section) {
					s.Hostname = parts[1]
				})
			case "port":
				if p, err := strconv.Atoi(parts[1]); err == nil {
					update(func(s *Section) {
						s.Port = p
					})
				}
			case "user":
				update(func(s *Section) {
					s.User = parts[1]
				})
			case "identityfile":
				update(func(s *Section) {
					s.IdentityFile = parts[1]
				})
			}
		}
	}
	return true
}
<|endoftext|>