<commit_before>package web

import (
	"encoding/json"
	"fmt"
	"html/template"
	"log"
	"net/http"
	"path/filepath"
	"text/template/parse"
	"time"

	"github.com/StackExchange/bosun/_third_party/github.com/MiniProfiler/go/miniprofiler"
	"github.com/StackExchange/bosun/_third_party/github.com/gorilla/mux"
	"github.com/StackExchange/bosun/conf"
	"github.com/StackExchange/bosun/relay"
	"github.com/StackExchange/bosun/sched"
)

var (
	templates *template.Template
	router    = mux.NewRouter()
	schedule  = sched.DefaultSched
)

func init() {
	miniprofiler.Position = "bottomleft"
	miniprofiler.StartHidden = true
}

func Listen(addr, dir, host string) error {
	var err error
	templates, err = template.New("").ParseFiles(
		dir + "/templates/index.html",
	)
	if err != nil {
		log.Fatal(err)
	}
	router.Handle("/api/action", JSON(Action))
	router.Handle("/api/alerts", JSON(Alerts))
	router.Handle("/api/config", miniprofiler.NewHandler(Config))
	router.Handle("/api/templates", JSON(Templates))
	router.Handle("/api/config_test", miniprofiler.NewHandler(ConfigTest))
	router.Handle("/api/egraph/{bs}.svg", JSON(ExprGraph))
	router.Handle("/api/expr", JSON(Expr))
	router.Handle("/api/graph", JSON(Graph))
	router.Handle("/api/metric", JSON(UniqueMetrics))
	router.Handle("/api/metric/{tagk}/{tagv}", JSON(MetricsByTagPair))
	router.Handle("/api/rule", JSON(Rule))
	router.Handle("/api/silence/clear", JSON(SilenceClear))
	router.Handle("/api/silence/get", JSON(SilenceGet))
	router.Handle("/api/silence/set", JSON(SilenceSet))
	router.Handle("/api/tagk/{metric}", JSON(TagKeysByMetric))
	router.Handle("/api/tagv/{tagk}", JSON(TagValuesByTagKey))
	router.Handle("/api/tagv/{tagk}/{metric}", JSON(TagValuesByMetricTagKey))
	router.HandleFunc("/api/put", relay.Handle(host))
	http.Handle("/", miniprofiler.NewHandler(Index))
	http.Handle("/api/", router)
	fs := http.FileServer(http.Dir(dir))
	http.Handle("/partials/", fs)
	http.Handle("/static/", fs)
	static := http.FileServer(http.Dir(filepath.Join(dir, "static")))
	http.Handle("/favicon.ico", static)
	log.Println("bosun web listening on:", addr)
	log.Println("bosun web directory:", dir)
	return http.ListenAndServe(addr, nil)
}

func Index(t miniprofiler.Timer, w http.ResponseWriter, r *http.Request) {
	if r.URL.Path == "/graph" {
		r.ParseForm()
		if _, present := r.Form["png"]; present {
			if _, err := Graph(t, w, r); err != nil {
				serveError(w, err)
			}
			return
		}
	}
	err := templates.ExecuteTemplate(w, "index.html", struct {
		Includes template.HTML
	}{
		t.Includes(),
	})
	if err != nil {
		serveError(w, err)
	}
}

func serveError(w http.ResponseWriter, err error) {
	http.Error(w, err.Error(), http.StatusInternalServerError)
}

func JSON(h func(miniprofiler.Timer, http.ResponseWriter, *http.Request) (interface{}, error)) http.Handler {
	return miniprofiler.NewHandler(func(t miniprofiler.Timer, w http.ResponseWriter, r *http.Request) {
		d, err := h(t, w, r)
		if err != nil {
			serveError(w, err)
			return
		}
		if d == nil {
			return
		}
		b, err := json.Marshal(d)
		if err != nil {
			serveError(w, err)
			return
		}
		w.Write(b)
	})
}

func Alerts(t miniprofiler.Timer, w http.ResponseWriter, r *http.Request) (interface{}, error) {
	return schedule, nil
}

func Action(t miniprofiler.Timer, w http.ResponseWriter, r *http.Request) (interface{}, error) {
	var data struct {
		Type    string
		User    string
		Message string
		Keys    []string
	}
	j := json.NewDecoder(r.Body)
	if err := j.Decode(&data); err != nil {
		return nil, err
	}
	var at sched.ActionType
	switch data.Type {
	case "ack":
		at = sched.ActionAcknowledge
	case "close":
		at = sched.ActionClose
	case "forget":
		at = sched.ActionForget
	}
	errs := make(MultiError)
	r.ParseForm()
	for _, key := range data.Keys {
		err := schedule.Action(data.User, data.Message, at, sched.AlertKey(key))
		if err != nil {
			errs[key] = err
		}
	}
	if len(errs) != 0 {
		return nil, errs
	}
	return nil, nil
}

type MultiError map[string]error

func (m MultiError) Error() string {
	return fmt.Sprint(map[string]error(m))
}

func SilenceGet(t miniprofiler.Timer, w http.ResponseWriter, r *http.Request) (interface{}, error) {
	return schedule.Silence, nil
}

var silenceLayouts = []string{
	"2006-01-02 15:04:05 MST",
	"2006-01-02 15:04:05 -0700",
	"2006-01-02 15:04 MST",
	"2006-01-02 15:04 -0700",
	"2006-01-02 15:04:05",
	"2006-01-02 15:04",
}

func SilenceSet(t miniprofiler.Timer, w http.ResponseWriter, r *http.Request) (interface{}, error) {
	var start, end time.Time
	var err error
	var data map[string]string
	j := json.NewDecoder(r.Body)
	if err := j.Decode(&data); err != nil {
		return nil, err
	}
	if s := data["start"]; s != "" {
		for _, layout := range silenceLayouts {
			start, err = time.Parse(layout, s)
			if err == nil {
				break
			}
		}
		if start.IsZero() {
			return nil, fmt.Errorf("unrecognized start time format: %s", s)
		}
	}
	if s := data["end"]; s != "" {
		for _, layout := range silenceLayouts {
			end, err = time.Parse(layout, s)
			if err == nil {
				break
			}
		}
		if end.IsZero() {
			return nil, fmt.Errorf("unrecognized end time format: %s", s)
		}
	}
	if start.IsZero() {
		start = time.Now().UTC()
	}
	if end.IsZero() {
		d, err := time.ParseDuration(data["duration"])
		if err != nil {
			return nil, err
		}
		end = start.Add(d)
	}
	return schedule.AddSilence(start, end, data["alert"], data["tags"], len(data["confirm"]) > 0, data["edit"])
}

func SilenceClear(t miniprofiler.Timer, w http.ResponseWriter, r *http.Request) (interface{}, error) {
	var data map[string]string
	j := json.NewDecoder(r.Body)
	if err := j.Decode(&data); err != nil {
		return nil, err
	}
	return nil, schedule.ClearSilence(data["id"])
}

func ConfigTest(t miniprofiler.Timer, w http.ResponseWriter, r *http.Request) {
	_, err := conf.New("test", r.FormValue("config_text"))
	if err != nil {
		fmt.Fprint(w, err.Error())
	}
}

func Config(t miniprofiler.Timer, w http.ResponseWriter, r *http.Request) {
	fmt.Fprint(w, schedule.Conf.RawText)
}

func nilFunc() {}

var builtins = template.FuncMap{
	"and":      nilFunc,
	"call":     nilFunc,
	"html":     nilFunc,
	"index":    nilFunc,
	"js":       nilFunc,
	"len":      nilFunc,
	"not":      nilFunc,
	"or":       nilFunc,
	"print":    nilFunc,
	"printf":   nilFunc,
	"println":  nilFunc,
	"urlquery": nilFunc,
	"eq":       nilFunc,
	"ge":       nilFunc,
	"gt":       nilFunc,
	"le":       nilFunc,
	"lt":       nilFunc,
	"ne":       nilFunc,

	// HTML-specific funcs
	"html_template_attrescaper":     nilFunc,
	"html_template_commentescaper":  nilFunc,
	"html_template_cssescaper":      nilFunc,
	"html_template_cssvaluefilter":  nilFunc,
	"html_template_htmlnamefilter":  nilFunc,
	"html_template_htmlescaper":     nilFunc,
	"html_template_jsregexpescaper": nilFunc,
	"html_template_jsstrescaper":    nilFunc,
	"html_template_jsvalescaper":    nilFunc,
	"html_template_nospaceescaper":  nilFunc,
	"html_template_rcdataescaper":   nilFunc,
	"html_template_urlescaper":      nilFunc,
	"html_template_urlfilter":       nilFunc,
	"html_template_urlnormalizer":   nilFunc,

	// bosun-specific funcs
	"V":       nilFunc,
	"bytes":   nilFunc,
	"replace": nilFunc,
	"short":   nilFunc,
}

func Templates(t miniprofiler.Timer, w http.ResponseWriter, r *http.Request) (interface{}, error) {
	templates := make(map[string]string)
	for name, template := range schedule.Conf.Templates {
		incl := map[string]bool{name: true}
		parseTemplate := func(s string) error {
			trees, err := parse.Parse("", s, "", "", nil, builtins)
			if err != nil {
				return err
			}
			for _, node := range trees[""].Root.Nodes {
				switch node := node.(type) {
				case *parse.TemplateNode:
					incl[node.Name] = true
				}
			}
			return nil
		}
		if template.Body != nil {
			if err := parseTemplate(template.Body.Tree.Root.String()); err != nil {
				return nil, err
			}
		}
		if template.Subject != nil {
			if err := parseTemplate(template.Subject.Tree.Root.String()); err != nil {
				return nil, err
			}
		}
		first := true
		for n := range incl {
			t := schedule.Conf.Templates[n]
			if t == nil {
				continue
			}
			if first {
				first = false
			} else {
				templates[name] += "\n\n"
			}
			templates[name] += t.Def
		}
	}
	return struct {
		Templates map[string]string
		Alerts    map[string]string
	}{
		templates,
		nil,
	}, nil
}
<commit_msg>Recursive template parsing<commit_after>package web

import (
	"encoding/json"
	"fmt"
	"html/template"
	"log"
	"net/http"
	"path/filepath"
	"text/template/parse"
	"time"

	"github.com/StackExchange/bosun/_third_party/github.com/MiniProfiler/go/miniprofiler"
	"github.com/StackExchange/bosun/_third_party/github.com/gorilla/mux"
	"github.com/StackExchange/bosun/conf"
	"github.com/StackExchange/bosun/relay"
	"github.com/StackExchange/bosun/sched"
)

var (
	templates *template.Template
	router    = mux.NewRouter()
	schedule  = sched.DefaultSched
)

func init() {
	miniprofiler.Position = "bottomleft"
	miniprofiler.StartHidden = true
}

func Listen(addr, dir, host string) error {
	var err error
	templates, err = template.New("").ParseFiles(
		dir + "/templates/index.html",
	)
	if err != nil {
		log.Fatal(err)
	}
	router.Handle("/api/action", JSON(Action))
	router.Handle("/api/alerts", JSON(Alerts))
	router.Handle("/api/config", miniprofiler.NewHandler(Config))
	router.Handle("/api/templates", JSON(Templates))
	router.Handle("/api/config_test", miniprofiler.NewHandler(ConfigTest))
	router.Handle("/api/egraph/{bs}.svg", JSON(ExprGraph))
	router.Handle("/api/expr", JSON(Expr))
	router.Handle("/api/graph", JSON(Graph))
	router.Handle("/api/metric", JSON(UniqueMetrics))
	router.Handle("/api/metric/{tagk}/{tagv}", JSON(MetricsByTagPair))
	router.Handle("/api/rule", JSON(Rule))
	router.Handle("/api/silence/clear", JSON(SilenceClear))
	router.Handle("/api/silence/get", JSON(SilenceGet))
	router.Handle("/api/silence/set", JSON(SilenceSet))
	router.Handle("/api/tagk/{metric}", JSON(TagKeysByMetric))
	router.Handle("/api/tagv/{tagk}", JSON(TagValuesByTagKey))
	router.Handle("/api/tagv/{tagk}/{metric}", JSON(TagValuesByMetricTagKey))
	router.HandleFunc("/api/put", relay.Handle(host))
	http.Handle("/", miniprofiler.NewHandler(Index))
	http.Handle("/api/", router)
	fs := http.FileServer(http.Dir(dir))
	http.Handle("/partials/", fs)
	http.Handle("/static/", fs)
	static := http.FileServer(http.Dir(filepath.Join(dir, "static")))
	http.Handle("/favicon.ico", static)
	log.Println("bosun web listening on:", addr)
	log.Println("bosun web directory:", dir)
	return http.ListenAndServe(addr, nil)
}

func Index(t miniprofiler.Timer, w http.ResponseWriter, r *http.Request) {
	if r.URL.Path == "/graph" {
		r.ParseForm()
		if _, present := r.Form["png"]; present {
			if _, err := Graph(t, w, r); err != nil {
				serveError(w, err)
			}
			return
		}
	}
	err := templates.ExecuteTemplate(w, "index.html", struct {
		Includes template.HTML
	}{
		t.Includes(),
	})
	if err != nil {
		serveError(w, err)
	}
}

func serveError(w http.ResponseWriter, err error) {
	http.Error(w, err.Error(), http.StatusInternalServerError)
}

func JSON(h func(miniprofiler.Timer, http.ResponseWriter, *http.Request) (interface{}, error)) http.Handler {
	return miniprofiler.NewHandler(func(t miniprofiler.Timer, w http.ResponseWriter, r *http.Request) {
		d, err := h(t, w, r)
		if err != nil {
			serveError(w, err)
			return
		}
		if d == nil {
			return
		}
		b, err := json.Marshal(d)
		if err != nil {
			serveError(w, err)
			return
		}
		w.Write(b)
	})
}

func Alerts(t miniprofiler.Timer, w http.ResponseWriter, r *http.Request) (interface{}, error) {
	return schedule, nil
}

func Action(t miniprofiler.Timer, w http.ResponseWriter, r *http.Request) (interface{}, error) {
	var data struct {
		Type    string
		User    string
		Message string
		Keys    []string
	}
	j := json.NewDecoder(r.Body)
	if err := j.Decode(&data); err != nil {
		return nil, err
	}
	var at sched.ActionType
	switch data.Type {
	case "ack":
		at = sched.ActionAcknowledge
	case "close":
		at = sched.ActionClose
	case "forget":
		at = sched.ActionForget
	}
	errs := make(MultiError)
	r.ParseForm()
	for _, key := range data.Keys {
		err := schedule.Action(data.User, data.Message, at, sched.AlertKey(key))
		if err != nil {
			errs[key] = err
		}
	}
	if len(errs) != 0 {
		return nil, errs
	}
	return nil, nil
}

type MultiError map[string]error

func (m MultiError) Error() string {
	return fmt.Sprint(map[string]error(m))
}

func SilenceGet(t miniprofiler.Timer, w http.ResponseWriter, r *http.Request) (interface{}, error) {
	return schedule.Silence, nil
}

var silenceLayouts = []string{
	"2006-01-02 15:04:05 MST",
	"2006-01-02 15:04:05 -0700",
	"2006-01-02 15:04 MST",
	"2006-01-02 15:04 -0700",
	"2006-01-02 15:04:05",
	"2006-01-02 15:04",
}

func SilenceSet(t miniprofiler.Timer, w http.ResponseWriter, r *http.Request) (interface{}, error) {
	var start, end time.Time
	var err error
	var data map[string]string
	j := json.NewDecoder(r.Body)
	if err := j.Decode(&data); err != nil {
		return nil, err
	}
	if s := data["start"]; s != "" {
		for _, layout := range silenceLayouts {
			start, err = time.Parse(layout, s)
			if err == nil {
				break
			}
		}
		if start.IsZero() {
			return nil, fmt.Errorf("unrecognized start time format: %s", s)
		}
	}
	if s := data["end"]; s != "" {
		for _, layout := range silenceLayouts {
			end, err = time.Parse(layout, s)
			if err == nil {
				break
			}
		}
		if end.IsZero() {
			return nil, fmt.Errorf("unrecognized end time format: %s", s)
		}
	}
	if start.IsZero() {
		start = time.Now().UTC()
	}
	if end.IsZero() {
		d, err := time.ParseDuration(data["duration"])
		if err != nil {
			return nil, err
		}
		end = start.Add(d)
	}
	return schedule.AddSilence(start, end, data["alert"], data["tags"], len(data["confirm"]) > 0, data["edit"])
}

func SilenceClear(t miniprofiler.Timer, w http.ResponseWriter, r *http.Request) (interface{}, error) {
	var data map[string]string
	j := json.NewDecoder(r.Body)
	if err := j.Decode(&data); err != nil {
		return nil, err
	}
	return nil, schedule.ClearSilence(data["id"])
}

func ConfigTest(t miniprofiler.Timer, w http.ResponseWriter, r *http.Request) {
	_, err := conf.New("test", r.FormValue("config_text"))
	if err != nil {
		fmt.Fprint(w, err.Error())
	}
}

func Config(t miniprofiler.Timer, w http.ResponseWriter, r *http.Request) {
	fmt.Fprint(w, schedule.Conf.RawText)
}

func nilFunc() {}

var builtins = template.FuncMap{
	"and":      nilFunc,
	"call":     nilFunc,
	"html":     nilFunc,
	"index":    nilFunc,
	"js":       nilFunc,
	"len":      nilFunc,
	"not":      nilFunc,
	"or":       nilFunc,
	"print":    nilFunc,
	"printf":   nilFunc,
	"println":  nilFunc,
	"urlquery": nilFunc,
	"eq":       nilFunc,
	"ge":       nilFunc,
	"gt":       nilFunc,
	"le":       nilFunc,
	"lt":       nilFunc,
	"ne":       nilFunc,

	// HTML-specific funcs
	"html_template_attrescaper":     nilFunc,
	"html_template_commentescaper":  nilFunc,
	"html_template_cssescaper":      nilFunc,
	"html_template_cssvaluefilter":  nilFunc,
	"html_template_htmlnamefilter":  nilFunc,
	"html_template_htmlescaper":     nilFunc,
	"html_template_jsregexpescaper": nilFunc,
	"html_template_jsstrescaper":    nilFunc,
	"html_template_jsvalescaper":    nilFunc,
	"html_template_nospaceescaper":  nilFunc,
	"html_template_rcdataescaper":   nilFunc,
	"html_template_urlescaper":      nilFunc,
	"html_template_urlfilter":       nilFunc,
	"html_template_urlnormalizer":   nilFunc,

	// bosun-specific funcs
	"V":       nilFunc,
	"bytes":   nilFunc,
	"replace": nilFunc,
	"short":   nilFunc,
}

func Templates(t miniprofiler.Timer, w http.ResponseWriter, r *http.Request) (interface{}, error) {
	templates := make(map[string]string)
	for name, template := range schedule.Conf.Templates {
		incl := map[string]bool{name: true}
		var parseSection func(*conf.Template) error
		parseTemplate := func(s string) error {
			trees, err := parse.Parse("", s, "", "", builtins)
			if err != nil {
				return err
			}
			for _, node := range trees[""].Root.Nodes {
				switch node := node.(type) {
				case *parse.TemplateNode:
					if incl[node.Name] {
						continue
					}
					incl[node.Name] = true
					if err := parseSection(schedule.Conf.Templates[node.Name]); err != nil {
						return err
					}
				}
			}
			return nil
		}
		parseSection = func(s *conf.Template) error {
			if s.Body != nil {
				if err := parseTemplate(s.Body.Tree.Root.String()); err != nil {
					return err
				}
			}
			if s.Subject != nil {
				if err := parseTemplate(s.Subject.Tree.Root.String()); err != nil {
					return err
				}
			}
			return nil
		}
		if err := parseSection(template); err != nil {
			return nil, err
		}
		delete(incl, name)
		templates[name] = template.Def
		for n := range incl {
			t := schedule.Conf.Templates[n]
			if t == nil {
				continue
			}
			templates[name] += "\n\n" + t.Def
		}
	}
	return struct {
		Templates map[string]string
		Alerts    map[string]string
	}{
		templates,
		nil,
	}, nil
}
<|endoftext|>


<commit_before>/*
Package web is a microframework inspired by Sinatra.

The underlying philosophy behind this package is that net/http is a very good
HTTP library which is only missing a few features. If you disagree with this
statement (e.g., you think that the interfaces it exposes are not especially
good, or if you're looking for a comprehensive "batteries included" feature
list), you're likely not going to have a good time using this library. In that
spirit, we have attempted wherever possible to be compatible with net/http. You
should be able to insert any net/http compliant handler into this library, or
use this library with any other net/http compliant mux.

This package attempts to solve three problems that net/http does not. First, it
allows you to specify URL patterns with Sinatra-like named wildcards and
regexps. Second, it allows you to write reconfigurable middleware stacks. And
finally, it allows you to attach additional context to requests, in a manner
that can be manipulated by both compliant middleware and handlers.

A usage example:

	m := web.New()

Use your favorite HTTP verbs:

	var legacyFooHttpHandler http.Handler // From elsewhere
	m.Get("/foo", legacyFooHttpHandler)
	m.Post("/bar", func(w http.ResponseWriter, r *http.Request) {
		w.Write("Hello world!")
	})

Bind parameters using either Sinatra-like patterns or regular expressions:

	m.Get("/hello/:name", func(c *web.C, w http.ResponseWriter, r *http.Request) {
		fmt.Fprintf(w, "Hello, %s!", c.UrlParams["name"])
	})
	pattern := regexp.MustCompile(`^/ip/(?P<ip>(?:\d{1,3}\.){3}\d{1,3})$`)
	m.Get(pattern, func(c *web.C, w http.ResponseWriter, r *http.Request) {
		fmt.Printf(w, "Info for IP address %s:", c.UrlParams["ip"])
	})

Middleware are functions that wrap http.Handlers, just like you'd use with raw
net/http. Middleware functions can optionally take a context parameter, which
will be threaded throughout the middleware stack and to the final handler, even
if not all of these things do not support contexts. Middleware are encouraged to
use the Env parameter to pass data to other middleware and to the final handler:

	m.Use(func(h http.Handler) http.Handler {
		handler := func(w http.ResponseWriter, r *http.Request) {
			log.Println("Before request")
			h.ServeHTTP(w, r)
			log.Println("After request")
		}
		return http.HandlerFunc(handler)
	})
	m.Use(func(c *web.C, h http.Handler) http.Handler {
		handler := func(w http.ResponseWriter, r *http.Request) {
			cookie, err := r.Cookie("user")
			if err == nil {
				c.Env["user"] = cookie.Raw
			}
			h.ServeHTTP(w, r)
		}
		return http.HandlerFunc(handler)
	})

	m.Get("/baz", func(c web.C, w http.ResponseWriter, r *http.Request) {
		if user, ok := c.Env["user"], ok {
			w.Write("Hello " + string(user))
		} else {
			w.Write("Hello Stranger!")
		}
	})
*/
package web

import (
	"net/http"
)

/*
Per-request context object. Threaded through all compliant middleware layers and
to the final request handler.

As an implementation detail, references to these structs are reused between
requests to reduce allocation churn, but the maps they contain are created fresh
on every request. If you are closing over a context (especially relevant for
middleware), you should not close over either the UrlParams or Env objects,
instead accessing them through the context whenever they are required.
*/
type C struct {
	// The parameters parsed by the mux from the URL itself. In most cases,
	// will contain a map from programmer-specified identifiers to the
	// strings that matched those identifiers, but if a unnamed regex
	// capture is used, it will be assigned to the special identifiers "$1",
	// "$2", etc.
	UrlParams map[string]string
	// A free-form environment, similar to Rack or PEP 333's environments.
	// Middleware layers are encouraged to pass data to downstream layers
	// and other handlers using this map, and are even more strongly
	// encouraged to document and maybe namespace they keys they use.
	Env map[string]interface{}
}

// A superset of net/http's http.Handler, which also includes a mechanism for
// serving requests with a context. If your handler does not support the use of
// contexts, we encourage you to use http.Handler instead.
type Handler interface {
	http.Handler
	ServeHTTPC(C, http.ResponseWriter, *http.Request)
}

// Like net/http's http.HandlerFunc, but supports a context object. Implements
// both http.Handler and web.Handler free of charge.
type HandlerFunc func(C, http.ResponseWriter, *http.Request)

func (h HandlerFunc) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	h(C{}, w, r)
}

func (h HandlerFunc) ServeHTTPC(c C, w http.ResponseWriter, r *http.Request) {
	h(c, w, r)
}
<commit_msg>Fix old documentation<commit_after>/*
Package web is a microframework inspired by Sinatra.

The underlying philosophy behind this package is that net/http is a very good
HTTP library which is only missing a few features. If you disagree with this
statement (e.g., you think that the interfaces it exposes are not especially
good, or if you're looking for a comprehensive "batteries included" feature
list), you're likely not going to have a good time using this library. In that
spirit, we have attempted wherever possible to be compatible with net/http. You
should be able to insert any net/http compliant handler into this library, or
use this library with any other net/http compliant mux.

This package attempts to solve three problems that net/http does not. First, it
allows you to specify URL patterns with Sinatra-like named wildcards and
regexps. Second, it allows you to write reconfigurable middleware stacks. And
finally, it allows you to attach additional context to requests, in a manner
that can be manipulated by both compliant middleware and handlers.

A usage example:

	m := web.New()

Use your favorite HTTP verbs:

	var legacyFooHttpHandler http.Handler // From elsewhere
	m.Get("/foo", legacyFooHttpHandler)
	m.Post("/bar", func(w http.ResponseWriter, r *http.Request) {
		w.Write("Hello world!")
	})

Bind parameters using either Sinatra-like patterns or regular expressions:

	m.Get("/hello/:name", func(c web.C, w http.ResponseWriter, r *http.Request) {
		fmt.Fprintf(w, "Hello, %s!", c.UrlParams["name"])
	})
	pattern := regexp.MustCompile(`^/ip/(?P<ip>(?:\d{1,3}\.){3}\d{1,3})$`)
	m.Get(pattern, func(c web.C, w http.ResponseWriter, r *http.Request) {
		fmt.Printf(w, "Info for IP address %s:", c.UrlParams["ip"])
	})

Middleware are functions that wrap http.Handlers, just like you'd use with raw
net/http. Middleware functions can optionally take a context parameter, which
will be threaded throughout the middleware stack and to the final handler, even
if not all of these things do not support contexts. Middleware are encouraged to
use the Env parameter to pass data to other middleware and to the final handler:

	m.Use(func(h http.Handler) http.Handler {
		handler := func(w http.ResponseWriter, r *http.Request) {
			log.Println("Before request")
			h.ServeHTTP(w, r)
			log.Println("After request")
		}
		return http.HandlerFunc(handler)
	})
	m.Use(func(c *web.C, h http.Handler) http.Handler {
		handler := func(w http.ResponseWriter, r *http.Request) {
			cookie, err := r.Cookie("user")
			if err == nil {
				c.Env["user"] = cookie.Raw
			}
			h.ServeHTTP(w, r)
		}
		return http.HandlerFunc(handler)
	})

	m.Get("/baz", func(c web.C, w http.ResponseWriter, r *http.Request) {
		if user, ok := c.Env["user"], ok {
			w.Write("Hello " + string(user))
		} else {
			w.Write("Hello Stranger!")
		}
	})
*/
package web

import (
	"net/http"
)

/*
Per-request context object. Threaded through all compliant middleware layers and
to the final request handler.

As an implementation detail, references to these structs are reused between
requests to reduce allocation churn, but the maps they contain are created fresh
on every request. If you are closing over a context (especially relevant for
middleware), you should not close over either the UrlParams or Env objects,
instead accessing them through the context whenever they are required.
*/
type C struct {
	// The parameters parsed by the mux from the URL itself. In most cases,
	// will contain a map from programmer-specified identifiers to the
	// strings that matched those identifiers, but if a unnamed regex
	// capture is used, it will be assigned to the special identifiers "$1",
	// "$2", etc.
	UrlParams map[string]string
	// A free-form environment, similar to Rack or PEP 333's environments.
	// Middleware layers are encouraged to pass data to downstream layers
	// and other handlers using this map, and are even more strongly
	// encouraged to document and maybe namespace they keys they use.
	Env map[string]interface{}
}

// A superset of net/http's http.Handler, which also includes a mechanism for
// serving requests with a context. If your handler does not support the use of
// contexts, we encourage you to use http.Handler instead.
type Handler interface {
	http.Handler
	ServeHTTPC(C, http.ResponseWriter, *http.Request)
}

// Like net/http's http.HandlerFunc, but supports a context object. Implements
// both http.Handler and web.Handler free of charge.
type HandlerFunc func(C, http.ResponseWriter, *http.Request)

func (h HandlerFunc) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	h(C{}, w, r)
}

func (h HandlerFunc) ServeHTTPC(c C, w http.ResponseWriter, r *http.Request) {
	h(c, w, r)
}
<|endoftext|>


<commit_before>package rsync

import (
	"bufio"
	"bytes"
	"context"
	"errors"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"time"
	"unicode"

	"koding/klient/machine/index"
)

// englishEnv contains current environment with C locale.
var englishEnv []string

func init() {
	englishEnv = append(os.Environ(), "LC_ALL=C")
}

// Command describes rsync executable.
type Command struct {
	// Download indicates the direction of changes. If set to true, source path
	// defines remote machine.
	Download bool `json:"download"`

	// SourcePath defines source path from which file(s) will be pulled.
	// This field is required.
	SourcePath string `json:"sourcePath"`

	// DestinationPath defines destination path to which file(s) will be pushed.
	// This field is required.
	DestinationPath string `json:"destinationPath"`

	// Cmd defines command to run. If nil, default rsync command will be used.
	Cmd *exec.Cmd `json:"-"`

	// Username defines remote machine user name. If not set, localhost transfer
	// will be used.
	Username string `json:"username"`

	// Host defines the remote machine address. If not set, localhost transfer
	// will be used.
	Host string `json:"host"`

	// PrivateKeyPath if set, SSH remote shell will be used as a data transport.
	PrivateKeyPath string `json:"privateKeyPath"`

	// SSHPort defines custom remote shell port. If not set, default will be used.
	SSHPort int `json:"sshPort"`

	// Progress if set, rsync will be run in recursive and verbose mode. The
	// current status of downloading will be periodically sent to provided
	// progress callback function. io.EOF error is sent to the callback when
	// downloading is complete.
	Progress func(n, size, speed int64, err error) `json:"-"`

	// Change if set, joins change name to source and destination paths and
	// uses provided metadata to set rsync arguments that allow to sync the
	// change.
	Change *index.Change `json:"change,omitempty"`

	// Output specifies an optional writer which, if set, will receive rsync
	// command output.
	Output io.Writer `json:"-"`
}

// valid checks if command fields are valid.
func (c *Command) valid() error {
	if c == nil {
		return errors.New("rsync: command is nil")
	}

	if c.SourcePath == "" {
		return errors.New("rsync: source path is not set")
	}
	if c.DestinationPath == "" {
		return errors.New("rsync: destination path is not set")
	}

	return nil
}

// DryRun copies command and runs it in dry mode - no copying is performed. This
// method is useful when one need to check the amount and size of transferred
// files.
func (c *Command) DryRun(ctx context.Context) (n, size int64, err error) {
	copy := *c

	// Dissable progress bar.
	var found = false
	copy.Progress = func(nAll, sizeAll, _ int64, err error) {
		if err == nil {
			n, size, found = nAll, sizeAll, true
		}
	}

	// Shallow copy since Cmd can be used only once.
	if copy.Cmd == nil {
		copy.Cmd = exec.CommandContext(ctx, "rsync")
	} else {
		copyCmd := *c.Cmd
		copy.Cmd = &copyCmd
	}

	copy.Cmd.Args = append(copy.Cmd.Args, "--dry-run")
	if err := copy.run(ctx, copy.dryscan); err != nil {
		return 0, 0, err
	}

	if !found {
		return 0, 0, errors.New("data size and number was not found")
	}

	return n, size, nil
}

// Run starts new rsync process. And waits for it to complete.
func (c *Command) Run(ctx context.Context) error {
	return c.run(ctx, c.scan)
}

// Run starts new rsync process. And waits for it to complete.
func (c *Command) run(ctx context.Context, scan func(r io.Reader)) error {
	if err := c.valid(); err != nil {
		return err
	}

	if c.Cmd == nil {
		c.Cmd = exec.CommandContext(ctx, "rsync")
	}

	// Add default arguments.
	c.Cmd.Args = append(c.Cmd.Args, "-zlptgoDd")
	if c.Cmd.Env == nil {
		c.Cmd.Env = englishEnv
	} else {
		c.Cmd.Env = append(englishEnv, c.Cmd.Env...)
	}

	// Use remote shell if SSH private key path is set.
	if c.PrivateKeyPath != "" {
		// TODO(ppknap): check if RC4 cipher will work on every machine without
		// altering sshd_config on destination.
		rsh := []string{
			"ssh", "-T", "-x", "-i", c.PrivateKeyPath,
			"-oCompression=no",
			"-oStrictHostKeychecking=no",
			"-oUserKnownHostsFile=/dev/null",
		}

		if c.SSHPort > 0 {
			rsh = append(rsh, " -p ", strconv.Itoa(c.SSHPort))
		}

		c.Cmd.Args = append(c.Cmd.Args, "-e", strings.Join(rsh, " "))
	}

	// Apply index change.
	if c.Change != nil {
		c.SourcePath = filepath.Join(c.SourcePath, c.Change.Path())
		c.DestinationPath = filepath.Join(c.DestinationPath, c.Change.Path())

		meta := c.Change.Meta()
		c.Download = meta&index.ChangeMetaLocal == 0 && meta&index.ChangeMetaRemote != 0
		if c.Download {
			c.SourcePath, c.DestinationPath = c.DestinationPath, c.SourcePath
		}

		if meta&index.ChangeMetaRemove != 0 {
			c.Cmd.Args = append(c.Cmd.Args, "--delete")
		}

		c.Cmd.Args = append(c.Cmd.Args, "--include='/"+filepath.Base(c.SourcePath)+"'", "--exclude='*'")
	}

	// Progress logic needs verbose mode with itemized changes.
	if c.Progress != nil {
		c.Cmd.Args = append(c.Cmd.Args, "-Piv")
		if c.Change == nil {
			// Do not use recursive downloads for index changes.
			c.Cmd.Args = append(c.Cmd.Args, "-r")
		}
	}

	if c.Username != "" && c.Host != "" {
		if c.Download {
			c.SourcePath = c.Username + "@" + c.Host + ":" + c.SourcePath
		} else {
			c.DestinationPath = c.Username + "@" + c.Host + ":" + c.DestinationPath
		}
	}

	if c.Change != nil {
		c.Cmd.Args = append(c.Cmd.Args, filepath.Dir(c.SourcePath)+"/", filepath.Dir(c.DestinationPath)+"/")
	} else {
		c.Cmd.Args = append(c.Cmd.Args, c.SourcePath, c.DestinationPath)
	}

	var errBuf bytes.Buffer
	c.Cmd.Stderr = noNilMultiWriter(c.Cmd.Stderr, c.Output, &errBuf)
	if c.Progress == nil {
		c.Cmd.Stdout = noNilMultiWriter(c.Cmd.Stdout, c.Output)
		return c.Cmd.Run()
	}

	// Set up progress callback when it's provided.
	rc, err := c.Cmd.StdoutPipe()
	if err != nil {
		return err
	}
	defer rc.Close()

	var r io.Reader = rc
	if c.Output != nil {
		r = io.TeeReader(rc, c.Output)
	}

	if err := c.Cmd.Start(); err != nil {
		return err
	}

	scan(r)
	if err = c.Cmd.Wait(); err != nil {
		c.Progress(0, 0, 0, err)
		if ee, ok := err.(*exec.ExitError); ok {
			ee.Stderr = errBuf.Bytes()
		}
	} else {
		c.Progress(0, 0, 0, io.EOF)
	}

	return err
}

var (
	rmComma = strings.NewReplacer(",", "")
	bitRe   = regexp.MustCompile(`^[.><ch*].......... .`)
	sizeRe  = regexp.MustCompile(`^\s*([\d,]+)\s+\d+%.*$`)
	totalRe = regexp.MustCompile(`^[^\d]*([\d,]+).*DRY\sRUN.*$`)
)

func (c *Command) dryscan(r io.Reader) {
	var n int64

	scanner := bufio.NewScanner(r)
	scanner.Split(scanNonControl)
	for scanner.Scan() {
		line := scanner.Text()
		if bitRe.MatchString(line) {
			n++
			continue
		}

		ms := totalRe.FindStringSubmatch(line)
		if len(ms) < 2 {
			continue
		}
		rmComma.Replace(ms[1])
		size, err := strconv.Atoi(rmComma.Replace(ms[1]))
		if err != nil {
			continue
		}

		c.Progress(n, int64(size), 0, nil)
	}
}

func (c *Command) scan(r io.Reader) {
	var (
		now           time.Time
		n, size, part int64
	)

	scanner := bufio.NewScanner(r)
	scanner.Split(scanNonControl)
	for scanner.Scan() {
		if now.IsZero() {
			now = time.Now()
		}

		line := scanner.Text()
		if bitRe.MatchString(line) {
			size += part
			n++
			part = 0
			continue
		}

		ms := sizeRe.FindStringSubmatch(line)
		if len(ms) < 2 {
			continue
		}
		rmComma.Replace(ms[1])
		p, err := strconv.Atoi(rmComma.Replace(ms[1]))
		if err != nil {
			continue
		}

		part = int64(p)

		speed := int64(float64(part+size)/(float64(time.Since(now))/float64(time.Second)) + 0.5)
		c.Progress(n, part+size, speed, nil)
	}
}

func scanNonControl(data []byte, atEOF bool) (advance int, token []byte, err error) {
	if atEOF && len(data) == 0 {
		return 0, nil, nil
	}

	if i := bytes.IndexFunc(data, unicode.IsControl); i >= 0 {
		return i + 1, dropCR(data[0:i]), nil
	}

	if atEOF {
		return len(data), dropCR(data), nil
	}

	return 0, nil, nil
}

// dropCR drops a terminal \r from the data. This function was copied from
// standard library.
func dropCR(data []byte) []byte {
	if len(data) > 0 && data[len(data)-1] == '\r' {
		return data[0 : len(data)-1]
	}
	return data
}

// noNilMultiWriter creates a multi writer from provided non-nil writers. If
// all writers are nil, this function will return nil writer.
func noNilMultiWriter(ws ...io.Writer) io.Writer {
	var writers []io.Writer
	for _, w := range ws {
		if w != nil {
			writers = append(writers, w)
		}
	}

	if len(writers) == 0 {
		return nil
	}

	return io.MultiWriter(writers...)
}
<commit_msg>klient/rsync: use koding/os package to replae environments<commit_after>package rsync

import (
	"bufio"
	"bytes"
	"context"
	"errors"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"time"
	"unicode"

	"koding/klient/machine/index"
	kos "koding/klient/os"
)

// englishEnv contains current environment with C locale.
var englishEnv []string

func init() {
	englishEnv = kos.NewEnviron(os.Environ()).Encode(kos.ParseEnviron("LANG=C,LC_ALL=C"))
}

// Command describes rsync executable.
type Command struct {
	// Download indicates the direction of changes. If set to true, source path
	// defines remote machine.
	Download bool `json:"download"`

	// SourcePath defines source path from which file(s) will be pulled.
	// This field is required.
	SourcePath string `json:"sourcePath"`

	// DestinationPath defines destination path to which file(s) will be pushed.
	// This field is required.
	DestinationPath string `json:"destinationPath"`

	// Cmd defines command to run. If nil, default rsync command will be used.
	Cmd *exec.Cmd `json:"-"`

	// Username defines remote machine user name. If not set, localhost transfer
	// will be used.
	Username string `json:"username"`

	// Host defines the remote machine address. If not set, localhost transfer
	// will be used.
	Host string `json:"host"`

	// PrivateKeyPath if set, SSH remote shell will be used as a data transport.
	PrivateKeyPath string `json:"privateKeyPath"`

	// SSHPort defines custom remote shell port. If not set, default will be used.
	SSHPort int `json:"sshPort"`

	// Progress if set, rsync will be run in recursive and verbose mode. The
	// current status of downloading will be periodically sent to provided
	// progress callback function. io.EOF error is sent to the callback when
	// downloading is complete.
	Progress func(n, size, speed int64, err error) `json:"-"`

	// Change if set, joins change name to source and destination paths and
	// uses provided metadata to set rsync arguments that allow to sync the
	// change.
	Change *index.Change `json:"change,omitempty"`

	// Output specifies an optional writer which, if set, will receive rsync
	// command output.
	Output io.Writer `json:"-"`
}

// valid checks if command fields are valid.
func (c *Command) valid() error {
	if c == nil {
		return errors.New("rsync: command is nil")
	}

	if c.SourcePath == "" {
		return errors.New("rsync: source path is not set")
	}
	if c.DestinationPath == "" {
		return errors.New("rsync: destination path is not set")
	}

	return nil
}

// DryRun copies command and runs it in dry mode - no copying is performed. This
// method is useful when one need to check the amount and size of transferred
// files.
func (c *Command) DryRun(ctx context.Context) (n, size int64, err error) {
	copy := *c

	// Dissable progress bar.
	var found = false
	copy.Progress = func(nAll, sizeAll, _ int64, err error) {
		if err == nil {
			n, size, found = nAll, sizeAll, true
		}
	}

	// Shallow copy since Cmd can be used only once.
	if copy.Cmd == nil {
		copy.Cmd = exec.CommandContext(ctx, "rsync")
	} else {
		copyCmd := *c.Cmd
		copy.Cmd = &copyCmd
	}

	copy.Cmd.Args = append(copy.Cmd.Args, "--dry-run")
	if err := copy.run(ctx, copy.dryscan); err != nil {
		return 0, 0, err
	}

	if !found {
		return 0, 0, errors.New("data size and number was not found")
	}

	return n, size, nil
}

// Run starts new rsync process. And waits for it to complete.
func (c *Command) Run(ctx context.Context) error {
	return c.run(ctx, c.scan)
}

// Run starts new rsync process. And waits for it to complete.
func (c *Command) run(ctx context.Context, scan func(r io.Reader)) error {
	if err := c.valid(); err != nil {
		return err
	}

	if c.Cmd == nil {
		c.Cmd = exec.CommandContext(ctx, "rsync")
	}

	// Add default arguments.
	c.Cmd.Args = append(c.Cmd.Args, "-zlptgoDd")
	if c.Cmd.Env == nil {
		c.Cmd.Env = englishEnv
	} else {
		c.Cmd.Env = kos.NewEnviron(englishEnv).Encode(kos.NewEnviron(c.Cmd.Env))
	}

	// Use remote shell if SSH private key path is set.
	if c.PrivateKeyPath != "" {
		// TODO(ppknap): check if RC4 cipher will work on every machine without
		// altering sshd_config on destination.
		rsh := []string{
			"ssh", "-T", "-x", "-i", c.PrivateKeyPath,
			"-oCompression=no",
			"-oStrictHostKeychecking=no",
			"-oUserKnownHostsFile=/dev/null",
		}

		if c.SSHPort > 0 {
			rsh = append(rsh, " -p ", strconv.Itoa(c.SSHPort))
		}

		c.Cmd.Args = append(c.Cmd.Args, "-e", strings.Join(rsh, " "))
	}

	// Apply index change.
	if c.Change != nil {
		c.SourcePath = filepath.Join(c.SourcePath, c.Change.Path())
		c.DestinationPath = filepath.Join(c.DestinationPath, c.Change.Path())

		meta := c.Change.Meta()
		c.Download = meta&index.ChangeMetaLocal == 0 && meta&index.ChangeMetaRemote != 0
		if c.Download {
			c.SourcePath, c.DestinationPath = c.DestinationPath, c.SourcePath
		}

		if meta&index.ChangeMetaRemove != 0 {
			c.Cmd.Args = append(c.Cmd.Args, "--delete")
		}

		c.Cmd.Args = append(c.Cmd.Args, "--include='/"+filepath.Base(c.SourcePath)+"'", "--exclude='*'")
	}

	// Progress logic needs verbose mode with itemized changes.
	if c.Progress != nil {
		c.Cmd.Args = append(c.Cmd.Args, "-Piv")
		if c.Change == nil {
			// Do not use recursive downloads for index changes.
			c.Cmd.Args = append(c.Cmd.Args, "-r")
		}
	}

	if c.Username != "" && c.Host != "" {
		if c.Download {
			c.SourcePath = c.Username + "@" + c.Host + ":" + c.SourcePath
		} else {
			c.DestinationPath = c.Username + "@" + c.Host + ":" + c.DestinationPath
		}
	}

	if c.Change != nil {
		c.Cmd.Args = append(c.Cmd.Args, filepath.Dir(c.SourcePath)+"/", filepath.Dir(c.DestinationPath)+"/")
	} else {
		c.Cmd.Args = append(c.Cmd.Args, c.SourcePath, c.DestinationPath)
	}

	var errBuf bytes.Buffer
	c.Cmd.Stderr = noNilMultiWriter(c.Cmd.Stderr, c.Output, &errBuf)
	if c.Progress == nil {
		c.Cmd.Stdout = noNilMultiWriter(c.Cmd.Stdout, c.Output)
		return c.Cmd.Run()
	}

	// Set up progress callback when it's provided.
	rc, err := c.Cmd.StdoutPipe()
	if err != nil {
		return err
	}
	defer rc.Close()

	var r io.Reader = rc
	if c.Output != nil {
		r = io.TeeReader(rc, c.Output)
	}

	if err := c.Cmd.Start(); err != nil {
		return err
	}

	scan(r)
	if err = c.Cmd.Wait(); err != nil {
		c.Progress(0, 0, 0, err)
		if ee, ok := err.(*exec.ExitError); ok {
			ee.Stderr = errBuf.Bytes()
		}
	} else {
		c.Progress(0, 0, 0, io.EOF)
	}

	return err
}

var (
	rmComma = strings.NewReplacer(",", "")
	bitRe   = regexp.MustCompile(`^[.><ch*].......... .`)
	sizeRe  = regexp.MustCompile(`^\s*([\d,]+)\s+\d+%.*$`)
	totalRe = regexp.MustCompile(`^[^\d]*([\d,]+).*DRY\sRUN.*$`)
)

func (c *Command) dryscan(r io.Reader) {
	var n int64

	scanner := bufio.NewScanner(r)
	scanner.Split(scanNonControl)
	for scanner.Scan() {
		line := scanner.Text()
		if bitRe.MatchString(line) {
			n++
			continue
		}

		ms := totalRe.FindStringSubmatch(line)
		if len(ms) < 2 {
			continue
		}
		rmComma.Replace(ms[1])
		size, err := strconv.Atoi(rmComma.Replace(ms[1]))
		if err != nil {
			continue
		}

		c.Progress(n, int64(size), 0, nil)
	}
}

func (c *Command) scan(r io.Reader) {
	var (
		now           time.Time
		n, size, part int64
	)

	scanner := bufio.NewScanner(r)
	scanner.Split(scanNonControl)
	for scanner.Scan() {
		if now.IsZero() {
			now = time.Now()
		}

		line := scanner.Text()
		if bitRe.MatchString(line) {
			size += part
			n++
			part = 0
			continue
		}

		ms := sizeRe.FindStringSubmatch(line)
		if len(ms) < 2 {
			continue
		}
		rmComma.Replace(ms[1])
		p, err := strconv.Atoi(rmComma.Replace(ms[1]))
		if err != nil {
			continue
		}

		part = int64(p)

		speed := int64(float64(part+size)/(float64(time.Since(now))/float64(time.Second)) + 0.5)
		c.Progress(n, part+size, speed, nil)
	}
}

func scanNonControl(data []byte, atEOF bool) (advance int, token []byte, err error) {
	if atEOF && len(data) == 0 {
		return 0, nil, nil
	}

	if i := bytes.IndexFunc(data, unicode.IsControl); i >= 0 {
		return i + 1, dropCR(data[0:i]), nil
	}

	if atEOF {
		return len(data), dropCR(data), nil
	}

	return 0, nil, nil
}

// dropCR drops a terminal \r from the data. This function was copied from
// standard library.
func dropCR(data []byte) []byte {
	if len(data) > 0 && data[len(data)-1] == '\r' {
		return data[0 : len(data)-1]
	}
	return data
}

// noNilMultiWriter creates a multi writer from provided non-nil writers. If
// all writers are nil, this function will return nil writer.
func noNilMultiWriter(ws ...io.Writer) io.Writer {
	var writers []io.Writer
	for _, w := range ws {
		if w != nil {
			writers = append(writers, w)
		}
	}

	if len(writers) == 0 {
		return nil
	}

	return io.MultiWriter(writers...)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"ieveapi/apicache"
	"log"
	"sync"
	"sync/atomic"
	"time"
)

type apiReq struct {
	url    string
	params map[string]string
	data   []byte
	apiErr apicache.APIError

	expires time.Time

	httpCode int
	err      error
	respChan chan apiReq
}

var workChan chan apiReq

var apiClient apicache.Client
var activeWorkerCount, workerCount int32

func APIReq(url string, params map[string]string) ([]byte, int, error) {
	if atomic.LoadInt32(&workerCount) <= 0 {
		startWorkers()
	}

	respChan := make(chan apiReq)
	req := apiReq{url: url, params: params, respChan: respChan}
	workChan <- req

	resp := <-respChan
	close(respChan)

	return resp.data, resp.httpCode, resp.err
}

var logActive int32

func worker(reqChan chan apiReq) {
	atomic.AddInt32(&workerCount, 1)
	for req := range reqChan {
		atomic.AddInt32(&activeWorkerCount, 1)
		apireq := apicache.NewRequest(req.url)
		for k, v := range req.params {
			apireq.Set(k, v)
		}

		resp, err := apireq.Do()
		req.data = resp.Data
		req.expires = resp.Expires
		req.apiErr = resp.Error
		req.httpCode = resp.HTTPCode
		req.err = err

		var errorStr string
		if resp.Error.ErrorCode != 0 {
			errorStr = fmt.Sprintf(" Error %d: %s", resp.Error.ErrorCode, resp.Error.ErrorText)
		}
		useLog := atomic.LoadInt32(&logActive)
		if useLog != 0 {
			log.Printf("%s - %+v FromCache: %v HTTP: %d Expires: %s%s", req.url, req.params, resp.FromCache, resp.HTTPCode, resp.Expires.Format("2006-01-02 15:04:05"), errorStr)
		}

		req.respChan <- req
		atomic.AddInt32(&activeWorkerCount, -1)
	}
	atomic.AddInt32(&workerCount, -1)
}

var startWorkersOnce = &sync.Once{}

func startWorkers() {
	startWorkersOnce.Do(realStartWorkers)
}

func realStartWorkers() {
	log.Printf("Starting %d Workers...", conf.Workers)
	workChan = make(chan apiReq)

	for i := 0; i < conf.Workers; i++ {
		go worker(workChan)
	}
}

func stopWorkers() {
	close(workChan)
	for atomic.LoadInt32(&activeWorkerCount) > 0 {
		time.Sleep(10 * time.Millisecond)
	}
	startWorkersOnce = &sync.Once{}
}

func PrintWorkerStats() {
	active, loaded := GetWorkerStats()
	log.Printf("%d workers idle, %d workers active.", loaded-active, active)
}

func EnableVerboseLogging() {
	atomic.AddInt32(&logActive, 1)
}

func DisableVerboseLogging() {
	useLog := atomic.LoadInt32(&logActive)
	atomic.AddInt32(&logActive, -useLog)
}

func GetWorkerStats() (int32, int32) {
	return atomic.LoadInt32(&activeWorkerCount), atomic.LoadInt32(&workerCount)
}
<commit_msg>Always log errors from the eve api<commit_after>package main

import (
	"fmt"
	"ieveapi/apicache"
	"log"
	"sync"
	"sync/atomic"
	"time"
)

type apiReq struct {
	url    string
	params map[string]string
	data   []byte
	apiErr apicache.APIError

	expires time.Time

	httpCode int
	err      error
	respChan chan apiReq
}

var workChan chan apiReq

var apiClient apicache.Client
var activeWorkerCount, workerCount int32

func APIReq(url string, params map[string]string) ([]byte, int, error) {
	if atomic.LoadInt32(&workerCount) <= 0 {
		startWorkers()
	}

	respChan := make(chan apiReq)
	req := apiReq{url: url, params: params, respChan: respChan}
	workChan <- req

	resp := <-respChan
	close(respChan)

	return resp.data, resp.httpCode, resp.err
}

var logActive int32

func worker(reqChan chan apiReq) {
	atomic.AddInt32(&workerCount, 1)
	for req := range reqChan {
		atomic.AddInt32(&activeWorkerCount, 1)
		apireq := apicache.NewRequest(req.url)
		for k, v := range req.params {
			apireq.Set(k, v)
		}

		resp, err := apireq.Do()
		req.data = resp.Data
		req.expires = resp.Expires
		req.apiErr = resp.Error
		req.httpCode = resp.HTTPCode
		req.err = err

		var errorStr string
		if resp.Error.ErrorCode != 0 {
			errorStr = fmt.Sprintf(" Error %d: %s", resp.Error.ErrorCode, resp.Error.ErrorText)
		}
		useLog := atomic.LoadInt32(&logActive)
		if useLog != 0 || resp.HTTPCode != 200 {
			log.Printf("%s - %+v FromCache: %v HTTP: %d Expires: %s%s", req.url, req.params, resp.FromCache, resp.HTTPCode, resp.Expires.Format("2006-01-02 15:04:05"), errorStr)
		}

		req.respChan <- req
		atomic.AddInt32(&activeWorkerCount, -1)
	}
	atomic.AddInt32(&workerCount, -1)
}

var startWorkersOnce = &sync.Once{}

func startWorkers() {
	startWorkersOnce.Do(realStartWorkers)
}

func realStartWorkers() {
	log.Printf("Starting %d Workers...", conf.Workers)
	workChan = make(chan apiReq)

	for i := 0; i < conf.Workers; i++ {
		go worker(workChan)
	}
}

func stopWorkers() {
	close(workChan)
	for atomic.LoadInt32(&activeWorkerCount) > 0 {
		time.Sleep(10 * time.Millisecond)
	}
	startWorkersOnce = &sync.Once{}
}

func PrintWorkerStats() {
	active, loaded := GetWorkerStats()
	log.Printf("%d workers idle, %d workers active.", loaded-active, active)
}

func EnableVerboseLogging() {
	atomic.AddInt32(&logActive, 1)
}

func DisableVerboseLogging() {
	useLog := atomic.LoadInt32(&logActive)
	atomic.AddInt32(&logActive, -useLog)
}

func GetWorkerStats() (int32, int32) {
	return atomic.LoadInt32(&activeWorkerCount), atomic.LoadInt32(&workerCount)
}
<|endoftext|>


<commit_before>package obj

import (
	"fmt"
	"reflect"

	. "github.com/polydawn/refmt/tok"
)

type ptrDerefDelegateUnmarshalMachine struct {
	UnmarshalMachine     // a delegate machine, already set for us by the slab
	peelCount        int // how deep the pointers go, already discoverd by the slab

	ptr_rv    reflect.Value // the top ptr, which we will use if setting to nil, or if we have to recursively make ptrs for `**X` types.
	firstStep bool
}

func (mach *ptrDerefDelegateUnmarshalMachine) Reset(slab *unmarshalSlab, rv reflect.Value, _ reflect.Type) error {
	mach.ptr_rv = rv
	mach.firstStep = true
	// we defer reseting the delegate machine until later, in case we get a nil, which can save a lot of time.
	return nil
}
func (mach *ptrDerefDelegateUnmarshalMachine) Step(driver *Unmarshaller, slab *unmarshalSlab, tok *Token) (done bool, err error) {
	// If first step: we have to do initializations.
	if mach.firstStep {
		mach.firstStep = false
		// If nil: easy road.  Nil the ptr.
		if tok.Type == TNull {
			mach.ptr_rv.Set(reflect.Zero(mach.ptr_rv.Type()))
			return true, nil
		}
		// Walk the pointers: if some already exist, we accept them unmodified;
		//  if any are nil, make a new one, and recursively.
		rv := mach.ptr_rv
		for i := 0; i < mach.peelCount; i++ {
			if rv.IsNil() {
				rv.Set(reflect.New(rv.Type().Elem()))
				rv = rv.Elem()
			} else {
				rv = rv.Elem()
			}
		}
		if err := mach.UnmarshalMachine.Reset(slab, rv, rv.Type()); err != nil {
			return true, err
		}
	}
	// The remainder of the time: it's just delegation.
	return mach.UnmarshalMachine.Step(driver, slab, tok)
}

type unmarshalMachinePrimitive struct {
	kind reflect.Kind

	rv reflect.Value
}

func (mach *unmarshalMachinePrimitive) Reset(_ *unmarshalSlab, rv reflect.Value, _ reflect.Type) error {
	mach.rv = rv
	return nil
}
func (mach *unmarshalMachinePrimitive) Step(_ *Unmarshaller, _ *unmarshalSlab, tok *Token) (done bool, err error) {
	switch mach.kind {
	case reflect.Bool:
		switch tok.Type {
		case TBool:
			mach.rv.SetBool(tok.Bool)
			return true, nil
		default:
			return true, ErrUnmarshalTypeCantFit{*tok, mach.rv, 0}
		}
	case reflect.String:
		switch tok.Type {
		case TString:
			mach.rv.SetString(tok.Str)
			return true, nil
		default:
			return true, ErrUnmarshalTypeCantFit{*tok, mach.rv, 0}
		}
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		switch tok.Type {
		case TInt:
			mach.rv.SetInt(tok.Int)
			return true, nil
		case TUint:
			mach.rv.SetInt(int64(tok.Uint)) // todo: overflow check
			return true, nil
		default:
			return true, ErrUnmarshalTypeCantFit{*tok, mach.rv, 0}
		}
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
		switch tok.Type {
		case TInt:
			if tok.Int >= 0 {
				mach.rv.SetUint(uint64(tok.Int))
				return true, nil
			}
			return true, ErrUnmarshalTypeCantFit{*tok, mach.rv, 0}
		case TUint:
			mach.rv.SetUint(tok.Uint)
			return true, nil
		default:
			return true, ErrUnmarshalTypeCantFit{*tok, mach.rv, 0}
		}
	case reflect.Float32, reflect.Float64:
		switch tok.Type {
		case TFloat64:
			mach.rv.SetFloat(tok.Float64)
			return true, nil
		case TInt:
			mach.rv.SetFloat(float64(tok.Int))
			return true, nil
		case TUint:
			mach.rv.SetFloat(float64(tok.Uint))
			return true, nil
		default:
			return true, ErrUnmarshalTypeCantFit{*tok, mach.rv, 0}
		}
	case reflect.Slice: // implicitly bytes; no other slices are "primitive"
		switch tok.Type {
		case TBytes:
			mach.rv.SetBytes(tok.Bytes)
			return true, nil
		default:
			return true, ErrUnmarshalTypeCantFit{*tok, mach.rv, 0}
		}
	case reflect.Array: // implicitly bytes; no other arrays are "primitive"
		switch tok.Type {
		case TBytes:
			// Unfortunately, there does not seem to be any efficient way to bulk set the contents of a byte array via reflect.
			// There are similar complaints regarding slices on the marshalling side: apparently, we have no choice but to loop.
			n := mach.rv.Len()
			// We match aggressively on length.  If the provided input is too short, we reject that too: we assume you asked for a fixed-length array for a reason.
			if len(tok.Bytes) != n {
				return true, ErrUnmarshalTypeCantFit{*tok, mach.rv, n}
			}
			for i := 0; i < n; i++ {
				mach.rv.Index(i).SetUint(uint64(tok.Bytes[i]))
			}
			return true, nil
		default:
			return true, ErrUnmarshalTypeCantFit{*tok, mach.rv, 0}
		}
	case reflect.Interface:
		switch tok.Type {
		case TString:
			mach.rv.Set(reflect.ValueOf(tok.Str))
		case TBytes:
			mach.rv.Set(reflect.ValueOf(tok.Bytes))
		case TBool:
			mach.rv.Set(reflect.ValueOf(tok.Bool))
		case TInt:
			mach.rv.Set(reflect.ValueOf(tok.Int))
		case TUint:
			mach.rv.Set(reflect.ValueOf(tok.Uint))
		case TFloat64:
			mach.rv.Set(reflect.ValueOf(tok.Float64))
		case TNull:
			mach.rv.Set(reflect.ValueOf(nil))
		default: // any of the other token types should not have been routed here to begin with.
			panic(fmt.Errorf("unhandled: %v", mach.kind))
		}
		return true, nil
	default:
		panic(fmt.Errorf("unhandled: %v", mach.kind))
	}
}
<commit_msg>fix unmarshaling null into []byte<commit_after>package obj

import (
	"fmt"
	"reflect"

	. "github.com/polydawn/refmt/tok"
)

type ptrDerefDelegateUnmarshalMachine struct {
	UnmarshalMachine     // a delegate machine, already set for us by the slab
	peelCount        int // how deep the pointers go, already discoverd by the slab

	ptr_rv    reflect.Value // the top ptr, which we will use if setting to nil, or if we have to recursively make ptrs for `**X` types.
	firstStep bool
}

func (mach *ptrDerefDelegateUnmarshalMachine) Reset(slab *unmarshalSlab, rv reflect.Value, _ reflect.Type) error {
	mach.ptr_rv = rv
	mach.firstStep = true
	// we defer reseting the delegate machine until later, in case we get a nil, which can save a lot of time.
	return nil
}
func (mach *ptrDerefDelegateUnmarshalMachine) Step(driver *Unmarshaller, slab *unmarshalSlab, tok *Token) (done bool, err error) {
	// If first step: we have to do initializations.
	if mach.firstStep {
		mach.firstStep = false
		// If nil: easy road.  Nil the ptr.
		if tok.Type == TNull {
			mach.ptr_rv.Set(reflect.Zero(mach.ptr_rv.Type()))
			return true, nil
		}
		// Walk the pointers: if some already exist, we accept them unmodified;
		//  if any are nil, make a new one, and recursively.
		rv := mach.ptr_rv
		for i := 0; i < mach.peelCount; i++ {
			if rv.IsNil() {
				rv.Set(reflect.New(rv.Type().Elem()))
				rv = rv.Elem()
			} else {
				rv = rv.Elem()
			}
		}
		if err := mach.UnmarshalMachine.Reset(slab, rv, rv.Type()); err != nil {
			return true, err
		}
	}
	// The remainder of the time: it's just delegation.
	return mach.UnmarshalMachine.Step(driver, slab, tok)
}

type unmarshalMachinePrimitive struct {
	kind reflect.Kind

	rv reflect.Value
}

func (mach *unmarshalMachinePrimitive) Reset(_ *unmarshalSlab, rv reflect.Value, _ reflect.Type) error {
	mach.rv = rv
	return nil
}
func (mach *unmarshalMachinePrimitive) Step(_ *Unmarshaller, _ *unmarshalSlab, tok *Token) (done bool, err error) {
	switch mach.kind {
	case reflect.Bool:
		switch tok.Type {
		case TBool:
			mach.rv.SetBool(tok.Bool)
			return true, nil
		default:
			return true, ErrUnmarshalTypeCantFit{*tok, mach.rv, 0}
		}
	case reflect.String:
		switch tok.Type {
		case TString:
			mach.rv.SetString(tok.Str)
			return true, nil
		default:
			return true, ErrUnmarshalTypeCantFit{*tok, mach.rv, 0}
		}
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		switch tok.Type {
		case TInt:
			mach.rv.SetInt(tok.Int)
			return true, nil
		case TUint:
			mach.rv.SetInt(int64(tok.Uint)) // todo: overflow check
			return true, nil
		default:
			return true, ErrUnmarshalTypeCantFit{*tok, mach.rv, 0}
		}
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
		switch tok.Type {
		case TInt:
			if tok.Int >= 0 {
				mach.rv.SetUint(uint64(tok.Int))
				return true, nil
			}
			return true, ErrUnmarshalTypeCantFit{*tok, mach.rv, 0}
		case TUint:
			mach.rv.SetUint(tok.Uint)
			return true, nil
		default:
			return true, ErrUnmarshalTypeCantFit{*tok, mach.rv, 0}
		}
	case reflect.Float32, reflect.Float64:
		switch tok.Type {
		case TFloat64:
			mach.rv.SetFloat(tok.Float64)
			return true, nil
		case TInt:
			mach.rv.SetFloat(float64(tok.Int))
			return true, nil
		case TUint:
			mach.rv.SetFloat(float64(tok.Uint))
			return true, nil
		default:
			return true, ErrUnmarshalTypeCantFit{*tok, mach.rv, 0}
		}
	case reflect.Slice: // implicitly bytes; no other slices are "primitive"
		switch tok.Type {
		case TBytes:
			mach.rv.SetBytes(tok.Bytes)
			return true, nil
		case TNull:
			mach.rv.SetBytes(nil)
			return true, nil
		default:
			return true, ErrUnmarshalTypeCantFit{*tok, mach.rv, 0}
		}
	case reflect.Array: // implicitly bytes; no other arrays are "primitive"
		switch tok.Type {
		case TBytes:
			// Unfortunately, there does not seem to be any efficient way to bulk set the contents of a byte array via reflect.
			// There are similar complaints regarding slices on the marshalling side: apparently, we have no choice but to loop.
			n := mach.rv.Len()
			// We match aggressively on length.  If the provided input is too short, we reject that too: we assume you asked for a fixed-length array for a reason.
			if len(tok.Bytes) != n {
				return true, ErrUnmarshalTypeCantFit{*tok, mach.rv, n}
			}
			for i := 0; i < n; i++ {
				mach.rv.Index(i).SetUint(uint64(tok.Bytes[i]))
			}
			return true, nil
		case TNull:
			if mach.rv.Len() != 0 {
				return true, ErrUnmarshalTypeCantFit{*tok, mach.rv, 0}
			}
			mach.rv.SetBytes(nil)
			return true, nil
		default:
			return true, ErrUnmarshalTypeCantFit{*tok, mach.rv, 0}
		}
	case reflect.Interface:
		switch tok.Type {
		case TString:
			mach.rv.Set(reflect.ValueOf(tok.Str))
		case TBytes:
			mach.rv.Set(reflect.ValueOf(tok.Bytes))
		case TBool:
			mach.rv.Set(reflect.ValueOf(tok.Bool))
		case TInt:
			mach.rv.Set(reflect.ValueOf(tok.Int))
		case TUint:
			mach.rv.Set(reflect.ValueOf(tok.Uint))
		case TFloat64:
			mach.rv.Set(reflect.ValueOf(tok.Float64))
		case TNull:
			mach.rv.Set(reflect.ValueOf(nil))
		default: // any of the other token types should not have been routed here to begin with.
			panic(fmt.Errorf("unhandled: %v", mach.kind))
		}
		return true, nil
	default:
		panic(fmt.Errorf("unhandled: %v", mach.kind))
	}
}
<|endoftext|>


<commit_before>package multipass

import (
	"crypto/rand"
	"crypto/rsa"
	"crypto/x509"
	"encoding/json"
	"encoding/pem"
	"errors"
	"fmt"
	"html"
	"log"
	"net"
	"net/http"
	"net/smtp"
	"net/url"
	"path"
	"strings"
	"time"

	"github.com/mholt/caddy/caddyhttp/httpserver"

	jose "gopkg.in/square/go-jose.v1"
)

var ErrInvalidToken = errors.New("invalid token")

type Auth struct {
	*Multipass
	Next httpserver.Handler
}

type Rule struct {
	Basepath  string
	Expires   time.Duration
	Resources []string
	Handles   []string

	SMTPAddr, SMTPUser, SMTPPass string
	MailFrom, MailTmpl           string
}

type Multipass struct {
	Resources []string
	Basepath  string
	SiteAddr  string
	Expires   time.Duration

	Handler HandleService
	signer  jose.Signer
	key     *rsa.PrivateKey
}

func NewMultipassFromRule(r Rule) (*Multipass, error) {
	m, err := NewMultipass()
	if err != nil {
		return nil, err
	}
	if len(r.Resources) > 0 {
		m.Resources = r.Resources
	}
	if len(r.Basepath) > 0 {
		m.Basepath = path.Join("/", r.Basepath)
	}
	if r.Expires > 0 {
		m.Expires = r.Expires
	}

	mailTmpl := emailTemplate
	if len(r.MailTmpl) > 0 {
		mailTmpl = r.MailTmpl
	}

	host := "localhost"
	port := "25"
	if h, p, err := net.SplitHostPort(r.SMTPAddr); err == nil {
		host = h
		port = p
	} else {
		host = r.SMTPAddr
	}

	var auth smtp.Auth
	if len(r.SMTPUser) > 0 && len(r.SMTPPass) > 0 {
		auth = smtp.PlainAuth("", r.SMTPUser, r.SMTPPass, host)
	}

	addr := strings.Join([]string{host, port}, ":")
	from := html.UnescapeString(r.MailFrom)
	handler := NewEmailHandler(addr, auth, from, mailTmpl)
	for _, handle := range r.Handles {
		handler.Register(handle)
	}
	m.Handler = handler

	return m, nil
}

func NewMultipass() (*Multipass, error) {
	pk, err := rsa.GenerateKey(rand.Reader, 2048)
	if err != nil {
		return nil, err
	}
	signer, err := jose.NewSigner(jose.PS512, pk)
	if err != nil {
		return nil, err
	}
	return &Multipass{
		Resources: []string{"/"},
		Basepath:  "/",
		Expires:   time.Hour * 24,
		key:       pk,
		signer:    signer,
	}, nil
}

// Claims are part of the JSON web token
type Claims struct {
	Handle    string   `json:"handle"`
	Resources []string `json:"resources"`
	Expires   int64    `json:"exp"`
}

func (m *Multipass) AccessToken(handle string) (tokenStr string, err error) {
	exp := time.Now().Add(m.Expires)
	claims := &Claims{
		Handle:    handle,
		Resources: m.Resources,
		Expires:   exp.Unix(),
	}
	payload, err := json.Marshal(claims)
	if err != nil {
		return "", err
	}
	jws, err := m.signer.Sign(payload)
	if err != nil {
		return "", err
	}

	return jws.CompactSerialize()
}

func NewLoginURL(siteaddr, basepath, token string, v url.Values) (*url.URL, error) {
	u, err := url.Parse(siteaddr)
	if err != nil {
		return u, err
	}
	u.Path = path.Join(basepath, "login")
	v.Set("token", token)
	u.RawQuery = v.Encode()
	return u, nil
}

func loginHandler(w http.ResponseWriter, r *http.Request, m *Multipass) (int, error) {
	if r.Method == "POST" {
		r.ParseForm()
		handle := r.PostForm.Get("handle")
		if len(handle) == 0 {
			loc := path.Join(m.Basepath, "login")
			http.Redirect(w, r, loc, http.StatusSeeOther)
			return http.StatusSeeOther, nil
		}
		if m.Handler.Listed(handle) {
			token, err := m.AccessToken(handle)
			if err != nil {
				log.Print(err)
			}
			values := url.Values{}
			if s := r.PostForm.Get("url"); len(s) > 0 {
				values.Set("url", s)
			}
			loginURL, err := NewLoginURL(m.SiteAddr, m.Basepath, token, values)
			if err != nil {
				log.Print(err)
			}
			if err := m.Handler.Notify(handle, loginURL.String()); err != nil {
				log.Print(err)
			}
		}
		location := path.Join(m.Basepath, "/login/confirm")
		http.Redirect(w, r, location, http.StatusSeeOther)
		return http.StatusSeeOther, nil
	}
	if r.Method == "GET" {
		if tokenStr := r.URL.Query().Get("token"); len(tokenStr) > 0 {
			cookie := &http.Cookie{
				Name:  "jwt_token",
				Value: tokenStr,
				Path:  "/",
			}
			http.SetCookie(w, cookie)

			nexturl := m.SiteAddr
			if s := r.URL.Query().Get("url"); len(s) > 0 {
				nexturl = s
			}
			http.Redirect(w, r, nexturl, http.StatusSeeOther)
			return http.StatusSeeOther, nil
		}
		nextURL, err := url.Parse(m.SiteAddr)
		if err != nil {
			return http.StatusInternalServerError, err
		}
		nextURL.Path = r.URL.Path
		nextURL.RawQuery = r.URL.RawQuery
		w.Header().Set("Content-Type", "text/html; charset=utf-8")
		w.Write([]byte(`

Multipass

This resource is protected by Multipass. Enter your user handle to gain access.

<form action="` + path.Join(m.Basepath, "/login") + `" method=POST>
<input type=hidden name=url value="` + nextURL.String() + `"/>
<input type=text name=handle />
<input type=submit>
</form>
`))
		return http.StatusOK, nil
	}
	return http.StatusMethodNotAllowed, nil
}

func confirmHandler(w http.ResponseWriter, r *http.Request, m *Multipass) (int, error) {
	w.Header().Add("Content-Type", "text/html; charset=utf-8")
	w.Write([]byte("A login link has been sent to you"))
	return http.StatusOK, nil
}

func signoutHandler(w http.ResponseWriter, r *http.Request, m *Multipass) (int, error) {
	if cookie, err := r.Cookie("jwt_token"); err == nil {
		cookie.Expires = time.Now().AddDate(-1, 0, 0)
		cookie.MaxAge = -1
		cookie.Path = "/"
		http.SetCookie(w, cookie)
	}
	loc := path.Join(m.Basepath, "login")
	http.Redirect(w, r, loc, http.StatusSeeOther)
	return http.StatusSeeOther, nil
}

func publickeyHandler(w http.ResponseWriter, r *http.Request, m *Multipass) (int, error) {
	data, err := x509.MarshalPKIXPublicKey(&m.key.PublicKey)
	if err != nil {
		return http.StatusInternalServerError, err
	}
	block := &pem.Block{
		Type:  "PUBLIC KEY",
		Bytes: data,
	}
	w.Header().Set("Content-Type", "application/pkix-cert")
	if err := pem.Encode(w, block); err != nil {
		return http.StatusInternalServerError, err
	}
	return http.StatusOK, nil
}

func tokenHandler(w http.ResponseWriter, r *http.Request, m *Multipass) (int, error) {
	// Extract token from HTTP header, query parameter or cookie
	tokenStr, err := extractToken(r)
	if err != nil {
		return http.StatusUnauthorized, ErrInvalidToken
	}
	var claims *Claims
	if claims, err = validateToken(tokenStr, m.key.PublicKey); err != nil {
		return http.StatusUnauthorized, ErrInvalidToken
	}
	// Authorize handle claim
	if ok := m.Handler.Listed(claims.Handle); !ok {
		return http.StatusUnauthorized, ErrInvalidToken
	}
	// Verify path claim
	var match bool
	for _, path := range claims.Resources {
		if httpserver.Path(r.URL.Path).Matches(path) {
			match = true
			continue
		}
	}
	if !match {
		return http.StatusUnauthorized, ErrInvalidToken
	}
	return http.StatusOK, nil
}

func (a *Auth) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) {
	m := a.Multipass

	if httpserver.Path(r.URL.Path).Matches(m.Basepath) {
		switch r.URL.Path {
		case path.Join(m.Basepath, "pub.cer"):
			return publickeyHandler(w, r, m)
		case path.Join(m.Basepath, "login"):
			return loginHandler(w, r, m)
		case path.Join(m.Basepath, "login/confirm"):
			return confirmHandler(w, r, m)
		case path.Join(m.Basepath, "signout"):
			return signoutHandler(w, r, m)
		default:
			return http.StatusNotFound, nil
		}
	}

	for _, path := range m.Resources {
		if httpserver.Path(r.URL.Path).Matches(path) {
			if code, err := tokenHandler(w, r, m); err != nil {
				w.WriteHeader(code)
				return loginHandler(w, r, m)
			}
		}
	}

	return a.Next.ServeHTTP(w, r)
}

// extractToken returns the JWT token embedded in the given request.
// JWT tokens can be embedded in the header prefixed with "Bearer ", with a
// "token" key query parameter or a cookie named "jwt_token".
func extractToken(r *http.Request) (string, error) {
	//from header
	if h := r.Header.Get("Authorization"); strings.HasPrefix(h, "Bearer ") {
		if len(h) > 7 {
			return h[7:], nil
		}
	}

	//from query parameter
	if token := r.URL.Query().Get("token"); len(token) > 0 {
		return token, nil
	}

	//from cookie
	if cookie, err := r.Cookie("jwt_token"); err == nil {
		return cookie.Value, nil
	}

	return "", fmt.Errorf("no token found")
}

func validateToken(token string, key rsa.PublicKey) (*Claims, error) {
	claims := &Claims{}

	// Verify token signature
	payload, err := verifyToken(token, key)
	if err != nil {
		return nil, err
	}
	// Unmarshal token claims
	if err := json.Unmarshal(payload, claims); err != nil {
		return nil, err
	}
	// Verify expire claim
	if time.Unix(claims.Expires, 0).Before(time.Now()) {
		return nil, errors.New("Token expired")
	}
	return claims, nil
}

func verifyToken(token string, key rsa.PublicKey) ([]byte, error) {
	var data []byte

	obj, err := jose.ParseSigned(token)
	if err != nil {
		return data, err
	}
	data, err = obj.Verify(&key)
	if err != nil {
		return data, err
	}
	return data, nil
}
<commit_msg>Fix double WriteHeader<commit_after>package multipass

import (
	"crypto/rand"
	"crypto/rsa"
	"crypto/x509"
	"encoding/json"
	"encoding/pem"
	"errors"
	"fmt"
	"html"
	"log"
	"net"
	"net/http"
	"net/smtp"
	"net/url"
	"path"
	"strings"
	"time"

	"github.com/mholt/caddy/caddyhttp/httpserver"

	jose "gopkg.in/square/go-jose.v1"
)

var ErrInvalidToken = errors.New("invalid token")

type Auth struct {
	*Multipass
	Next httpserver.Handler
}

type Rule struct {
	Basepath  string
	Expires   time.Duration
	Resources []string
	Handles   []string

	SMTPAddr, SMTPUser, SMTPPass string
	MailFrom, MailTmpl           string
}

type Multipass struct {
	Resources []string
	Basepath  string
	SiteAddr  string
	Expires   time.Duration

	Handler HandleService
	signer  jose.Signer
	key     *rsa.PrivateKey
}

func NewMultipassFromRule(r Rule) (*Multipass, error) {
	m, err := NewMultipass()
	if err != nil {
		return nil, err
	}
	if len(r.Resources) > 0 {
		m.Resources = r.Resources
	}
	if len(r.Basepath) > 0 {
		m.Basepath = path.Join("/", r.Basepath)
	}
	if r.Expires > 0 {
		m.Expires = r.Expires
	}

	mailTmpl := emailTemplate
	if len(r.MailTmpl) > 0 {
		mailTmpl = r.MailTmpl
	}

	host := "localhost"
	port := "25"
	if h, p, err := net.SplitHostPort(r.SMTPAddr); err == nil {
		host = h
		port = p
	} else {
		host = r.SMTPAddr
	}

	var auth smtp.Auth
	if len(r.SMTPUser) > 0 && len(r.SMTPPass) > 0 {
		auth = smtp.PlainAuth("", r.SMTPUser, r.SMTPPass, host)
	}

	addr := strings.Join([]string{host, port}, ":")
	from := html.UnescapeString(r.MailFrom)
	handler := NewEmailHandler(addr, auth, from, mailTmpl)
	for _, handle := range r.Handles {
		handler.Register(handle)
	}
	m.Handler = handler

	return m, nil
}

func NewMultipass() (*Multipass, error) {
	pk, err := rsa.GenerateKey(rand.Reader, 2048)
	if err != nil {
		return nil, err
	}
	signer, err := jose.NewSigner(jose.PS512, pk)
	if err != nil {
		return nil, err
	}
	return &Multipass{
		Resources: []string{"/"},
		Basepath:  "/",
		Expires:   time.Hour * 24,
		key:       pk,
		signer:    signer,
	}, nil
}

// Claims are part of the JSON web token
type Claims struct {
	Handle    string   `json:"handle"`
	Resources []string `json:"resources"`
	Expires   int64    `json:"exp"`
}

func (m *Multipass) AccessToken(handle string) (tokenStr string, err error) {
	exp := time.Now().Add(m.Expires)
	claims := &Claims{
		Handle:    handle,
		Resources: m.Resources,
		Expires:   exp.Unix(),
	}
	payload, err := json.Marshal(claims)
	if err != nil {
		return "", err
	}
	jws, err := m.signer.Sign(payload)
	if err != nil {
		return "", err
	}

	return jws.CompactSerialize()
}

func NewLoginURL(siteaddr, basepath, token string, v url.Values) (*url.URL, error) {
	u, err := url.Parse(siteaddr)
	if err != nil {
		return u, err
	}
	u.Path = path.Join(basepath, "login")
	v.Set("token", token)
	u.RawQuery = v.Encode()
	return u, nil
}

func loginHandler(w http.ResponseWriter, r *http.Request, m *Multipass) (int, error) {
	if r.Method == "POST" {
		r.ParseForm()
		handle := r.PostForm.Get("handle")
		if len(handle) == 0 {
			loc := path.Join(m.Basepath, "login")
			http.Redirect(w, r, loc, http.StatusSeeOther)
			return http.StatusSeeOther, nil
		}
		if m.Handler.Listed(handle) {
			token, err := m.AccessToken(handle)
			if err != nil {
				log.Print(err)
			}
			values := url.Values{}
			if s := r.PostForm.Get("url"); len(s) > 0 {
				values.Set("url", s)
			}
			loginURL, err := NewLoginURL(m.SiteAddr, m.Basepath, token, values)
			if err != nil {
				log.Print(err)
			}
			if err := m.Handler.Notify(handle, loginURL.String()); err != nil {
				log.Print(err)
			}
		}
		location := path.Join(m.Basepath, "/login/confirm")
		http.Redirect(w, r, location, http.StatusSeeOther)
		return http.StatusSeeOther, nil
	}
	if r.Method == "GET" {
		if tokenStr := r.URL.Query().Get("token"); len(tokenStr) > 0 {
			cookie := &http.Cookie{
				Name:  "jwt_token",
				Value: tokenStr,
				Path:  "/",
			}
			http.SetCookie(w, cookie)

			nexturl := m.SiteAddr
			if s := r.URL.Query().Get("url"); len(s) > 0 {
				nexturl = s
			}
			http.Redirect(w, r, nexturl, http.StatusSeeOther)
			return http.StatusSeeOther, nil
		}
		nextURL, err := url.Parse(m.SiteAddr)
		if err != nil {
			return http.StatusInternalServerError, err
		}
		nextURL.Path = r.URL.Path
		nextURL.RawQuery = r.URL.RawQuery
		w.Header().Set("Content-Type", "text/html; charset=utf-8")
		w.Write([]byte(`

Multipass

This resource is protected by Multipass. Enter your user handle to gain access.

<form action="` + path.Join(m.Basepath, "/login") + `" method=POST>
<input type=hidden name=url value="` + nextURL.String() + `"/>
<input type=text name=handle />
<input type=submit>
</form>
`))
		return http.StatusOK, nil
	}
	return http.StatusMethodNotAllowed, nil
}

func confirmHandler(w http.ResponseWriter, r *http.Request, m *Multipass) (int, error) {
	w.Header().Add("Content-Type", "text/html; charset=utf-8")
	w.Write([]byte("A login link has been sent to you"))
	return http.StatusOK, nil
}

func signoutHandler(w http.ResponseWriter, r *http.Request, m *Multipass) (int, error) {
	if cookie, err := r.Cookie("jwt_token"); err == nil {
		cookie.Expires = time.Now().AddDate(-1, 0, 0)
		cookie.MaxAge = -1
		cookie.Path = "/"
		http.SetCookie(w, cookie)
	}
	loc := path.Join(m.Basepath, "login")
	http.Redirect(w, r, loc, http.StatusSeeOther)
	return http.StatusSeeOther, nil
}

func publickeyHandler(w http.ResponseWriter, r *http.Request, m *Multipass) (int, error) {
	data, err := x509.MarshalPKIXPublicKey(&m.key.PublicKey)
	if err != nil {
		return http.StatusInternalServerError, err
	}
	block := &pem.Block{
		Type:  "PUBLIC KEY",
		Bytes: data,
	}
	w.Header().Set("Content-Type", "application/pkix-cert")
	if err := pem.Encode(w, block); err != nil {
		return http.StatusInternalServerError, err
	}
	return http.StatusOK, nil
}

func tokenHandler(w http.ResponseWriter, r *http.Request, m *Multipass) (int, error) {
	// Extract token from HTTP header, query parameter or cookie
	tokenStr, err := extractToken(r)
	if err != nil {
		return http.StatusUnauthorized, ErrInvalidToken
	}
	var claims *Claims
	if claims, err = validateToken(tokenStr, m.key.PublicKey); err != nil {
		return http.StatusUnauthorized, ErrInvalidToken
	}
	// Authorize handle claim
	if ok := m.Handler.Listed(claims.Handle); !ok {
		return http.StatusUnauthorized, ErrInvalidToken
	}
	// Verify path claim
	var match bool
	for _, path := range claims.Resources {
		if httpserver.Path(r.URL.Path).Matches(path) {
			match = true
			continue
		}
	}
	if !match {
		return http.StatusUnauthorized, ErrInvalidToken
	}
	return http.StatusOK, nil
}

func (a *Auth) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) {
	m := a.Multipass

	if httpserver.Path(r.URL.Path).Matches(m.Basepath) {
		switch r.URL.Path {
		case path.Join(m.Basepath, "pub.cer"):
			return publickeyHandler(w, r, m)
		case path.Join(m.Basepath, "login"):
			return loginHandler(w, r, m)
		case path.Join(m.Basepath, "login/confirm"):
			return confirmHandler(w, r, m)
		case path.Join(m.Basepath, "signout"):
			return signoutHandler(w, r, m)
		default:
			return http.StatusNotFound, nil
		}
	}

	for _, path := range m.Resources {
		if httpserver.Path(r.URL.Path).Matches(path) {
			if _, err := tokenHandler(w, r, m); err != nil {
				return loginHandler(w, r, m)
			}
		}
	}

	return a.Next.ServeHTTP(w, r)
}

// extractToken returns the JWT token embedded in the given request.
// JWT tokens can be embedded in the header prefixed with "Bearer ", with a
// "token" key query parameter or a cookie named "jwt_token".
func extractToken(r *http.Request) (string, error) {
	//from header
	if h := r.Header.Get("Authorization"); strings.HasPrefix(h, "Bearer ") {
		if len(h) > 7 {
			return h[7:], nil
		}
	}

	//from query parameter
	if token := r.URL.Query().Get("token"); len(token) > 0 {
		return token, nil
	}

	//from cookie
	if cookie, err := r.Cookie("jwt_token"); err == nil {
		return cookie.Value, nil
	}

	return "", fmt.Errorf("no token found")
}

func validateToken(token string, key rsa.PublicKey) (*Claims, error) {
	claims := &Claims{}

	// Verify token signature
	payload, err := verifyToken(token, key)
	if err != nil {
		return nil, err
	}
	// Unmarshal token claims
	if err := json.Unmarshal(payload, claims); err != nil {
		return nil, err
	}
	// Verify expire claim
	if time.Unix(claims.Expires, 0).Before(time.Now()) {
		return nil, errors.New("Token expired")
	}
	return claims, nil
}

func verifyToken(token string, key rsa.PublicKey) ([]byte, error) {
	var data []byte

	obj, err := jose.ParseSigned(token)
	if err != nil {
		return data, err
	}
	data, err = obj.Verify(&key)
	if err != nil {
		return data, err
	}
	return data, nil
}
<|endoftext|>


<commit_before>package PoolAndAgent

import (
	"database/sql"
	"fmt"
	"strconv"
	"time"

	"github.com/3zheng/railgun/DialManager"
	ListenManager "github.com/3zheng/railgun/TcpListenManager"
	bs_proto "github.com/3zheng/railgun/protodefine"
	bs_types "github.com/3zheng/railgun/protodefine/mytype"
	bs_router "github.com/3zheng/railgun/protodefine/router"
	bs_tcp "github.com/3zheng/railgun/protodefine/tcpnet"

	_ "github.com/go-sql-driver/mysql"
	proto "github.com/golang/protobuf/proto"
)

type NetAgent struct {
	IpAddress string
}

func (*NetAgent) SendMsg(req *bs_tcp.TCPTransferMsg) {
	var connId uint64 = req.Base.ConnId
	sess := ListenManager.GetSessionByConnId(connId)
	if sess == nil { //判断是否为空
		return
	}
	//往MsgWriteCh里写req
	select {
	case sess.MsgWriteCh <- req:
	case <-time.After(5 * time.Second):
		fmt.Println("严重事件，sess.MsgWriteCh已经阻塞超时5秒, connId =", connId)
	}
}

//创建一个NetAgent,ipAdd是个带端口的ip地址，如果是监听一个端口使用0.0.0.0:port
//NetAgent是TcpManager的映射，在CreateNetAgent并不和TcpManager联系起来，而是在SingleMsgPool的InitAndRun来创建TcpManager的TCP端口监听
func CreateNetAgent(ipAdd string) *NetAgent {
	agent := new(NetAgent)
	agent.IpAddress = ipAdd
	return agent
}

type RouterAgent struct {
	IpAddress   string
	DialSession *DialManager.ConnectionSession
}

func CreateRouterAgent(ipAdd string) *RouterAgent {
	agent := new(RouterAgent)
	agent.IpAddress = ipAdd
	return agent
}

//启动运行RouterAgent
func (this *RouterAgent) RunRouterAgent(RouterToLogicChannel chan proto.Message, myAppId uint32, myAppType uint32) {
	fmt.Println("RunRouterAgent() myAppId=", myAppId, "myAppType=", myAppType)
	isClosed := false
	this.DialSession = DialManager.CreateClient(this.IpAddress, RouterToLogicChannel)
	for {
		//在这个router session关闭后，需要不停的尝试重新连接
		for isClosed || this.DialSession == nil {
			select {
			case <-time.After(700 * time.Millisecond): //每700毫秒尝试重新连接一次
			}
			this.DialSession = DialManager.CreateClient(this.IpAddress, RouterToLogicChannel)
			if this.DialSession != nil {
				//重连成功了
				fmt.Println("router Agent重连成功了")
				//这里用break是不能跳出for循环的，只能跳出当前的select，所以只能isClosed = false
				isClosed = false
			}
		}
		//连上router后向router发送注册报文
		req := new(bs_router.RegisterAppReq)
		bs_proto.SetBaseKindAndSubId(req)
		req.AppType = myAppType
		req.AppId = myAppId
		buff, err := proto.Marshal(req)
		if err != nil {
			fmt.Println("序列化RegisterAppReq出错")
		}
		tcpMsg := new(bs_tcp.TCPTransferMsg)
		bs_proto.SetBaseKindAndSubId(tcpMsg)
		tcpMsg.Data = buff
		tcpMsg.DataKindId = uint32(bs_types.CMDKindId_IDKindRouter)
		tcpMsg.DataSubId = uint32(bs_router.CMDID_Router_IDRegisterAppReq)
		this.SendMsg(tcpMsg)
		fmt.Println("向router app发送了注册请求")
		//阻塞在这里直到session关闭
		select {
		case v := <-this.DialSession.Quit:
			if v {
				fmt.Println("router agent的连接已关闭")
				isClosed = true
				this.DialSession = nil
			}
		}
	}
}

func (this *RouterAgent) SendMsg(req *bs_tcp.TCPTransferMsg) {
	var connId uint64 = req.Base.ConnId
	if this.DialSession == nil { //判断是否为空
		return
	}
	//往MsgWriteCh里写req
	select {
	case this.DialSession.MsgWriteCh <- req:
	case <-time.After(5 * time.Second):
		fmt.Println("严重事件，DialSession.MsgWriteCh已经阻塞超时5秒, connId =", connId)
	}

}

type DBReadInfo struct {
	TheRows    *sql.Rows      //如果是读操作，这里是返回的数据集
	TheColumns map[string]int //返回的列名,key是列名，value是表示这个是第几列，在返回的时候好查找
	ArrValues  [][]string     //返回的结果值
	RowNum     int            //行数，其实就是len(ArrValues)
}

func (this *DBReadInfo) Clear() {
	this.TheRows = nil
	this.TheColumns = nil
	this.ArrValues = nil
	this.RowNum = 0
}

type DBWriteInfo struct {
	LastId       int64 //自增列的当前值
	AffectedRows int64 //写操作影响的行数
}

//数据库的映射
type CADODatabase struct {
	DBSourceString string      //mysql的数据库连接串	格式"root:123456@tcp(localhost:3306)/sns?charset=utf8"
	Err            error       //错误
	TheDB          *sql.DB     //数据库对象
	ReadInfo       DBReadInfo  //如果是select的读操作，相关返回结果存这里
	WriteInfo      DBWriteInfo //如果是update或insert的写操作，相关返回结果存这里
}

func CreateADODatabase(DBSourceString string) *CADODatabase {
	DBAgent := new(CADODatabase)
	DBAgent.DBSourceString = DBSourceString
	return DBAgent
}

func (this *CADODatabase) InitDB() {
	db, err := sql.Open("mysql", this.DBSourceString)
	if err != nil {
		fmt.Println("初始化失败，error=", err.Error())
		panic(err.Error())
		this.Err = err
	}
	fmt.Println("数据库连接初始化成功")
	this.TheDB = db
}

//从数据库里读数据
func (this *CADODatabase) ReadFromDB(sqlExpress string) {
	//在读数据前先clear DBReadInfo
	this.ReadInfo.Clear()
	rows, err := this.TheDB.Query(sqlExpress)
	if err != nil {
		fmt.Println("读数据库出错,sql语句=", sqlExpress, "error=", err.Error())
		this.Err = err
		return
	}
	this.ReadInfo.TheRows = rows
	columns, err := rows.Columns()
	if err != nil {
		fmt.Println("取列名出错,sql语句=", sqlExpress, "error=", err.Error())
		this.Err = err
		return
	}

	this.ReadInfo.TheColumns = make(map[string]int)
	//记录列名及其对应的数组下标
	for i, v := range columns {
		this.ReadInfo.TheColumns[v] = i
	}
	colNum := len(columns) //列的数量
	// Make a slice for the values
	values := make([]sql.RawBytes, colNum)
	// rows.Scan wants '[]interface{}' as an argument, so we must copy the
	// references into such a slice
	// See http://code.google.com/p/go-wiki/wiki/InterfaceSlice for details
	scanArgs := make([]interface{}, colNum)
	for i := range values {
		scanArgs[i] = &values[i]
	}

	// Fetch rows
	for rows.Next() {
		// get RawBytes from data
		err = rows.Scan(scanArgs...)
		if err != nil {
			fmt.Println("从rows里取数据出错,sql语句=", sqlExpress, "error=", err.Error())
			this.Err = err
			return
		}

		// Now do something with the data.
		// Here we just print each column as a string.
		var value string
		rowValue := make([]string, colNum)
		for i, col := range values {
			// Here we can check if the value is nil (NULL value)
			if col == nil {
				value = "NULL"
			} else {
				value = string(col)
			}
			rowValue[i] = value
		}
		//把取出来的值放进二维数组中
		this.ReadInfo.ArrValues = append(this.ReadInfo.ArrValues, rowValue)
	}
	this.ReadInfo.RowNum = len(this.ReadInfo.ArrValues)
}

//向数据库写数据
func (this *CADODatabase) WriteToDB(sqlExpress string) {
	result, err := this.TheDB.Exec(sqlExpress)
	if err != nil {
		fmt.Println("写数据库出错,sql语句=", sqlExpress, "error=", err.Error())
		this.Err = err
		return
	}
	this.WriteInfo.LastId, _ = result.LastInsertId()
	this.WriteInfo.AffectedRows, _ = result.RowsAffected()
}

//在ReadFromDB从结果集里获取数据,结果放入value中，所以value要传地址。rowId从0开始为第一行
//成功返回true，失败返回false
func (this *CADODatabase) GetValueByRowIdAndColName(rowId int, colName string, value interface{}) bool {
	//先根据列名获得列的数组下标
	colId, ok := this.ReadInfo.TheColumns[colName]
	if !ok {
		fmt.Println("不存在的列名,colName=", colName)
		return false
	}
	if rowId >= this.ReadInfo.RowNum {
		fmt.Println("行号越界,rowId=", rowId, ",RowNum=", this.ReadInfo.RowNum)
		return false
	}
	//进行数据转换
	strValue := this.ReadInfo.ArrValues[rowId][colId]
	var err error
	switch data := value.(type) {
	case *int:
		*data, err = strconv.Atoi(strValue)
	case *int32:
		var v int64
		v, err = strconv.ParseInt(strValue, 10, 32)
		*data = int32(v)
	case *int64:
		*data, err = strconv.ParseInt(strValue, 10, 64)
	case *uint:
		var v uint64
		v, err = strconv.ParseUint(strValue, 10, 64)
		*data = uint(v)
	case *uint32:
		var v uint64
		v, err = strconv.ParseUint(strValue, 10, 32)
		*data = uint32(v)
	case *uint64:
		*data, err = strconv.ParseUint(strValue, 10, 64)
	case *float32:
		var v float64
		v, err = strconv.ParseFloat(strValue, 32)
		*data = float32(v)
	case *float64:
		*data, err = strconv.ParseFloat(strValue, 64)
	case *string:
		*data = strValue
	}
	if err != nil {
		fmt.Println("数据转换出错")
		return false
	}

	return true
}

type ILogicProcess interface {
	Init(myPool *SingleMsgPool) bool                       //初始化
	ProcessReq(req proto.Message, pDatabase *CADODatabase) //处理来自所绑定的SingleMsgPool发送的报文
	OnPulse(ms uint64)                                     //定时函数，每隔200ms调用一次
}
<commit_msg>避免重复创建<commit_after>package PoolAndAgent

import (
	"database/sql"
	"fmt"
	"strconv"
	"time"

	"github.com/3zheng/railgun/DialManager"
	ListenManager "github.com/3zheng/railgun/TcpListenManager"
	bs_proto "github.com/3zheng/railgun/protodefine"
	bs_types "github.com/3zheng/railgun/protodefine/mytype"
	bs_router "github.com/3zheng/railgun/protodefine/router"
	bs_tcp "github.com/3zheng/railgun/protodefine/tcpnet"

	_ "github.com/go-sql-driver/mysql"
	proto "github.com/golang/protobuf/proto"
)

type NetAgent struct {
	IpAddress string
}

var netAgentList []NetAgent

func (*NetAgent) SendMsg(req *bs_tcp.TCPTransferMsg) {
	var connId uint64 = req.Base.ConnId
	sess := ListenManager.GetSessionByConnId(connId)
	if sess == nil { //判断是否为空
		return
	}
	//往MsgWriteCh里写req
	select {
	case sess.MsgWriteCh <- req:
	case <-time.After(5 * time.Second):
		fmt.Println("严重事件，sess.MsgWriteCh已经阻塞超时5秒, connId =", connId)
	}
}

//创建一个NetAgent,ipAdd是个带端口的ip地址，如果是监听一个端口使用0.0.0.0:port
//NetAgent是TcpManager的映射，在CreateNetAgent并不和TcpManager联系起来，而是在SingleMsgPool的InitAndRun来创建TcpManager的TCP端口监听
func CreateNetAgent(ipAdd string) *NetAgent {
	for _, v := range netAgentList {
		if v.IpAddress == ipAdd {
			fmt.Println("已存在相同IP地址和端口")
			return nil
		}
	}
	agent := new(NetAgent)
	agent.IpAddress = ipAdd
	netAgentList = append(netAgentList, agent)
	return agent
}

type RouterAgent struct {
	IpAddress   string
	DialSession *DialManager.ConnectionSession
}

func CreateRouterAgent(ipAdd string) *RouterAgent {
	agent := new(RouterAgent)
	agent.IpAddress = ipAdd
	return agent
}

//启动运行RouterAgent
func (this *RouterAgent) RunRouterAgent(RouterToLogicChannel chan proto.Message, myAppId uint32, myAppType uint32) {
	fmt.Println("RunRouterAgent() myAppId=", myAppId, "myAppType=", myAppType)
	isClosed := false
	this.DialSession = DialManager.CreateClient(this.IpAddress, RouterToLogicChannel)
	for {
		//在这个router session关闭后，需要不停的尝试重新连接
		for isClosed || this.DialSession == nil {
			select {
			case <-time.After(700 * time.Millisecond): //每700毫秒尝试重新连接一次
			}
			this.DialSession = DialManager.CreateClient(this.IpAddress, RouterToLogicChannel)
			if this.DialSession != nil {
				//重连成功了
				fmt.Println("router Agent重连成功了")
				//这里用break是不能跳出for循环的，只能跳出当前的select，所以只能isClosed = false
				isClosed = false
			}
		}
		//连上router后向router发送注册报文
		req := new(bs_router.RegisterAppReq)
		bs_proto.SetBaseKindAndSubId(req)
		req.AppType = myAppType
		req.AppId = myAppId
		buff, err := proto.Marshal(req)
		if err != nil {
			fmt.Println("序列化RegisterAppReq出错")
		}
		tcpMsg := new(bs_tcp.TCPTransferMsg)
		bs_proto.SetBaseKindAndSubId(tcpMsg)
		tcpMsg.Data = buff
		tcpMsg.DataKindId = uint32(bs_types.CMDKindId_IDKindRouter)
		tcpMsg.DataSubId = uint32(bs_router.CMDID_Router_IDRegisterAppReq)
		this.SendMsg(tcpMsg)
		fmt.Println("向router app发送了注册请求")
		//阻塞在这里直到session关闭
		select {
		case v := <-this.DialSession.Quit:
			if v {
				fmt.Println("router agent的连接已关闭")
				isClosed = true
				this.DialSession = nil
			}
		}
	}
}

func (this *RouterAgent) SendMsg(req *bs_tcp.TCPTransferMsg) {
	var connId uint64 = req.Base.ConnId
	if this.DialSession == nil { //判断是否为空
		return
	}
	//往MsgWriteCh里写req
	select {
	case this.DialSession.MsgWriteCh <- req:
	case <-time.After(5 * time.Second):
		fmt.Println("严重事件，DialSession.MsgWriteCh已经阻塞超时5秒, connId =", connId)
	}

}

type DBReadInfo struct {
	TheRows    *sql.Rows      //如果是读操作，这里是返回的数据集
	TheColumns map[string]int //返回的列名,key是列名，value是表示这个是第几列，在返回的时候好查找
	ArrValues  [][]string     //返回的结果值
	RowNum     int            //行数，其实就是len(ArrValues)
}

func (this *DBReadInfo) Clear() {
	this.TheRows = nil
	this.TheColumns = nil
	this.ArrValues = nil
	this.RowNum = 0
}

type DBWriteInfo struct {
	LastId       int64 //自增列的当前值
	AffectedRows int64 //写操作影响的行数
}

//数据库的映射
type CADODatabase struct {
	DBSourceString string      //mysql的数据库连接串	格式"root:123456@tcp(localhost:3306)/sns?charset=utf8"
	Err            error       //错误
	TheDB          *sql.DB     //数据库对象
	ReadInfo       DBReadInfo  //如果是select的读操作，相关返回结果存这里
	WriteInfo      DBWriteInfo //如果是update或insert的写操作，相关返回结果存这里
}

func CreateADODatabase(DBSourceString string) *CADODatabase {
	DBAgent := new(CADODatabase)
	DBAgent.DBSourceString = DBSourceString
	return DBAgent
}

func (this *CADODatabase) InitDB() {
	db, err := sql.Open("mysql", this.DBSourceString)
	if err != nil {
		fmt.Println("初始化失败，error=", err.Error())
		panic(err.Error())
		this.Err = err
	}
	fmt.Println("数据库连接初始化成功")
	this.TheDB = db
}

//从数据库里读数据
func (this *CADODatabase) ReadFromDB(sqlExpress string) {
	//在读数据前先clear DBReadInfo
	this.ReadInfo.Clear()
	rows, err := this.TheDB.Query(sqlExpress)
	if err != nil {
		fmt.Println("读数据库出错,sql语句=", sqlExpress, "error=", err.Error())
		this.Err = err
		return
	}
	this.ReadInfo.TheRows = rows
	columns, err := rows.Columns()
	if err != nil {
		fmt.Println("取列名出错,sql语句=", sqlExpress, "error=", err.Error())
		this.Err = err
		return
	}

	this.ReadInfo.TheColumns = make(map[string]int)
	//记录列名及其对应的数组下标
	for i, v := range columns {
		this.ReadInfo.TheColumns[v] = i
	}
	colNum := len(columns) //列的数量
	// Make a slice for the values
	values := make([]sql.RawBytes, colNum)
	// rows.Scan wants '[]interface{}' as an argument, so we must copy the
	// references into such a slice
	// See http://code.google.com/p/go-wiki/wiki/InterfaceSlice for details
	scanArgs := make([]interface{}, colNum)
	for i := range values {
		scanArgs[i] = &values[i]
	}

	// Fetch rows
	for rows.Next() {
		// get RawBytes from data
		err = rows.Scan(scanArgs...)
		if err != nil {
			fmt.Println("从rows里取数据出错,sql语句=", sqlExpress, "error=", err.Error())
			this.Err = err
			return
		}

		// Now do something with the data.
		// Here we just print each column as a string.
		var value string
		rowValue := make([]string, colNum)
		for i, col := range values {
			// Here we can check if the value is nil (NULL value)
			if col == nil {
				value = "NULL"
			} else {
				value = string(col)
			}
			rowValue[i] = value
		}
		//把取出来的值放进二维数组中
		this.ReadInfo.ArrValues = append(this.ReadInfo.ArrValues, rowValue)
	}
	this.ReadInfo.RowNum = len(this.ReadInfo.ArrValues)
}

//向数据库写数据
func (this *CADODatabase) WriteToDB(sqlExpress string) {
	result, err := this.TheDB.Exec(sqlExpress)
	if err != nil {
		fmt.Println("写数据库出错,sql语句=", sqlExpress, "error=", err.Error())
		this.Err = err
		return
	}
	this.WriteInfo.LastId, _ = result.LastInsertId()
	this.WriteInfo.AffectedRows, _ = result.RowsAffected()
}

//在ReadFromDB从结果集里获取数据,结果放入value中，所以value要传地址。rowId从0开始为第一行
//成功返回true，失败返回false
func (this *CADODatabase) GetValueByRowIdAndColName(rowId int, colName string, value interface{}) bool {
	//先根据列名获得列的数组下标
	colId, ok := this.ReadInfo.TheColumns[colName]
	if !ok {
		fmt.Println("不存在的列名,colName=", colName)
		return false
	}
	if rowId >= this.ReadInfo.RowNum {
		fmt.Println("行号越界,rowId=", rowId, ",RowNum=", this.ReadInfo.RowNum)
		return false
	}
	//进行数据转换
	strValue := this.ReadInfo.ArrValues[rowId][colId]
	var err error
	switch data := value.(type) {
	case *int:
		*data, err = strconv.Atoi(strValue)
	case *int32:
		var v int64
		v, err = strconv.ParseInt(strValue, 10, 32)
		*data = int32(v)
	case *int64:
		*data, err = strconv.ParseInt(strValue, 10, 64)
	case *uint:
		var v uint64
		v, err = strconv.ParseUint(strValue, 10, 64)
		*data = uint(v)
	case *uint32:
		var v uint64
		v, err = strconv.ParseUint(strValue, 10, 32)
		*data = uint32(v)
	case *uint64:
		*data, err = strconv.ParseUint(strValue, 10, 64)
	case *float32:
		var v float64
		v, err = strconv.ParseFloat(strValue, 32)
		*data = float32(v)
	case *float64:
		*data, err = strconv.ParseFloat(strValue, 64)
	case *string:
		*data = strValue
	}
	if err != nil {
		fmt.Println("数据转换出错")
		return false
	}

	return true
}

type ILogicProcess interface {
	Init(myPool *SingleMsgPool) bool                       //初始化
	ProcessReq(req proto.Message, pDatabase *CADODatabase) //处理来自所绑定的SingleMsgPool发送的报文
	OnPulse(ms uint64)                                     //定时函数，每隔200ms调用一次
}
<|endoftext|>


<commit_before>package middleware

import (
	"bytes"
	"log"
	"net/http"
	"time"
	"code.google.com/p/go.net/context"

	"github.com/vanackere/slim/web"
	"github.com/vanackere/slim/web/util"
)

// Logger is a middleware that logs the start and end of each request, along
// with some useful data about what was requested, what the response status was,
// and how long it took to return. When standard output is a TTY, Logger will
// print in color, otherwise it will print in black and white.
//
// Logger prints a request ID if one is provided.
func Logger(ctx context.Context, w http.ResponseWriter, r *http.Request, next web.Handler) {
	reqID := GetReqID(ctx)

	printStart(reqID, r)

	lw := util.WrapWriter(w)

	t1 := time.Now()
	next.ServeHTTPC(ctx, lw, r)

	if lw.Status() == 0 {
		lw.WriteHeader(http.StatusOK)
	}
	t2 := time.Now()

	printEnd(reqID, lw, t2.Sub(t1))
}

func printStart(reqID string, r *http.Request) {
	var buf bytes.Buffer

	if reqID != "" {
		cW(&buf, bBlack, "[%s] ", reqID)
	}
	buf.WriteString("Started ")
	cW(&buf, bMagenta, "%s ", r.Method)
	cW(&buf, nBlue, "%q ", r.URL.String())
	buf.WriteString("from ")
	buf.WriteString(r.RemoteAddr)

	log.Print(buf.String())
}

func printEnd(reqID string, w util.WriterProxy, dt time.Duration) {
	var buf bytes.Buffer

	if reqID != "" {
		cW(&buf, bBlack, "[%s] ", reqID)
	}
	buf.WriteString("Returning ")
	status := w.Status()
	if status < 200 {
		cW(&buf, bBlue, "%03d", status)
	} else if status < 300 {
		cW(&buf, bGreen, "%03d", status)
	} else if status < 400 {
		cW(&buf, bCyan, "%03d", status)
	} else if status < 500 {
		cW(&buf, bYellow, "%03d", status)
	} else {
		cW(&buf, bRed, "%03d", status)
	}
	buf.WriteString(" in ")
	if dt < 500*time.Millisecond {
		cW(&buf, nGreen, "%s", dt)
	} else if dt < 5*time.Second {
		cW(&buf, nYellow, "%s", dt)
	} else {
		cW(&buf, nRed, "%s", dt)
	}

	log.Print(buf.String())
}
<commit_msg>Add a note about outgrowing Logger<commit_after>package middleware

import (
	"bytes"
	"log"
	"net/http"
	"time"

	"code.google.com/p/go.net/context"

	"github.com/vanackere/slim/web"
	"github.com/vanackere/slim/web/util"
)

// Logger is a middleware that logs the start and end of each request, along
// with some useful data about what was requested, what the response status was,
// and how long it took to return. When standard output is a TTY, Logger will
// print in color, otherwise it will print in black and white.
//
// Logger prints a request ID if one is provided.
//
// Logger has been designed explicitly to be Good Enough for use in small
// applications and for people just getting started with Slim. It is expected
// that applications will eventually outgrow this middleware and replace it with
// a custom request logger, such as one that produces machine-parseable output,
// outputs logs to a different service (e.g., syslog), or formats lines like
// those printed elsewhere in the application.
func Logger(ctx context.Context, w http.ResponseWriter, r *http.Request, next web.Handler) {
	reqID := GetReqID(ctx)

	printStart(reqID, r)

	lw := util.WrapWriter(w)

	t1 := time.Now()
	next.ServeHTTPC(ctx, lw, r)

	if lw.Status() == 0 {
		lw.WriteHeader(http.StatusOK)
	}
	t2 := time.Now()

	printEnd(reqID, lw, t2.Sub(t1))
}

func printStart(reqID string, r *http.Request) {
	var buf bytes.Buffer

	if reqID != "" {
		cW(&buf, bBlack, "[%s] ", reqID)
	}
	buf.WriteString("Started ")
	cW(&buf, bMagenta, "%s ", r.Method)
	cW(&buf, nBlue, "%q ", r.URL.String())
	buf.WriteString("from ")
	buf.WriteString(r.RemoteAddr)

	log.Print(buf.String())
}

func printEnd(reqID string, w util.WriterProxy, dt time.Duration) {
	var buf bytes.Buffer

	if reqID != "" {
		cW(&buf, bBlack, "[%s] ", reqID)
	}
	buf.WriteString("Returning ")
	status := w.Status()
	if status < 200 {
		cW(&buf, bBlue, "%03d", status)
	} else if status < 300 {
		cW(&buf, bGreen, "%03d", status)
	} else if status < 400 {
		cW(&buf, bCyan, "%03d", status)
	} else if status < 500 {
		cW(&buf, bYellow, "%03d", status)
	} else {
		cW(&buf, bRed, "%03d", status)
	}
	buf.WriteString(" in ")
	if dt < 500*time.Millisecond {
		cW(&buf, nGreen, "%s", dt)
	} else if dt < 5*time.Second {
		cW(&buf, nYellow, "%s", dt)
	} else {
		cW(&buf, nRed, "%s", dt)
	}

	log.Print(buf.String())
}
<|endoftext|>