<commit_before>// +build !solaris

package tail

import (
	"fmt"
	"strings"
	"sync"

	"github.com/influxdata/tail"

	"github.com/influxdata/telegraf"
	"github.com/influxdata/telegraf/internal/globpath"
	"github.com/influxdata/telegraf/plugins/inputs"
	"github.com/influxdata/telegraf/plugins/parsers"
)

const (
	defaultWatchMethod = "inotify"
)

type Tail struct {
	Files         []string
	FromBeginning bool
	Pipe          bool
	WatchMethod   string

	tailers []*tail.Tail
	parser  parsers.Parser
	wg      sync.WaitGroup
	acc     telegraf.Accumulator

	sync.Mutex
}

func NewTail() *Tail {
	return &Tail{
		FromBeginning: false,
	}
}

const sampleConfig = `
  ## files to tail.
  ## These accept standard unix glob matching rules, but with the addition of
  ## ** as a "super asterisk". ie:
  ##   "/var/log/**.log"  -> recursively find all .log files in /var/log
  ##   "/var/log/*/*.log" -> find all .log files with a parent dir in /var/log
  ##   "/var/log/apache.log" -> just tail the apache log file
  ##
  ## See https://github.com/gobwas/glob for more examples
  ##
  files = ["/var/mymetrics.out"]
  ## Read file from beginning.
  from_beginning = false
  ## Whether file is a named pipe
  pipe = false

  ## Method used to watch for file updates.  Can be either "inotify" or "poll".
  # watch_method = "inotify"

  ## Data format to consume.
  ## Each data format has its own unique set of configuration options, read
  ## more about them here:
  ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md
  data_format = "influx"
`

func (t *Tail) SampleConfig() string {
	return sampleConfig
}

func (t *Tail) Description() string {
	return "Stream a log file, like the tail -f command"
}

func (t *Tail) Gather(acc telegraf.Accumulator) error {
	return nil
}

func (t *Tail) Start(acc telegraf.Accumulator) error {
	t.Lock()
	defer t.Unlock()

	t.acc = acc

	var seek *tail.SeekInfo
	if !t.Pipe && !t.FromBeginning {
		seek = &tail.SeekInfo{
			Whence: 2,
			Offset: 0,
		}
	}

	var poll bool
	if t.WatchMethod == "poll" {
		poll = true
	}

	// Create a "tailer" for each file
	for _, filepath := range t.Files {
		g, err := globpath.Compile(filepath)
		if err != nil {
			t.acc.AddError(fmt.Errorf("E! Error Glob %s failed to compile, %s", filepath, err))
		}
		for file, _ := range g.Match() {
			tailer, err := tail.TailFile(file,
				tail.Config{
					ReOpen:    true,
					Follow:    true,
					Location:  seek,
					MustExist: true,
					Poll:      poll,
					Pipe:      t.Pipe,
					Logger:    tail.DiscardingLogger,
				})
			if err != nil {
				acc.AddError(err)
				continue
			}
			// create a goroutine for each "tailer"
			t.wg.Add(1)
			go t.receiver(tailer)
			t.tailers = append(t.tailers, tailer)
		}
	}

	return nil
}

// this is launched as a goroutine to continuously watch a tailed logfile
// for changes, parse any incoming msgs, and add to the accumulator.
func (t *Tail) receiver(tailer *tail.Tail) {
	defer t.wg.Done()

	var m telegraf.Metric
	var err error
	var line *tail.Line
	for line = range tailer.Lines {
		if line.Err != nil {
			t.acc.AddError(fmt.Errorf("E! Error tailing file %s, Error: %s\n",
				tailer.Filename, err))
			continue
		}
		// Fix up files with Windows line endings.
		text := strings.TrimRight(line.Text, "\r")

		m, err = t.parser.ParseLine(text)
		if err == nil {
			if m != nil {
				tags := m.Tags()
				tags["path"] = tailer.Filename
				t.acc.AddFields(m.Name(), m.Fields(), tags, m.Time())
			}
		} else {
			t.acc.AddError(fmt.Errorf("E! Malformed log line in %s: [%s], Error: %s\n",
				tailer.Filename, line.Text, err))
		}
	}
	if err := tailer.Err(); err != nil {
		t.acc.AddError(fmt.Errorf("E! Error tailing file %s, Error: %s\n",
			tailer.Filename, err))
	}
}

func (t *Tail) Stop() {
	t.Lock()
	defer t.Unlock()

	for _, tailer := range t.tailers {
		err := tailer.Stop()
		if err != nil {
			t.acc.AddError(fmt.Errorf("E! Error stopping tail on file %s\n", tailer.Filename))
		}
		tailer.Cleanup()
	}
	t.wg.Wait()
}

func (t *Tail) SetParser(parser parsers.Parser) {
	t.parser = parser
}

func init() {
	inputs.Add("tail", func() telegraf.Input {
		return NewTail()
	})
}
<commit_msg>Support tailing files created after startup in tail input (#4704)<commit_after>// +build !solaris

package tail

import (
	"fmt"
	"strings"
	"sync"

	"github.com/influxdata/tail"

	"github.com/influxdata/telegraf"
	"github.com/influxdata/telegraf/internal/globpath"
	"github.com/influxdata/telegraf/plugins/inputs"
	"github.com/influxdata/telegraf/plugins/parsers"
)

const (
	defaultWatchMethod = "inotify"
)

type Tail struct {
	Files         []string
	FromBeginning bool
	Pipe          bool
	WatchMethod   string

	tailers map[string]*tail.Tail
	parser  parsers.Parser
	wg      sync.WaitGroup
	acc     telegraf.Accumulator

	sync.Mutex
}

func NewTail() *Tail {
	return &Tail{
		FromBeginning: false,
	}
}

const sampleConfig = `
  ## files to tail.
  ## These accept standard unix glob matching rules, but with the addition of
  ## ** as a "super asterisk". ie:
  ##   "/var/log/**.log"  -> recursively find all .log files in /var/log
  ##   "/var/log/*/*.log" -> find all .log files with a parent dir in /var/log
  ##   "/var/log/apache.log" -> just tail the apache log file
  ##
  ## See https://github.com/gobwas/glob for more examples
  ##
  files = ["/var/mymetrics.out"]
  ## Read file from beginning.
  from_beginning = false
  ## Whether file is a named pipe
  pipe = false

  ## Method used to watch for file updates.  Can be either "inotify" or "poll".
  # watch_method = "inotify"

  ## Data format to consume.
  ## Each data format has its own unique set of configuration options, read
  ## more about them here:
  ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md
  data_format = "influx"
`

func (t *Tail) SampleConfig() string {
	return sampleConfig
}

func (t *Tail) Description() string {
	return "Stream a log file, like the tail -f command"
}

func (t *Tail) Gather(acc telegraf.Accumulator) error {
	t.Lock()
	defer t.Unlock()

	return t.tailNewFiles(true)
}

func (t *Tail) Start(acc telegraf.Accumulator) error {
	t.Lock()
	defer t.Unlock()

	t.acc = acc
	t.tailers = make(map[string]*tail.Tail)

	return t.tailNewFiles(t.FromBeginning)
}

func (t *Tail) tailNewFiles(fromBeginning bool) error {
	var seek *tail.SeekInfo
	if !t.Pipe && !fromBeginning {
		seek = &tail.SeekInfo{
			Whence: 2,
			Offset: 0,
		}
	}

	var poll bool
	if t.WatchMethod == "poll" {
		poll = true
	}

	// Create a "tailer" for each file
	for _, filepath := range t.Files {
		g, err := globpath.Compile(filepath)
		if err != nil {
			t.acc.AddError(fmt.Errorf("E! Error Glob %s failed to compile, %s", filepath, err))
		}
		for file, _ := range g.Match() {
			if _, ok := t.tailers[file]; ok {
				// we're already tailing this file
				continue
			}

			tailer, err := tail.TailFile(file,
				tail.Config{
					ReOpen:    true,
					Follow:    true,
					Location:  seek,
					MustExist: true,
					Poll:      poll,
					Pipe:      t.Pipe,
					Logger:    tail.DiscardingLogger,
				})
			if err != nil {
				t.acc.AddError(err)
				continue
			}
			// create a goroutine for each "tailer"
			t.wg.Add(1)
			go t.receiver(tailer)
			t.tailers[file] = tailer
		}
	}
	return nil
}

// this is launched as a goroutine to continuously watch a tailed logfile
// for changes, parse any incoming msgs, and add to the accumulator.
func (t *Tail) receiver(tailer *tail.Tail) {
	defer t.wg.Done()

	var m telegraf.Metric
	var err error
	var line *tail.Line
	for line = range tailer.Lines {
		if line.Err != nil {
			t.acc.AddError(fmt.Errorf("E! Error tailing file %s, Error: %s\n",
				tailer.Filename, err))
			continue
		}
		// Fix up files with Windows line endings.
		text := strings.TrimRight(line.Text, "\r")

		m, err = t.parser.ParseLine(text)
		if err == nil {
			if m != nil {
				tags := m.Tags()
				tags["path"] = tailer.Filename
				t.acc.AddFields(m.Name(), m.Fields(), tags, m.Time())
			}
		} else {
			t.acc.AddError(fmt.Errorf("E! Malformed log line in %s: [%s], Error: %s\n",
				tailer.Filename, line.Text, err))
		}
	}
	if err := tailer.Err(); err != nil {
		t.acc.AddError(fmt.Errorf("E! Error tailing file %s, Error: %s\n",
			tailer.Filename, err))
	}
}

func (t *Tail) Stop() {
	t.Lock()
	defer t.Unlock()

	for _, tailer := range t.tailers {
		err := tailer.Stop()
		if err != nil {
			t.acc.AddError(fmt.Errorf("E! Error stopping tail on file %s\n", tailer.Filename))
		}
		tailer.Cleanup()
	}
	t.wg.Wait()
}

func (t *Tail) SetParser(parser parsers.Parser) {
	t.parser = parser
}

func init() {
	inputs.Add("tail", func() telegraf.Input {
		return NewTail()
	})
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/hex"
	"fmt"
	"io"
	"log"
	"net/http"
	"time"

	"github.com/gorilla/mux"
	"labix.org/v2/mgo/bson"
)

func get(w http.ResponseWriter, req *http.Request) {
	vars := mux.Vars(req)

	// validate _id
	d, e := hex.DecodeString(vars["_id"])
	if e != nil || len(d) != 12 {
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte("400 Bad Request. Error: " + e.Error()))
		return
	}

	// define main variables
	_id := bson.ObjectIdHex(vars["_id"])
	query := conf.DB.C(vars["coll"] + ".files").FindId(_id)
	meta := bson.M{}
	err := query.One(&meta)

	// found file or not
	if err != nil {
		if err.Error() == "not found" {
			w.WriteHeader(http.StatusNotFound)
			w.Write([]byte("404 Not Found"))
		} else {
			w.WriteHeader(http.StatusBadRequest)
			w.Write([]byte("400 Bad Request. Error: " + err.Error()))
		}
		return
	}

	// check cache headers & response
	tt := meta["uploadDate"].(time.Time)
	fmt.Printf("GET %s/%s/%s (%s) ", conf.Prefix, vars["coll"], vars["_id"], tt.Format(time.RFC822))

	if h := req.Header.Get("If-Modified-Since"); h == tt.Format(time.RFC822) {
		w.WriteHeader(http.StatusNotModified)
		w.Write([]byte("304 Not Modified"))
		return
	}

	// get file
	file, err := conf.DB.GridFS(vars["coll"]).OpenId(_id)
	if err != nil {
		log.Fatal(err.Error())
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte("400 Bad Request"))
		return
	}

	w.Header().Add("Accept-Ranges", "bytes")
	w.Header().Add("ETag", vars["_id"])
	w.Header().Add("Date", file.UploadDate().Format(time.RFC822))
	w.Header().Add("Last-Modified", file.UploadDate().Format(time.RFC822))
	w.Header().Add("Cache-Control", "public, max-age=31536000")
	w.Header().Add("Content-Type", file.ContentType())
	// w.Header().Add("Content-Length", fmt.Sprintf("%d", file.Size()))
	if file.ContentType() == "application/octet-stream" {
		w.Header().Add("Content-Disposition", "attachment; filename='"+file.Name()+"'")
	}

	// check to crop/resize
	req.ParseForm()
	cr, isCrop := req.Form["crop"]
	rsz, isResize := req.Form["resize"]

	isIn := ^in([]string{"image/png", "image/jpeg"}, file.ContentType()) != 0

	if isCrop && isIn && cr != nil {
		parsed := parseParams(cr[0])
		if parsed != nil {
			err = crop(w, file, parsed)
			fmt.Println("croped for:", parsed)
			if err != nil {
				fmt.Println("GET err:", err.Error())
			}
			return
		}
	} else if isResize && isIn && rsz != nil {
		parsed := parseParams(rsz[0])
		if parsed != nil {
			err = resize(w, file, parsed)
			fmt.Println("resized for:", parsed)
			if err != nil {
				fmt.Println("GET err:", err.Error())
			}
			return
		}
	} else {
		io.Copy(w, file)
	}
}
<commit_msg>logging info changed<commit_after>package main

import (
	"encoding/hex"
	"fmt"
	"io"
	"log"
	"net/http"
	"time"

	"github.com/gorilla/mux"
	"labix.org/v2/mgo/bson"
)

func get(w http.ResponseWriter, req *http.Request) {
	vars := mux.Vars(req)

	// validate _id
	d, e := hex.DecodeString(vars["_id"])
	if e != nil || len(d) != 12 {
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte("400 Bad Request. Error: " + e.Error()))
		return
	}

	// define main variables
	_id := bson.ObjectIdHex(vars["_id"])
	query := conf.DB.C(vars["coll"] + ".files").FindId(_id)
	meta := bson.M{}
	err := query.One(&meta)

	// found file or not
	if err != nil {
		if err.Error() == "not found" {
			w.WriteHeader(http.StatusNotFound)
			w.Write([]byte("404 Not Found"))
		} else {
			w.WriteHeader(http.StatusBadRequest)
			w.Write([]byte("400 Bad Request. Error: " + err.Error()))
		}
		return
	}

	// check cache headers & response
	tt := meta["uploadDate"].(time.Time)
	fmt.Printf("GET %s/%s/%s (%s) ", conf.Prefix, vars["coll"], vars["_id"], tt.Format(time.RFC822))

	if h := req.Header.Get("If-Modified-Since"); h == tt.Format(time.RFC822) {
		w.WriteHeader(http.StatusNotModified)
		w.Write([]byte("304 Not Modified"))
		fmt.Println("304")
		return
	}

	// get file
	file, err := conf.DB.GridFS(vars["coll"]).OpenId(_id)
	if err != nil {
		log.Fatal(err.Error())
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte("400 Bad Request"))
		return
	}

	w.Header().Add("Accept-Ranges", "bytes")
	w.Header().Add("ETag", vars["_id"])
	w.Header().Add("Date", file.UploadDate().Format(time.RFC822))
	w.Header().Add("Last-Modified", file.UploadDate().Format(time.RFC822))
	w.Header().Add("Cache-Control", "public, max-age=31536000")
	w.Header().Add("Content-Type", file.ContentType())
	// w.Header().Add("Content-Length", fmt.Sprintf("%d", file.Size()))
	if file.ContentType() == "application/octet-stream" {
		w.Header().Add("Content-Disposition", "attachment; filename='"+file.Name()+"'")
	}

	// check to crop/resize
	req.ParseForm()
	cr, isCrop := req.Form["crop"]
	rsz, isResize := req.Form["resize"]

	isIn := ^in([]string{"image/png", "image/jpeg"}, file.ContentType()) != 0

	if isCrop && isIn && cr != nil {
		parsed := parseParams(cr[0])
		if parsed != nil {
			err = crop(w, file, parsed)
			fmt.Println("croped for:", parsed)
			if err != nil {
				fmt.Println("GET err:", err.Error())
			}
			return
		}
	} else if isResize && isIn && rsz != nil {
		parsed := parseParams(rsz[0])
		if parsed != nil {
			err = resize(w, file, parsed)
			fmt.Println("resized for:", parsed)
			if err != nil {
				fmt.Println("GET err:", err.Error())
			}
			return
		}
	} else {
		fmt.Println("as is")
		io.Copy(w, file)
	}
}
<|endoftext|>


<commit_before>// Copyright 2014 Manu Martinez-Almeida.  All rights reserved.
// Use of this source code is governed by a MIT style
// license that can be found in the LICENSE file.

package gin

import (
	"html/template"
	"net"
	"net/http"
	"os"
	"sync"

	"github.com/gin-gonic/gin/render"
)

// Framework's version
const Version = "v1.0rc2"

var default404Body = []byte("404 page not found")
var default405Body = []byte("405 method not allowed")

type HandlerFunc func(*Context)
type HandlersChain []HandlerFunc

// Last returns the last handler in the chain. ie. the last handler is the main own.
func (c HandlersChain) Last() HandlerFunc {
	length := len(c)
	if length > 0 {
		return c[length-1]
	}
	return nil
}

type (
	RoutesInfo []RouteInfo
	RouteInfo  struct {
		Method  string
		Path    string
		Handler string
	}

	// Engine is the framework's instance, it contains the muxer, middleware and configuration settings.
	// Create an instance of Engine, by using New() or Default()
	Engine struct {
		RouterGroup
		HTMLRender  render.HTMLRender
		allNoRoute  HandlersChain
		allNoMethod HandlersChain
		noRoute     HandlersChain
		noMethod    HandlersChain
		pool        sync.Pool
		trees       methodTrees

		// Enables automatic redirection if the current route can't be matched but a
		// handler for the path with (without) the trailing slash exists.
		// For example if /foo/ is requested but a route only exists for /foo, the
		// client is redirected to /foo with http status code 301 for GET requests
		// and 307 for all other request methods.
		RedirectTrailingSlash bool

		// If enabled, the router tries to fix the current request path, if no
		// handle is registered for it.
		// First superfluous path elements like ../ or // are removed.
		// Afterwards the router does a case-insensitive lookup of the cleaned path.
		// If a handle can be found for this route, the router makes a redirection
		// to the corrected path with status code 301 for GET requests and 307 for
		// all other request methods.
		// For example /FOO and /..//Foo could be redirected to /foo.
		// RedirectTrailingSlash is independent of this option.
		RedirectFixedPath bool

		// If enabled, the router checks if another method is allowed for the
		// current route, if the current request can not be routed.
		// If this is the case, the request is answered with 'Method Not Allowed'
		// and HTTP status code 405.
		// If no other Method is allowed, the request is delegated to the NotFound
		// handler.
		HandleMethodNotAllowed bool
		ForwardedByClientIP    bool
	}
)

var _ IRouter = &Engine{}

// New returns a new blank Engine instance without any middleware attached.
// By default the configuration is:
// - RedirectTrailingSlash:  true
// - RedirectFixedPath:      false
// - HandleMethodNotAllowed: false
// - ForwardedByClientIP:    true
func New() *Engine {
	debugPrintWARNINGNew()
	engine := &Engine{
		RouterGroup: RouterGroup{
			Handlers: nil,
			basePath: "/",
			root:     true,
		},
		RedirectTrailingSlash:  true,
		RedirectFixedPath:      false,
		HandleMethodNotAllowed: false,
		ForwardedByClientIP:    true,
		trees:                  make(methodTrees, 0, 9),
	}
	engine.RouterGroup.engine = engine
	engine.pool.New = func() interface{} {
		return engine.allocateContext()
	}
	return engine
}

// Default returns an Engine instance with the Logger and Recovery middleware already attached.
func Default() *Engine {
	engine := New()
	engine.Use(Recovery(), Logger())
	return engine
}

func (engine *Engine) allocateContext() *Context {
	return &Context{engine: engine}
}

func (engine *Engine) LoadHTMLGlob(pattern string) {
	if IsDebugging() {
		debugPrintLoadTemplate(template.Must(template.ParseGlob(pattern)))
		engine.HTMLRender = render.HTMLDebug{Glob: pattern}
	} else {
		templ := template.Must(template.ParseGlob(pattern))
		engine.SetHTMLTemplate(templ)
	}
}

func (engine *Engine) LoadHTMLFiles(files ...string) {
	if IsDebugging() {
		engine.HTMLRender = render.HTMLDebug{Files: files}
	} else {
		templ := template.Must(template.ParseFiles(files...))
		engine.SetHTMLTemplate(templ)
	}
}

func (engine *Engine) SetHTMLTemplate(templ *template.Template) {
	if len(engine.trees) > 0 {
		debugPrintWARNINGSetHTMLTemplate()
	}
	engine.HTMLRender = render.HTMLProduction{Template: templ}
}

// Adds handlers for NoRoute. It return a 404 code by default.
func (engine *Engine) NoRoute(handlers ...HandlerFunc) {
	engine.noRoute = handlers
	engine.rebuild404Handlers()
}

// Sets the handlers called when... TODO
func (engine *Engine) NoMethod(handlers ...HandlerFunc) {
	engine.noMethod = handlers
	engine.rebuild405Handlers()
}

// Attachs a global middleware to the router. ie. the middleware attached though Use() will be
// included in the handlers chain for every single request. Even 404, 405, static files...
// For example, this is the right place for a logger or error management middleware.
func (engine *Engine) Use(middleware ...HandlerFunc) IRoutes {
	engine.RouterGroup.Use(middleware...)
	engine.rebuild404Handlers()
	engine.rebuild405Handlers()
	return engine
}

func (engine *Engine) rebuild404Handlers() {
	engine.allNoRoute = engine.combineHandlers(engine.noRoute)
}

func (engine *Engine) rebuild405Handlers() {
	engine.allNoMethod = engine.combineHandlers(engine.noMethod)
}

func (engine *Engine) addRoute(method, path string, handlers HandlersChain) {
	debugPrintRoute(method, path, handlers)

	if path[0] != '/' {
		panic("path must begin with '/'")
	}
	if method == "" {
		panic("HTTP method can not be empty")
	}
	if len(handlers) == 0 {
		panic("there must be at least one handler")
	}

	root := engine.trees.get(method)
	if root == nil {
		root = new(node)
		engine.trees = append(engine.trees, methodTree{
			method: method,
			root:   root,
		})
	}
	root.addRoute(path, handlers)
}

// Routes returns a slice of registered routes, including some useful information, such as:
// the http method, path and the handler name.
func (engine *Engine) Routes() (routes RoutesInfo) {
	for _, tree := range engine.trees {
		routes = iterate("", tree.method, routes, tree.root)
	}
	return routes
}

func iterate(path, method string, routes RoutesInfo, root *node) RoutesInfo {
	path += root.path
	if len(root.handlers) > 0 {
		routes = append(routes, RouteInfo{
			Method:  method,
			Path:    path,
			Handler: nameOfFunction(root.handlers.Last()),
		})
	}
	for _, child := range root.children {
		routes = iterate(path, method, routes, child)
	}
	return routes
}

// Run attaches the router to a http.Server and starts listening and serving HTTP requests.
// It is a shortcut for http.ListenAndServe(addr, router)
// Note: this method will block the calling goroutine undefinitelly unless an error happens.
func (engine *Engine) Run(addr ...string) (err error) {
	defer func() { debugPrintError(err) }()

	address := resolveAddress(addr)
	debugPrint("Listening and serving HTTP on %s\n", address)
	err = http.ListenAndServe(address, engine)
	return
}

// RunTLS attaches the router to a http.Server and starts listening and serving HTTPS (secure) requests.
// It is a shortcut for http.ListenAndServeTLS(addr, certFile, keyFile, router)
// Note: this method will block the calling goroutine undefinitelly unless an error happens.
func (engine *Engine) RunTLS(addr string, certFile string, keyFile string) (err error) {
	debugPrint("Listening and serving HTTPS on %s\n", addr)
	defer func() { debugPrintError(err) }()

	err = http.ListenAndServeTLS(addr, certFile, keyFile, engine)
	return
}

// RunUnix attaches the router to a http.Server and starts listening and serving HTTP requests
// through the specified unix socket (ie. a file).
// Note: this method will block the calling goroutine undefinitelly unless an error happens.
func (engine *Engine) RunUnix(file string) (err error) {
	debugPrint("Listening and serving HTTP on unix:/%s", file)
	defer func() { debugPrintError(err) }()

	os.Remove(file)
	listener, err := net.Listen("unix", file)
	if err != nil {
		return
	}
	defer listener.Close()
	err = http.Serve(listener, engine)
	return
}

// Conforms to the http.Handler interface.
func (engine *Engine) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	c := engine.pool.Get().(*Context)
	c.writermem.reset(w)
	c.Request = req
	c.reset()

	engine.handleHTTPRequest(c)

	engine.pool.Put(c)
}

func (engine *Engine) handleHTTPRequest(context *Context) {
	httpMethod := context.Request.Method
	path := context.Request.URL.Path

	// Find root of the tree for the given HTTP method
	t := engine.trees
	for i, tl := 0, len(t); i < tl; i++ {
		if t[i].method == httpMethod {
			root := t[i].root
			// Find route in tree
			handlers, params, tsr := root.getValue(path, context.Params)
			if handlers != nil {
				context.handlers = handlers
				context.Params = params
				context.Next()
				context.writermem.WriteHeaderNow()
				return

			} else if httpMethod != "CONNECT" && path != "/" {
				if tsr && engine.RedirectTrailingSlash {
					redirectTrailingSlash(context)
					return
				}
				if engine.RedirectFixedPath && redirectFixedPath(context, root, engine.RedirectFixedPath) {
					return
				}
			}
			break
		}
	}

	// TODO: unit test
	if engine.HandleMethodNotAllowed {
		for _, tree := range engine.trees {
			if tree.method != httpMethod {
				if handlers, _, _ := tree.root.getValue(path, nil); handlers != nil {
					context.handlers = engine.allNoMethod
					serveError(context, 405, default405Body)
					return
				}
			}
		}
	}
	context.handlers = engine.allNoRoute
	serveError(context, 404, default404Body)
}

var mimePlain = []string{MIMEPlain}

func serveError(c *Context, code int, defaultMessage []byte) {
	c.writermem.status = code
	c.Next()
	if !c.writermem.Written() {
		if c.writermem.Status() == code {
			c.writermem.Header()["Content-Type"] = mimePlain
			c.Writer.Write(defaultMessage)
		} else {
			c.writermem.WriteHeaderNow()
		}
	}
}

func redirectTrailingSlash(c *Context) {
	req := c.Request
	path := req.URL.Path
	code := 301 // Permanent redirect, request with GET method
	if req.Method != "GET" {
		code = 307
	}

	if len(path) > 1 && path[len(path)-1] == '/' {
		req.URL.Path = path[:len(path)-1]
	} else {
		req.URL.Path = path + "/"
	}
	debugPrint("redirecting request %d: %s --> %s", code, path, req.URL.String())
	http.Redirect(c.Writer, req, req.URL.String(), code)
	c.writermem.WriteHeaderNow()
}

func redirectFixedPath(c *Context, root *node, trailingSlash bool) bool {
	req := c.Request
	path := req.URL.Path

	fixedPath, found := root.findCaseInsensitivePath(
		cleanPath(path),
		trailingSlash,
	)
	if found {
		code := 301 // Permanent redirect, request with GET method
		if req.Method != "GET" {
			code = 307
		}
		req.URL.Path = string(fixedPath)
		debugPrint("redirecting request %d: %s --> %s", code, path, req.URL.String())
		http.Redirect(c.Writer, req, req.URL.String(), code)
		c.writermem.WriteHeaderNow()
		return true
	}
	return false
}
<commit_msg>undefinitely -> indefinitely<commit_after>// Copyright 2014 Manu Martinez-Almeida.  All rights reserved.
// Use of this source code is governed by a MIT style
// license that can be found in the LICENSE file.

package gin

import (
	"html/template"
	"net"
	"net/http"
	"os"
	"sync"

	"github.com/gin-gonic/gin/render"
)

// Framework's version
const Version = "v1.0rc2"

var default404Body = []byte("404 page not found")
var default405Body = []byte("405 method not allowed")

type HandlerFunc func(*Context)
type HandlersChain []HandlerFunc

// Last returns the last handler in the chain. ie. the last handler is the main own.
func (c HandlersChain) Last() HandlerFunc {
	length := len(c)
	if length > 0 {
		return c[length-1]
	}
	return nil
}

type (
	RoutesInfo []RouteInfo
	RouteInfo  struct {
		Method  string
		Path    string
		Handler string
	}

	// Engine is the framework's instance, it contains the muxer, middleware and configuration settings.
	// Create an instance of Engine, by using New() or Default()
	Engine struct {
		RouterGroup
		HTMLRender  render.HTMLRender
		allNoRoute  HandlersChain
		allNoMethod HandlersChain
		noRoute     HandlersChain
		noMethod    HandlersChain
		pool        sync.Pool
		trees       methodTrees

		// Enables automatic redirection if the current route can't be matched but a
		// handler for the path with (without) the trailing slash exists.
		// For example if /foo/ is requested but a route only exists for /foo, the
		// client is redirected to /foo with http status code 301 for GET requests
		// and 307 for all other request methods.
		RedirectTrailingSlash bool

		// If enabled, the router tries to fix the current request path, if no
		// handle is registered for it.
		// First superfluous path elements like ../ or // are removed.
		// Afterwards the router does a case-insensitive lookup of the cleaned path.
		// If a handle can be found for this route, the router makes a redirection
		// to the corrected path with status code 301 for GET requests and 307 for
		// all other request methods.
		// For example /FOO and /..//Foo could be redirected to /foo.
		// RedirectTrailingSlash is independent of this option.
		RedirectFixedPath bool

		// If enabled, the router checks if another method is allowed for the
		// current route, if the current request can not be routed.
		// If this is the case, the request is answered with 'Method Not Allowed'
		// and HTTP status code 405.
		// If no other Method is allowed, the request is delegated to the NotFound
		// handler.
		HandleMethodNotAllowed bool
		ForwardedByClientIP    bool
	}
)

var _ IRouter = &Engine{}

// New returns a new blank Engine instance without any middleware attached.
// By default the configuration is:
// - RedirectTrailingSlash:  true
// - RedirectFixedPath:      false
// - HandleMethodNotAllowed: false
// - ForwardedByClientIP:    true
func New() *Engine {
	debugPrintWARNINGNew()
	engine := &Engine{
		RouterGroup: RouterGroup{
			Handlers: nil,
			basePath: "/",
			root:     true,
		},
		RedirectTrailingSlash:  true,
		RedirectFixedPath:      false,
		HandleMethodNotAllowed: false,
		ForwardedByClientIP:    true,
		trees:                  make(methodTrees, 0, 9),
	}
	engine.RouterGroup.engine = engine
	engine.pool.New = func() interface{} {
		return engine.allocateContext()
	}
	return engine
}

// Default returns an Engine instance with the Logger and Recovery middleware already attached.
func Default() *Engine {
	engine := New()
	engine.Use(Recovery(), Logger())
	return engine
}

func (engine *Engine) allocateContext() *Context {
	return &Context{engine: engine}
}

func (engine *Engine) LoadHTMLGlob(pattern string) {
	if IsDebugging() {
		debugPrintLoadTemplate(template.Must(template.ParseGlob(pattern)))
		engine.HTMLRender = render.HTMLDebug{Glob: pattern}
	} else {
		templ := template.Must(template.ParseGlob(pattern))
		engine.SetHTMLTemplate(templ)
	}
}

func (engine *Engine) LoadHTMLFiles(files ...string) {
	if IsDebugging() {
		engine.HTMLRender = render.HTMLDebug{Files: files}
	} else {
		templ := template.Must(template.ParseFiles(files...))
		engine.SetHTMLTemplate(templ)
	}
}

func (engine *Engine) SetHTMLTemplate(templ *template.Template) {
	if len(engine.trees) > 0 {
		debugPrintWARNINGSetHTMLTemplate()
	}
	engine.HTMLRender = render.HTMLProduction{Template: templ}
}

// Adds handlers for NoRoute. It return a 404 code by default.
func (engine *Engine) NoRoute(handlers ...HandlerFunc) {
	engine.noRoute = handlers
	engine.rebuild404Handlers()
}

// Sets the handlers called when... TODO
func (engine *Engine) NoMethod(handlers ...HandlerFunc) {
	engine.noMethod = handlers
	engine.rebuild405Handlers()
}

// Attachs a global middleware to the router. ie. the middleware attached though Use() will be
// included in the handlers chain for every single request. Even 404, 405, static files...
// For example, this is the right place for a logger or error management middleware.
func (engine *Engine) Use(middleware ...HandlerFunc) IRoutes {
	engine.RouterGroup.Use(middleware...)
	engine.rebuild404Handlers()
	engine.rebuild405Handlers()
	return engine
}

func (engine *Engine) rebuild404Handlers() {
	engine.allNoRoute = engine.combineHandlers(engine.noRoute)
}

func (engine *Engine) rebuild405Handlers() {
	engine.allNoMethod = engine.combineHandlers(engine.noMethod)
}

func (engine *Engine) addRoute(method, path string, handlers HandlersChain) {
	debugPrintRoute(method, path, handlers)

	if path[0] != '/' {
		panic("path must begin with '/'")
	}
	if method == "" {
		panic("HTTP method can not be empty")
	}
	if len(handlers) == 0 {
		panic("there must be at least one handler")
	}

	root := engine.trees.get(method)
	if root == nil {
		root = new(node)
		engine.trees = append(engine.trees, methodTree{
			method: method,
			root:   root,
		})
	}
	root.addRoute(path, handlers)
}

// Routes returns a slice of registered routes, including some useful information, such as:
// the http method, path and the handler name.
func (engine *Engine) Routes() (routes RoutesInfo) {
	for _, tree := range engine.trees {
		routes = iterate("", tree.method, routes, tree.root)
	}
	return routes
}

func iterate(path, method string, routes RoutesInfo, root *node) RoutesInfo {
	path += root.path
	if len(root.handlers) > 0 {
		routes = append(routes, RouteInfo{
			Method:  method,
			Path:    path,
			Handler: nameOfFunction(root.handlers.Last()),
		})
	}
	for _, child := range root.children {
		routes = iterate(path, method, routes, child)
	}
	return routes
}

// Run attaches the router to a http.Server and starts listening and serving HTTP requests.
// It is a shortcut for http.ListenAndServe(addr, router)
// Note: this method will block the calling goroutine indefinitely unless an error happens.
func (engine *Engine) Run(addr ...string) (err error) {
	defer func() { debugPrintError(err) }()

	address := resolveAddress(addr)
	debugPrint("Listening and serving HTTP on %s\n", address)
	err = http.ListenAndServe(address, engine)
	return
}

// RunTLS attaches the router to a http.Server and starts listening and serving HTTPS (secure) requests.
// It is a shortcut for http.ListenAndServeTLS(addr, certFile, keyFile, router)
// Note: this method will block the calling goroutine indefinitely unless an error happens.
func (engine *Engine) RunTLS(addr string, certFile string, keyFile string) (err error) {
	debugPrint("Listening and serving HTTPS on %s\n", addr)
	defer func() { debugPrintError(err) }()

	err = http.ListenAndServeTLS(addr, certFile, keyFile, engine)
	return
}

// RunUnix attaches the router to a http.Server and starts listening and serving HTTP requests
// through the specified unix socket (ie. a file).
// Note: this method will block the calling goroutine indefinitely unless an error happens.
func (engine *Engine) RunUnix(file string) (err error) {
	debugPrint("Listening and serving HTTP on unix:/%s", file)
	defer func() { debugPrintError(err) }()

	os.Remove(file)
	listener, err := net.Listen("unix", file)
	if err != nil {
		return
	}
	defer listener.Close()
	err = http.Serve(listener, engine)
	return
}

// Conforms to the http.Handler interface.
func (engine *Engine) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	c := engine.pool.Get().(*Context)
	c.writermem.reset(w)
	c.Request = req
	c.reset()

	engine.handleHTTPRequest(c)

	engine.pool.Put(c)
}

func (engine *Engine) handleHTTPRequest(context *Context) {
	httpMethod := context.Request.Method
	path := context.Request.URL.Path

	// Find root of the tree for the given HTTP method
	t := engine.trees
	for i, tl := 0, len(t); i < tl; i++ {
		if t[i].method == httpMethod {
			root := t[i].root
			// Find route in tree
			handlers, params, tsr := root.getValue(path, context.Params)
			if handlers != nil {
				context.handlers = handlers
				context.Params = params
				context.Next()
				context.writermem.WriteHeaderNow()
				return

			} else if httpMethod != "CONNECT" && path != "/" {
				if tsr && engine.RedirectTrailingSlash {
					redirectTrailingSlash(context)
					return
				}
				if engine.RedirectFixedPath && redirectFixedPath(context, root, engine.RedirectFixedPath) {
					return
				}
			}
			break
		}
	}

	// TODO: unit test
	if engine.HandleMethodNotAllowed {
		for _, tree := range engine.trees {
			if tree.method != httpMethod {
				if handlers, _, _ := tree.root.getValue(path, nil); handlers != nil {
					context.handlers = engine.allNoMethod
					serveError(context, 405, default405Body)
					return
				}
			}
		}
	}
	context.handlers = engine.allNoRoute
	serveError(context, 404, default404Body)
}

var mimePlain = []string{MIMEPlain}

func serveError(c *Context, code int, defaultMessage []byte) {
	c.writermem.status = code
	c.Next()
	if !c.writermem.Written() {
		if c.writermem.Status() == code {
			c.writermem.Header()["Content-Type"] = mimePlain
			c.Writer.Write(defaultMessage)
		} else {
			c.writermem.WriteHeaderNow()
		}
	}
}

func redirectTrailingSlash(c *Context) {
	req := c.Request
	path := req.URL.Path
	code := 301 // Permanent redirect, request with GET method
	if req.Method != "GET" {
		code = 307
	}

	if len(path) > 1 && path[len(path)-1] == '/' {
		req.URL.Path = path[:len(path)-1]
	} else {
		req.URL.Path = path + "/"
	}
	debugPrint("redirecting request %d: %s --> %s", code, path, req.URL.String())
	http.Redirect(c.Writer, req, req.URL.String(), code)
	c.writermem.WriteHeaderNow()
}

func redirectFixedPath(c *Context, root *node, trailingSlash bool) bool {
	req := c.Request
	path := req.URL.Path

	fixedPath, found := root.findCaseInsensitivePath(
		cleanPath(path),
		trailingSlash,
	)
	if found {
		code := 301 // Permanent redirect, request with GET method
		if req.Method != "GET" {
			code = 307
		}
		req.URL.Path = string(fixedPath)
		debugPrint("redirecting request %d: %s --> %s", code, path, req.URL.String())
		http.Redirect(c.Writer, req, req.URL.String(), code)
		c.writermem.WriteHeaderNow()
		return true
	}
	return false
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"regexp"
)

var (
	GitRemoteRegexp    = regexp.MustCompile("(https|git)(@|://)github.com(:|/)([a-zA-Z0-9-_]+)/([a-zA-Z0-9-_]+)(?:\\.git)?")
	acceptableBranches = []string{"master", "main", "staging", "dev"}
)

func contains(s []string, e string) bool {
	for _, a := range s {
		if a == e {
			return true
		}
	}
	return false
}

func currentBranch() string {
	return shellOutput("git", "rev-parse", "--abbrev-ref", "HEAD")
}

func isAcceptableCurrentBranch() error {
	currBranch := currentBranch()
	if !contains(acceptableBranches, currBranch) {
		return fmt.Errorf("Unacceptable local branch: %s", currBranch)
	}
	return nil
}

func fetchRepoOwnerAndName() (string, string) {
	return extractOwnerAndNameFromRemote(gitOriginRemote())
}

func extractOwnerAndNameFromRemote(url string) (string, string) {
	matches := GitRemoteRegexp.FindStringSubmatch(url)
	if len(matches) < 2 {
		return "", ""
	}
	return matches[len(matches)-2], matches[len(matches)-1]
}

func gitOriginRemote() string {
	return shellOutput("git", "config", "remote.origin.url")
}

func gitPull() error {
	return shellExec("git", "pull", "--rebase")
}

func gitPush() {
	shellExec("git", "push")
}

func commitChangesToHistoryFile(pr string) {
	shellExec("git", "add", historyFile())
	shellExec(
		"git",
		"commit",
		"-m",
		"Update history to reflect merge of #"+pr,
		"-m",
		"[ci skip]",
	)
}
<commit_msg>Add \ before . so it doesn't include wrong hosts<commit_after>package main

import (
	"fmt"
	"regexp"
)

var (
	GitRemoteRegexp    = regexp.MustCompile("(https|git)(@|://)github\\.com(:|/)([a-zA-Z0-9-_]+)/([a-zA-Z0-9-_]+)(?:\\.git)?")
	acceptableBranches = []string{"master", "main", "staging", "dev"}
)

func contains(s []string, e string) bool {
	for _, a := range s {
		if a == e {
			return true
		}
	}
	return false
}

func currentBranch() string {
	return shellOutput("git", "rev-parse", "--abbrev-ref", "HEAD")
}

func isAcceptableCurrentBranch() error {
	currBranch := currentBranch()
	if !contains(acceptableBranches, currBranch) {
		return fmt.Errorf("Unacceptable local branch: %s", currBranch)
	}
	return nil
}

func fetchRepoOwnerAndName() (string, string) {
	return extractOwnerAndNameFromRemote(gitOriginRemote())
}

func extractOwnerAndNameFromRemote(url string) (string, string) {
	matches := GitRemoteRegexp.FindStringSubmatch(url)
	if len(matches) < 2 {
		return "", ""
	}
	return matches[len(matches)-2], matches[len(matches)-1]
}

func gitOriginRemote() string {
	return shellOutput("git", "config", "remote.origin.url")
}

func gitPull() error {
	return shellExec("git", "pull", "--rebase")
}

func gitPush() {
	shellExec("git", "push")
}

func commitChangesToHistoryFile(pr string) {
	shellExec("git", "add", historyFile())
	shellExec(
		"git",
		"commit",
		"-m",
		"Update history to reflect merge of #"+pr,
		"-m",
		"[ci skip]",
	)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"time"
	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/tools/clientcmd"
	clusapiclnt "k8s.io/kube-deploy/cluster-api/client"
	"k8s.io/apimachinery/pkg/util/wait"
)

var (
	kubeconfig = flag.String("kubeconfig", "", "path to kubeconfig file")
	kubeversion = flag.String("version", "", "The kubernetes version to be upgraded to")
)

func main() {
	flag.Parse()

	if *kubeversion == "" {
		panic("You need to specify kubernetes version being upgraded to.")
	}

	config, err := clientcmd.BuildConfigFromFlags("", *kubeconfig)
	if err != nil {
		panic(err.Error())
	}

	client, err := clusapiclnt.NewForConfig(config)
	if err != nil {
		panic(err.Error())
	}

	// Fetch Cluster object.
	clusters, err := client.Clusters().List(metav1.ListOptions{})
	if len(clusters.Items) != 1 {
		panic("There is more than one cluster returned!")
	}

	// Modify the control plane version
	clusters.Items[0].Spec.KubernetesVersion.Version = *kubeversion

	// Update the cluster.
	cluster, err := client.Clusters().Update(&clusters.Items[0])

	// Polling the cluster until new control plan is ready.
	err = wait.Poll(5*time.Second, 10*time.Minute, func() (bool, error) {
		cluster, err = client.Clusters().Get(cluster.Name, metav1.GetOptions{})
		if err == nil && cluster.Status.Ready {
			return true, err
		}
		return false, err
	})

	fmt.Println("Successfully upgraded control plan.")

	// Now continue to update all the node's state.
	machine_list, err := client.Machines().List(metav1.ListOptions{})
	if err != nil {
		panic(err.Error())
	}

	// Polling the cluster until nodes are updated.
	for _, machine := range machine_list.Items {
		machine.Spec.Versions.Kubelet = *kubeversion
		new_machine, err := client.Machines().Update(&machine)
		if err != nil {
			panic(err.Error())
		}
		err = wait.Poll(5*time.Second, 10*time.Minute, func() (bool, error) {
			new_machine, err = client.Machines().Get(new_machine.Name, metav1.GetOptions{})
			if err == nil && new_machine.Status.Ready {
				return true, err
			}
			return false, err
		})
		if err != nil {
			panic(err.Error())
		}
	}

	fmt.Println("Successfully upgraded the cluster to version: ", *kubeversion)
}
<commit_msg>Some tweaks for the upgrader.<commit_after>package main

import (
	"flag"
	"fmt"
	"time"
	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/tools/clientcmd"
	clusapiclnt "k8s.io/kube-deploy/cluster-api/client"
	"k8s.io/apimachinery/pkg/util/wait"
	machinesv1 "k8s.io/kube-deploy/cluster-api/api/machines/v1alpha1"
)

var (
	kubeconfig = flag.String("kubeconfig", "", "path to kubeconfig file")
	kubeversion = flag.String("version", "", "The kubernetes version to be upgraded to")
)

func main() {
	flag.Parse()

	if *kubeversion == "" {
		fmt.Errorf("You need to specify kubernetes version being upgraded to.")
	}

	config, err := clientcmd.BuildConfigFromFlags("", *kubeconfig)
	if err != nil {
		panic(err.Error())
	}

	client, err := clusapiclnt.NewForConfig(config)
	if err != nil {
		panic(err.Error())
	}

	// Fetch Cluster object.
	clusters, err := client.Clusters().List(metav1.ListOptions{})
	if err != nil {
		panic(err.Error())
	} else if len(clusters.Items) != 1 {
		panic("There is zero or more than one cluster returned!")
	}

	// Modify the control plane version
	clusters.Items[0].Spec.KubernetesVersion.Version = *kubeversion

	// Update the cluster.
	cluster, err := client.Clusters().Update(&clusters.Items[0])
	if err != nil {
		panic(err.Error())
	}

	// Polling the cluster until new control plan is ready.
	err = wait.Poll(5*time.Second, 10*time.Minute, func() (bool, error) {
		cluster, err = client.Clusters().Get(cluster.Name, metav1.GetOptions{})
		//if err == nil && cluster.Status.Ready {
		//	return true, err
		//}
		//return false, err
		return true, nil
	})
	if err != nil {
		fmt.Println("Failed to upgraded control plan. Error = %v", err)
		return
	} else {
		fmt.Println("Successfully upgraded control plan.")
	}

	// Now continue to update all the node's state.
	machine_list, err := client.Machines().List(metav1.ListOptions{})
	if err != nil {
		panic(err.Error())
	}

	// Polling the cluster until nodes are updated.
	errors := make(chan error, len(machine_list.Items))
	for i, _ := range machine_list.Items {
		go func(mach *machinesv1.Machine) {
			mach.Spec.Versions.Kubelet = *kubeversion
			fmt.Println("Updating machine:", mach.Name)
			new_machine, err := client.Machines().Update(mach)
			if err == nil {
				err = wait.Poll(5*time.Second, 10*time.Minute, func() (bool, error) {
					new_machine, err = client.Machines().Get(new_machine.Name, metav1.GetOptions{})
					//if err == nil && new_machine.Status.Ready {
					//	return true, err
					//}
					//return false, err
					if err != nil {
						panic(err.Error())
					}
					return true, nil
				})
			}
			if err != nil {
				panic(err.Error())
			}
			errors <- err
		} (&machine_list.Items[i])
	}

	for i := 0; i < len(machine_list.Items); i++ {
		if err = <-errors; err != nil {
			panic(err.Error())
		}
	}

	fmt.Println("Successfully upgraded the cluster to version: ", *kubeversion)
}
<|endoftext|>


<commit_before>package gmx

import (
	"encoding/json"
	"fmt"
	"log"
	"net"
	"os"
	"sync"
)

const GMX_VERSION = 0

var (
	r = &registry{
		entries: make(map[string]func() interface{}),
	}
)

func init() {
	s, err := localSocket()
	if err != nil {
		log.Printf("gmx: unable to open local socket: %v", err)
		return
	}

	// register the registries keys for discovery
	Publish("keys", func() interface{} {
		return r.keys()
	})
	go serve(s, r)
}

func localSocket() (net.Listener, error) {
	return net.ListenUnix("unix", localSocketAddr())
}

func localSocketAddr() *net.UnixAddr {
	return &net.UnixAddr{
		fmt.Sprintf("/tmp/.gmx.%d.%d", os.Getpid(), GMX_VERSION),
		"unix",
	}
}

// Publish registers the function f with the supplied key.
func Publish(key string, f func() interface{}) {
	r.register(key, f)
}

func serve(l net.Listener, r *registry) {
	// if listener is a unix socket, try to delete it on shutdown
	if l, ok := l.(*net.UnixListener); ok {
		if a, ok := l.Addr().(*net.UnixAddr); ok {
			defer os.Remove(a.Name)
		}
	}
	defer l.Close()
	for {
		c, err := l.Accept()
		if err != nil {
			return
		}
		go handle(c, r)
	}
}

func handle(nc net.Conn, reg *registry) {
	// conn makes it easier to send and receive json
	type conn struct {
		net.Conn
		*json.Encoder
		*json.Decoder
	}
	c := conn{
		nc,
		json.NewEncoder(nc),
		json.NewDecoder(nc),
	}
	defer c.Close()
	for {
		var keys []string
		if err := c.Decode(&keys); err != nil {
			log.Printf("gmx: client %v sent invalid json request: %v", c.RemoteAddr(), err)
			return
		}
		var result = make(map[string]interface{})
		for _, key := range keys {
			if f, ok := reg.value(key); ok {
				// invoke the function for key and store the result
				result[key] = f()
			}
		}
		if err := c.Encode(result); err != nil {
			log.Printf("gmx: could not send response to client %v: %v", c.RemoteAddr(), err)
			return
		}
	}
}

type registry struct {
	sync.Mutex // protects entries from concurrent mutation
	entries    map[string]func() interface{}
}

func (r *registry) register(key string, f func() interface{}) {
	r.Lock()
	defer r.Unlock()
	r.entries[key] = f
}

func (r *registry) value(key string) (func() interface{}, bool) {
	r.Lock()
	defer r.Unlock()
	f, ok := r.entries[key]
	return f, ok
}

func (r *registry) keys() (k []string) {
	r.Lock()
	defer r.Unlock()
	for e := range r.entries {
		k = append(k, e)
	}
	return
}
<commit_msg>Do not report EOF when client disconnects<commit_after>package gmx

import (
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net"
	"os"
	"sync"
)

const GMX_VERSION = 0

var (
	r = &registry{
		entries: make(map[string]func() interface{}),
	}
)

func init() {
	s, err := localSocket()
	if err != nil {
		log.Printf("gmx: unable to open local socket: %v", err)
		return
	}

	// register the registries keys for discovery
	Publish("keys", func() interface{} {
		return r.keys()
	})
	go serve(s, r)
}

func localSocket() (net.Listener, error) {
	return net.ListenUnix("unix", localSocketAddr())
}

func localSocketAddr() *net.UnixAddr {
	return &net.UnixAddr{
		fmt.Sprintf("/tmp/.gmx.%d.%d", os.Getpid(), GMX_VERSION),
		"unix",
	}
}

// Publish registers the function f with the supplied key.
func Publish(key string, f func() interface{}) {
	r.register(key, f)
}

func serve(l net.Listener, r *registry) {
	// if listener is a unix socket, try to delete it on shutdown
	if l, ok := l.(*net.UnixListener); ok {
		if a, ok := l.Addr().(*net.UnixAddr); ok {
			defer os.Remove(a.Name)
		}
	}
	defer l.Close()
	for {
		c, err := l.Accept()
		if err != nil {
			return
		}
		go handle(c, r)
	}
}

func handle(nc net.Conn, reg *registry) {
	// conn makes it easier to send and receive json
	type conn struct {
		net.Conn
		*json.Encoder
		*json.Decoder
	}
	c := conn{
		nc,
		json.NewEncoder(nc),
		json.NewDecoder(nc),
	}
	defer c.Close()
	for {
		var keys []string
		if err := c.Decode(&keys); err != nil {
			if err != io.EOF {
				log.Printf("gmx: client %v sent invalid json request: %v", c.RemoteAddr(), err)
			}
			return
		}
		var result = make(map[string]interface{})
		for _, key := range keys {
			if f, ok := reg.value(key); ok {
				// invoke the function for key and store the result
				result[key] = f()
			}
		}
		if err := c.Encode(result); err != nil {
			log.Printf("gmx: could not send response to client %v: %v", c.RemoteAddr(), err)
			return
		}
	}
}

type registry struct {
	sync.Mutex // protects entries from concurrent mutation
	entries    map[string]func() interface{}
}

func (r *registry) register(key string, f func() interface{}) {
	r.Lock()
	defer r.Unlock()
	r.entries[key] = f
}

func (r *registry) value(key string) (func() interface{}, bool) {
	r.Lock()
	defer r.Unlock()
	f, ok := r.entries[key]
	return f, ok
}

func (r *registry) keys() (k []string) {
	r.Lock()
	defer r.Unlock()
	for e := range r.entries {
		k = append(k, e)
	}
	return
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/binary"
	"fmt"
	"math"
	"math/rand"
	"os"
	"sort"
)

type (
	neuron    []float64
	nLayer    []neuron
	neuralNet []nLayer
)

func randWeight() float64 {
	return rand.Float64() - rand.Float64()
}

func NewNeuron(numInputs int64) neuron {
	var n neuron
	for i := int64(0); i < numInputs+1; i++ {
		n = append(n, randWeight())
	}
	return n
}

func NewNeuronLayer(numNeurons int64, numInputs int64) nLayer {
	var layer nLayer
	for i := int64(0); i < numNeurons; i++ {
		layer = append(layer, NewNeuron(numInputs))
	}
	return layer
}

func NewNeuralNet(numInputs int64, numOutputs int64, numHiddenLayers int64, numNeuronsPerLayer int64) neuralNet {
	var net neuralNet
	if numHiddenLayers > 0 {
		net = append(net, NewNeuronLayer(numNeuronsPerLayer, numInputs))
		for i := int64(0); i < numHiddenLayers-1; i++ {
			net = append(net, NewNeuronLayer(numNeuronsPerLayer, numNeuronsPerLayer))
		}
		net = append(net, NewNeuronLayer(numOutputs, numNeuronsPerLayer))
	} else {
		net = append(net, NewNeuronLayer(numOutputs, numInputs))
	}
	return net
}

func (net neuralNet) getWeights() []float64 {
	weights := make([]float64, 0)

	for _, layer := range net {
		for _, neuron := range layer {
			for _, weight := range neuron {
				weights = append(weights, weight)
			}
		}
	}
	return weights
}

func (net neuralNet) setWeights(newWeights []float64) {
	index := 0
	for l, _ := range net {
		for n, _ := range net[l] {
			for e, _ := range net[l][n] {
				net[l][n][e] = newWeights[index]
				index++
			}
		}
	}
}

func (net neuralNet) printNet() {
	for c, layer := range net {
		fmt.Printf("Layer: %d (%d neurons)\n", c, len(layer))
		for i, neuron := range layer {
			fmt.Printf("Neuron: %d\n", i)
			for i, weight := range neuron {
				if i == len(neuron)-1 {
					fmt.Printf("Bias: %f\n", weight)
				} else {
					fmt.Println(weight)
				}
			}
		}
	}
}

func sigmoid(input float64) float64 {
	p := 1.0
	return 1.0 / (1.0 + math.Exp(-1*input/p))
}

func (n neuron) bias() float64 {
	return n[len(n)-1]
}

func (n neuron) updateNeuron(inputs []float64) float64 {
	var output float64 = 0.0
	for i := 0; i < len(n)-1; i++ {
		output += n[i] * inputs[i]
	}
	output -= n.bias()
	return sigmoid(output)
}

func (layer nLayer) updateLayer(inputs []float64) []float64 {
	var output []float64
	for _, n := range layer {
		output = append(output, n.updateNeuron(inputs))
	}
	return output
}

func (net neuralNet) update(inputs []float64) []float64 {
	var outputs []float64

	for _, layer := range net {
		outputs = layer.updateLayer(inputs)
		inputs = outputs
	}

	return outputs
}

func (net neuralNet) mutate(mutationRate, maxPerturbation float64, numInputs, numOutputs, numHiddenLayers, numNeuronsPerLayer int64) neuralNet {
	mutatedNet := NewNeuralNet(numInputs, numOutputs, numHiddenLayers, numNeuronsPerLayer)

	genome := net.getWeights()

	for i, _ := range genome {
		if rand.Float64() < mutationRate {
			genome[i] += randWeight() * maxPerturbation
		}
	}
	mutatedNet.setWeights(genome)

	return mutatedNet
}

func seedRand() {
	f, err := os.Open("/dev/urandom")
	if err != nil {
		fmt.Printf("Failed to open random source: %s\n", err)
		os.Exit(1)
	}
	seed := make([]byte, 8)
	i, err := f.Read(seed)
	if err != nil {
		fmt.Printf("Failed to read random source: %s\n", err)
		os.Exit(1)
	}
	if i != 8 {
		fmt.Printf("Was expecting to read 8 bytes of seeds, read %d bytes instead\n", i)
		os.Exit(1)
	}
	seed64 := int64(binary.LittleEndian.Uint64(seed))
	rand.Seed(seed64)
}

type chromosome struct {
	Fitness float64
	Net     neuralNet
}
type Species []chromosome

func (s Species) Len() int {
	return len(s)
}
func (s Species) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}
func (s Species) Less(i, j int) bool {
	return s[i].Fitness < s[j].Fitness
}

func main() {
	seedRand()

	var inputs [11]float64
	var answers [11]float64
	for i := 0; i <= 10; i++ {
		x := float64(i) / 10.0
		inputs[i] = x
		answers[i] = 1 - x
	}

	species := make(Species, 20)
	for i, _ := range species {
		species[i].Fitness = 0.0
		species[i].Net = NewNeuralNet(1, 1, 3, 10)
	}

	for i := 0; i < 100000; i++ {
		fmt.Printf(".")
		for c, _ := range species {
			for y, input := range inputs {
				species[c].Fitness += math.Abs(answers[y] - species[c].Net.update([]float64{input})[0])
			}
		}

		// Get best 5 nets, make 3 mutated children for each, and start again
		sort.Sort(species)
		for chromo := 0; chromo < 5; chromo++ {
			species[chromo].Fitness = 0.0
			species[3*chromo+5].Fitness = 0.0
			species[3*chromo+5].Net = species[chromo].Net.mutate(0.025, 0.3, 1, 1, 3, 10)
			species[3*chromo+6].Fitness = 0.0
			species[3*chromo+6].Net = species[chromo].Net.mutate(0.05, 0.3, 1, 1, 3, 10)
			species[3*chromo+7].Fitness = 0.0
			species[3*chromo+7].Net = species[chromo].Net.mutate(0.1, 0.3, 1, 1, 3, 10)
		}
	}
	fmt.Println()

	species[0].Net.printNet()
	for i, input := range inputs {
		fmt.Printf("Input: %f, Output: %f, Answer: %f\n", input, species[0].Net.update([]float64{input})[0], answers[i])
	}

}
<commit_msg>Choose more sane number of training rounds<commit_after>package main

import (
	"encoding/binary"
	"fmt"
	"math"
	"math/rand"
	"os"
	"sort"
)

type (
	neuron    []float64
	nLayer    []neuron
	neuralNet []nLayer
)

func randWeight() float64 {
	return rand.Float64() - rand.Float64()
}

func NewNeuron(numInputs int64) neuron {
	var n neuron
	for i := int64(0); i < numInputs+1; i++ {
		n = append(n, randWeight())
	}
	return n
}

func NewNeuronLayer(numNeurons int64, numInputs int64) nLayer {
	var layer nLayer
	for i := int64(0); i < numNeurons; i++ {
		layer = append(layer, NewNeuron(numInputs))
	}
	return layer
}

func NewNeuralNet(numInputs int64, numOutputs int64, numHiddenLayers int64, numNeuronsPerLayer int64) neuralNet {
	var net neuralNet
	if numHiddenLayers > 0 {
		net = append(net, NewNeuronLayer(numNeuronsPerLayer, numInputs))
		for i := int64(0); i < numHiddenLayers-1; i++ {
			net = append(net, NewNeuronLayer(numNeuronsPerLayer, numNeuronsPerLayer))
		}
		net = append(net, NewNeuronLayer(numOutputs, numNeuronsPerLayer))
	} else {
		net = append(net, NewNeuronLayer(numOutputs, numInputs))
	}
	return net
}

func (net neuralNet) getWeights() []float64 {
	weights := make([]float64, 0)

	for _, layer := range net {
		for _, neuron := range layer {
			for _, weight := range neuron {
				weights = append(weights, weight)
			}
		}
	}
	return weights
}

func (net neuralNet) setWeights(newWeights []float64) {
	index := 0
	for l, _ := range net {
		for n, _ := range net[l] {
			for e, _ := range net[l][n] {
				net[l][n][e] = newWeights[index]
				index++
			}
		}
	}
}

func (net neuralNet) printNet() {
	for c, layer := range net {
		fmt.Printf("Layer: %d (%d neurons)\n", c, len(layer))
		for i, neuron := range layer {
			fmt.Printf("Neuron: %d\n", i)
			for i, weight := range neuron {
				if i == len(neuron)-1 {
					fmt.Printf("Bias: %f\n", weight)
				} else {
					fmt.Println(weight)
				}
			}
		}
	}
}

func sigmoid(input float64) float64 {
	p := 1.0
	return 1.0 / (1.0 + math.Exp(-1*input/p))
}

func (n neuron) bias() float64 {
	return n[len(n)-1]
}

func (n neuron) updateNeuron(inputs []float64) float64 {
	var output float64 = 0.0
	for i := 0; i < len(n)-1; i++ {
		output += n[i] * inputs[i]
	}
	output -= n.bias()
	return sigmoid(output)
}

func (layer nLayer) updateLayer(inputs []float64) []float64 {
	var output []float64
	for _, n := range layer {
		output = append(output, n.updateNeuron(inputs))
	}
	return output
}

func (net neuralNet) update(inputs []float64) []float64 {
	var outputs []float64

	for _, layer := range net {
		outputs = layer.updateLayer(inputs)
		inputs = outputs
	}

	return outputs
}

func (net neuralNet) mutate(mutationRate, maxPerturbation float64, numInputs, numOutputs, numHiddenLayers, numNeuronsPerLayer int64) neuralNet {
	mutatedNet := NewNeuralNet(numInputs, numOutputs, numHiddenLayers, numNeuronsPerLayer)

	genome := net.getWeights()

	for i, _ := range genome {
		if rand.Float64() < mutationRate {
			genome[i] += randWeight() * maxPerturbation
		}
	}
	mutatedNet.setWeights(genome)

	return mutatedNet
}

func seedRand() {
	f, err := os.Open("/dev/urandom")
	if err != nil {
		fmt.Printf("Failed to open random source: %s\n", err)
		os.Exit(1)
	}
	seed := make([]byte, 8)
	i, err := f.Read(seed)
	if err != nil {
		fmt.Printf("Failed to read random source: %s\n", err)
		os.Exit(1)
	}
	if i != 8 {
		fmt.Printf("Was expecting to read 8 bytes of seeds, read %d bytes instead\n", i)
		os.Exit(1)
	}
	seed64 := int64(binary.LittleEndian.Uint64(seed))
	rand.Seed(seed64)
}

type chromosome struct {
	Fitness float64
	Net     neuralNet
}
type Species []chromosome

func (s Species) Len() int {
	return len(s)
}
func (s Species) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}
func (s Species) Less(i, j int) bool {
	return s[i].Fitness < s[j].Fitness
}

func main() {
	seedRand()

	var inputs [11]float64
	var answers [11]float64
	for i := 0; i <= 10; i++ {
		x := float64(i) / 10.0
		inputs[i] = x
		answers[i] = 1 - x
	}

	species := make(Species, 20)
	for i, _ := range species {
		species[i].Fitness = 0.0
		species[i].Net = NewNeuralNet(1, 1, 3, 10)
	}

	for i := 0; i < 10000; i++ {
		fmt.Printf(".")
		for c, _ := range species {
			for y, input := range inputs {
				species[c].Fitness += math.Abs(answers[y] - species[c].Net.update([]float64{input})[0])
			}
		}

		// Get best 5 nets, make 3 mutated children for each, and start again
		sort.Sort(species)
		for chromo := 0; chromo < 5; chromo++ {
			species[chromo].Fitness = 0.0
			species[3*chromo+5].Fitness = 0.0
			species[3*chromo+5].Net = species[chromo].Net.mutate(0.025, 0.3, 1, 1, 3, 10)
			species[3*chromo+6].Fitness = 0.0
			species[3*chromo+6].Net = species[chromo].Net.mutate(0.05, 0.3, 1, 1, 3, 10)
			species[3*chromo+7].Fitness = 0.0
			species[3*chromo+7].Net = species[chromo].Net.mutate(0.1, 0.3, 1, 1, 3, 10)
		}
	}
	fmt.Println()

	species[0].Net.printNet()
	for i, input := range inputs {
		fmt.Printf("Input: %f, Output: %f, Answer: %f\n", input, species[0].Net.update([]float64{input})[0], answers[i])
	}

}
<|endoftext|>


<commit_before>package gop

import (
    "github.com/gorilla/mux"
    "github.com/jbert/timber"

    "fmt"
    "net"
    "time"
    "runtime"
    "net/http"
    "strings"

    "os/user"
    "syscall"
    "strconv"
)

// Stuff we include in both App and Req, for convenience
type common struct {
    timber.Logger
    Cfg             *Config
    Stats           StatsdClient
}

// Represents a managed gop app. Returned by init.
// Embeds logging, provides .Cfg for configuration access.
type App struct {
    common

    AppName         string
    ProjectName     string
    GorillaRouter   *mux.Router
    listener        net.Listener
    wantReq         chan *wantReq
    doneReq         chan *Req
    getReqs         chan chan *Req
    startTime       time.Time
    currentReqs     int
    totalReqs       int
}

// Used for RPC to get a new request
type wantReq struct {
    r       *http.Request
    reply   chan *Req
}

// Per request struct. has convenience references to functionality
// in the app singleton. Passed into the request handler.
type Req struct {
    common

    id              int
    startTime       time.Time
    app             *App
    r               *http.Request
}

// The function signature your http handlers need.
type HandlerFunc func(g *Req, w http.ResponseWriter, r *http.Request)

// Set up the application. Reads config. Panic if runtime environment is deficient.
func Init(projectName, appName string) *App {
    app := &App{
        AppName:        appName,
        ProjectName:    projectName,
        GorillaRouter:  mux.NewRouter(),
        wantReq:        make(chan *wantReq),
        doneReq:        make(chan *Req),
        getReqs:        make(chan chan *Req),
        startTime:      time.Now(),
    }

    app.loadAppConfigFile()

    app.setUserAndGroup()

    app.initLogging()

    maxProcs, _ := app.Cfg.GetInt("gop", "maxprocs", 4 * runtime.NumCPU())
    app.Debug("Seting maxprocs to %d\n", maxProcs)
    runtime.GOMAXPROCS(maxProcs)

    app.initStatsd()

    app.goAgainSetup()

    app.registerGopHandlers()

    go app.watchdog()

    go app.requestMaker()

    return app
}

func (a *App) setUserAndGroup() {
    // We do not have logging set up yet. We just panic() on error.

    desiredUserName, _ := a.Cfg.Get("gop", "user", a.AppName)
    desiredUser, err := user.Lookup(desiredUserName)
    // NickG would prefer we die screaming instead of running on as root
    if err != nil {
        panic(fmt.Sprintf("Can't find user [%s] - please set config correctly and/or create user", desiredUserName))
    }

    currentUser, err := user.Current()
    if err != nil {
        panic(fmt.Sprintf("Can't find current user: %s", err.Error()))
    }
    if currentUser.Uid != desiredUser.Uid {
        numericId, err := strconv.Atoi(desiredUser.Uid)
        if err != nil {
            panic(fmt.Sprintf("Can't interpret [%s] as a numeric user id [following lookup of usernmae %s]", desiredUser.Uid, desiredUserName))
        }
        err = syscall.Setuid(numericId)
        if err != nil {
            panic(fmt.Sprintf("Can't setuid to [%s]: %s", desiredUser.Uid, err.Error()))
        }
    }

// Can't log at this stage :-/
//    a.Info("Running as user %s (%d)", desiredUserName, desiredUser.Uid)
}

// Hands out 'request' objects
func (a *App) requestMaker() {
    nextReqId := 0
    openReqs := make(map[int] *Req)
    for {
        select {
            case wantReq := <- a.wantReq: {
                req := Req{
                    common: common{
                        Logger:     a.Logger,
                        Cfg:        a.Cfg,
                        Stats:      a.Stats,
                    },

                    id:         nextReqId,
                    app:        a,
                    startTime:  time.Now(),
                    r:          wantReq.r,
                }
                openReqs[req.id] = &req
                nextReqId++
                a.Stats.Inc("http_reqs", 1)
                a.Stats.Inc("current_http_reqs", 1)
                a.totalReqs++
                a.currentReqs++
                wantReq.reply <- &req
            }
            case doneReq := <- a.doneReq: {
                _, found := openReqs[doneReq.id]
                if !found {
                    a.Error("BUG! Unknown request id [%d] being retired")
                } else {
                    a.Stats.Dec("current_http_reqs", 1)
// We don't yet have access to status codes. TODO: wrap response in handler so
// we can capture them
//                    statsKey := fmt.Sprintf("http_status.%d", doneReq.r.StatusCode)
//                    a.Stats.Inc(statsKey, 1)
                    doneReq.finished()
                    a.currentReqs--
                    delete (openReqs, doneReq.id)
                }
            }
            case reply := <- a.getReqs: {
                go func() {
                    for _, req := range openReqs {
                        reply <- req
                    }
                    close(reply)
                }()
            }
        }
    }
}

// Ask requestMaker for a request
func (a *App) getReq(r *http.Request) *Req {
    reply := make(chan *Req)
    a.wantReq <- &wantReq{r: r, reply: reply}
    return <- reply
}

func (g *Req) finished() {
    reqDuration := time.Since(g.startTime)
    slowReqSecs, _ := g.Cfg.GetFloat32("gop", "slow_req_secs", 10)
    if reqDuration.Seconds() > float64(slowReqSecs) {
        g.Error("Slow request [%s] took %s", g.r.URL, reqDuration)
    } else {
        g.Info("Request took %s", reqDuration)
    }
}

func (a *App) initLogging() {
    defaultLogDir, _ := a.Cfg.Get("gop", "log_dir", "/var/log")
    defaultLogFname := defaultLogDir + "/" + a.ProjectName + "/" + a.AppName + ".log"
    logFname, _ := a.Cfg.Get("gop", "log_file", defaultLogFname)

    writer, err := timber.NewFileWriter(logFname)
    if err != nil {
        panic(fmt.Sprintf("Can't open log file: %s", err))
    }
    configLogger := timber.ConfigLogger{
        LogWriter:  writer,
        Level:      timber.INFO,
        Formatter:  timber.NewPatFormatter("[%D %T] [%L] %S %M"),
    }

    logLevelStr, _ := a.Cfg.Get("gop", "log_level", "INFO")
    logLevelStr = strings.ToUpper(logLevelStr)
    for logLevel, levelStr := range timber.LongLevelStrings {
        if logLevelStr == levelStr {
            configLogger.Level = timber.Level(logLevel)
            break
        }
    }

    // Hack all logging to stdout for dev
    forceStdout, _:= a.Cfg.GetBool("gop", "stdout_only_logging", false)
    if forceStdout {
        configLogger.LogWriter = new(timber.ConsoleWriter)
    }

    l := timber.NewTimber()
    l.AddLogger(configLogger)
    a.Logger = l
}

func (a *App) watchdog() {
    repeat, _ := a.Cfg.GetInt("gop", "watchdog_secs", 300)
    ticker := time.Tick(time.Second * time.Duration(repeat))

    for {
        sysMemBytesLimit, _ := a.Cfg.GetInt64("gop", "sysmem_bytes_limit", 0)
        sysMemBytes := int64(getSysMemBytesUsed())  // Only supports 2^63 byte systems
        a.Info("TICK: %d bytes %d current %d total", getSysMemBytesUsed, a.currentReqs, a.totalReqs)
        if sysMemBytesLimit > 0 && sysMemBytes >= sysMemBytesLimit {
            a.Error("MEM LIMIT REACHED [%d >= %d] - starting graceful restart", sysMemBytes, sysMemBytesLimit)
            a.StartGracefulRestart("Memory limit reached")
        }
        <- ticker
    }
}

func (a *App) WrapHandler(h HandlerFunc) http.HandlerFunc {
    // Wrap the handler, so we can do before/after logic
    f := func(w http.ResponseWriter, r *http.Request) {
        gopRequest := a.getReq(r)
        defer func() {
            a.doneReq <- gopRequest
        }()
        // Pass in the gop, for logging, cfg etc
        h(gopRequest, w, r)
    }

    return http.HandlerFunc(f)
}

// Register an http handler managed by gop.
// We use Gorilla muxxer, since it is back-compatible and nice to use :-)
func (a *App) HandleFunc(u string, h HandlerFunc) {
    gopHandler := a.WrapHandler(h)

    a.GorillaRouter.HandleFunc(u, gopHandler)
}

func (a *App) Run() {
    listenAddr, _ := a.Cfg.Get("gop", "listen_addr", ":http")
    listenNet, _ := a.Cfg.Get("gop", "listen_net", "tcp")
    a.goAgainListenAndServe(listenNet, listenAddr)
}

func (a *App) Serve(l net.Listener) {
    http.Serve(l, a.GorillaRouter)
}

func getSysMemBytesUsed() uint64 {
    var memStats runtime.MemStats
    runtime.ReadMemStats(&memStats)
    // There's lots of stuff in here.
    return memStats.Sys
}
<commit_msg>support log_pattern in config<commit_after>package gop

import (
    "github.com/gorilla/mux"
    "github.com/jbert/timber"

    "fmt"
    "net"
    "time"
    "runtime"
    "net/http"
    "strings"

    "os/user"
    "syscall"
    "strconv"
)

// Stuff we include in both App and Req, for convenience
type common struct {
    timber.Logger
    Cfg             *Config
    Stats           StatsdClient
}

// Represents a managed gop app. Returned by init.
// Embeds logging, provides .Cfg for configuration access.
type App struct {
    common

    AppName         string
    ProjectName     string
    GorillaRouter   *mux.Router
    listener        net.Listener
    wantReq         chan *wantReq
    doneReq         chan *Req
    getReqs         chan chan *Req
    startTime       time.Time
    currentReqs     int
    totalReqs       int
}

// Used for RPC to get a new request
type wantReq struct {
    r       *http.Request
    reply   chan *Req
}

// Per request struct. has convenience references to functionality
// in the app singleton. Passed into the request handler.
type Req struct {
    common

    id              int
    startTime       time.Time
    app             *App
    r               *http.Request
}

// The function signature your http handlers need.
type HandlerFunc func(g *Req, w http.ResponseWriter, r *http.Request)

// Set up the application. Reads config. Panic if runtime environment is deficient.
func Init(projectName, appName string) *App {
    app := &App{
        AppName:        appName,
        ProjectName:    projectName,
        GorillaRouter:  mux.NewRouter(),
        wantReq:        make(chan *wantReq),
        doneReq:        make(chan *Req),
        getReqs:        make(chan chan *Req),
        startTime:      time.Now(),
    }

    app.loadAppConfigFile()

    app.setUserAndGroup()

    app.initLogging()

    maxProcs, _ := app.Cfg.GetInt("gop", "maxprocs", 4 * runtime.NumCPU())
    app.Debug("Seting maxprocs to %d\n", maxProcs)
    runtime.GOMAXPROCS(maxProcs)

    app.initStatsd()

    app.goAgainSetup()

    app.registerGopHandlers()

    go app.watchdog()

    go app.requestMaker()

    return app
}

func (a *App) setUserAndGroup() {
    // We do not have logging set up yet. We just panic() on error.

    desiredUserName, _ := a.Cfg.Get("gop", "user", a.AppName)
    desiredUser, err := user.Lookup(desiredUserName)
    // NickG would prefer we die screaming instead of running on as root
    if err != nil {
        panic(fmt.Sprintf("Can't find user [%s] - please set config correctly and/or create user", desiredUserName))
    }

    currentUser, err := user.Current()
    if err != nil {
        panic(fmt.Sprintf("Can't find current user: %s", err.Error()))
    }
    if currentUser.Uid != desiredUser.Uid {
        numericId, err := strconv.Atoi(desiredUser.Uid)
        if err != nil {
            panic(fmt.Sprintf("Can't interpret [%s] as a numeric user id [following lookup of usernmae %s]", desiredUser.Uid, desiredUserName))
        }
        err = syscall.Setuid(numericId)
        if err != nil {
            panic(fmt.Sprintf("Can't setuid to [%s]: %s", desiredUser.Uid, err.Error()))
        }
    }

// Can't log at this stage :-/
//    a.Info("Running as user %s (%d)", desiredUserName, desiredUser.Uid)
}

// Hands out 'request' objects
func (a *App) requestMaker() {
    nextReqId := 0
    openReqs := make(map[int] *Req)
    for {
        select {
            case wantReq := <- a.wantReq: {
                req := Req{
                    common: common{
                        Logger:     a.Logger,
                        Cfg:        a.Cfg,
                        Stats:      a.Stats,
                    },

                    id:         nextReqId,
                    app:        a,
                    startTime:  time.Now(),
                    r:          wantReq.r,
                }
                openReqs[req.id] = &req
                nextReqId++
                a.Stats.Inc("http_reqs", 1)
                a.Stats.Inc("current_http_reqs", 1)
                a.totalReqs++
                a.currentReqs++
                wantReq.reply <- &req
            }
            case doneReq := <- a.doneReq: {
                _, found := openReqs[doneReq.id]
                if !found {
                    a.Error("BUG! Unknown request id [%d] being retired")
                } else {
                    a.Stats.Dec("current_http_reqs", 1)
// We don't yet have access to status codes. TODO: wrap response in handler so
// we can capture them
//                    statsKey := fmt.Sprintf("http_status.%d", doneReq.r.StatusCode)
//                    a.Stats.Inc(statsKey, 1)
                    doneReq.finished()
                    a.currentReqs--
                    delete (openReqs, doneReq.id)
                }
            }
            case reply := <- a.getReqs: {
                go func() {
                    for _, req := range openReqs {
                        reply <- req
                    }
                    close(reply)
                }()
            }
        }
    }
}

// Ask requestMaker for a request
func (a *App) getReq(r *http.Request) *Req {
    reply := make(chan *Req)
    a.wantReq <- &wantReq{r: r, reply: reply}
    return <- reply
}

func (g *Req) finished() {
    reqDuration := time.Since(g.startTime)
    slowReqSecs, _ := g.Cfg.GetFloat32("gop", "slow_req_secs", 10)
    if reqDuration.Seconds() > float64(slowReqSecs) {
        g.Error("Slow request [%s] took %s", g.r.URL, reqDuration)
    } else {
        g.Info("Request took %s", reqDuration)
    }
}

func (a *App) initLogging() {
    defaultLogDir, _ := a.Cfg.Get("gop", "log_dir", "/var/log")
    defaultLogFname := defaultLogDir + "/" + a.ProjectName + "/" + a.AppName + ".log"
    logFname, _ := a.Cfg.Get("gop", "log_file", defaultLogFname)

    logPattern, _ := a.Cfg.Get("gop", "log_pattern", "[%D %T] [%L] %S %M")

    writer, err := timber.NewFileWriter(logFname)
    if err != nil {
        panic(fmt.Sprintf("Can't open log file: %s", err))
    }
    configLogger := timber.ConfigLogger{
        LogWriter:  writer,
        Level:      timber.INFO,
        Formatter:  timber.NewPatFormatter(logPattern),
    }

    logLevelStr, _ := a.Cfg.Get("gop", "log_level", "INFO")
    logLevelStr = strings.ToUpper(logLevelStr)
    for logLevel, levelStr := range timber.LongLevelStrings {
        if logLevelStr == levelStr {
            configLogger.Level = timber.Level(logLevel)
            break
        }
    }

    // Hack all logging to stdout for dev
    forceStdout, _:= a.Cfg.GetBool("gop", "stdout_only_logging", false)
    if forceStdout {
        configLogger.LogWriter = new(timber.ConsoleWriter)
    }

    l := timber.NewTimber()
    l.AddLogger(configLogger)
    a.Logger = l
}

func (a *App) watchdog() {
    repeat, _ := a.Cfg.GetInt("gop", "watchdog_secs", 300)
    ticker := time.Tick(time.Second * time.Duration(repeat))

    for {
        sysMemBytesLimit, _ := a.Cfg.GetInt64("gop", "sysmem_bytes_limit", 0)
        sysMemBytes := int64(getSysMemBytesUsed())  // Only supports 2^63 byte systems
        a.Info("TICK: %d bytes %d current %d total", getSysMemBytesUsed, a.currentReqs, a.totalReqs)
        if sysMemBytesLimit > 0 && sysMemBytes >= sysMemBytesLimit {
            a.Error("MEM LIMIT REACHED [%d >= %d] - starting graceful restart", sysMemBytes, sysMemBytesLimit)
            a.StartGracefulRestart("Memory limit reached")
        }
        <- ticker
    }
}

func (a *App) WrapHandler(h HandlerFunc) http.HandlerFunc {
    // Wrap the handler, so we can do before/after logic
    f := func(w http.ResponseWriter, r *http.Request) {
        gopRequest := a.getReq(r)
        defer func() {
            a.doneReq <- gopRequest
        }()
        // Pass in the gop, for logging, cfg etc
        h(gopRequest, w, r)
    }

    return http.HandlerFunc(f)
}

// Register an http handler managed by gop.
// We use Gorilla muxxer, since it is back-compatible and nice to use :-)
func (a *App) HandleFunc(u string, h HandlerFunc) {
    gopHandler := a.WrapHandler(h)

    a.GorillaRouter.HandleFunc(u, gopHandler)
}

func (a *App) Run() {
    listenAddr, _ := a.Cfg.Get("gop", "listen_addr", ":http")
    listenNet, _ := a.Cfg.Get("gop", "listen_net", "tcp")
    a.goAgainListenAndServe(listenNet, listenAddr)
}

func (a *App) Serve(l net.Listener) {
    http.Serve(l, a.GorillaRouter)
}

func getSysMemBytesUsed() uint64 {
    var memStats runtime.MemStats
    runtime.ReadMemStats(&memStats)
    // There's lots of stuff in here.
    return memStats.Sys
}
<|endoftext|>