<commit_before>// Copyright 2012 The Gorilla Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Package mux implements a request router and dispatcher.

The name mux stands for "HTTP request multiplexer". Like the standard
http.ServeMux, mux.Router matches incoming requests against a list of
registered routes and calls a handler for the route that matches the URL
or other conditions. The main features are:

	* Requests can be matched based on URL host, path, path prefix, schemes,
	  header and query values, HTTP methods or using custom matchers.
	* URL hosts, paths and query values can have variables with an optional
	  regular expression.
	* Registered URLs can be built, or "reversed", which helps maintaining
	  references to resources.
	* Routes can be used as subrouters: nested routes are only tested if the
	  parent route matches. This is useful to define groups of routes that
	  share common conditions like a host, a path prefix or other repeated
	  attributes. As a bonus, this optimizes request matching.
	* It implements the http.Handler interface so it is compatible with the
	  standard http.ServeMux.

Let's start registering a couple of URL paths and handlers:

	func main() {
		r := mux.NewRouter()
		r.HandleFunc("/", HomeHandler)
		r.HandleFunc("/products", ProductsHandler)
		r.HandleFunc("/articles", ArticlesHandler)
		http.Handle("/", r)
	}

Here we register three routes mapping URL paths to handlers. This is
equivalent to how http.HandleFunc() works: if an incoming request URL matches
one of the paths, the corresponding handler is called passing
(http.ResponseWriter, *http.Request) as parameters.

Paths can have variables. They are defined using the format {name} or
{name:pattern}. If a regular expression pattern is not defined, the matched
variable will be anything until the next slash. For example:

	r := mux.NewRouter()
	r.HandleFunc("/products/{key}", ProductHandler)
	r.HandleFunc("/articles/{category}/", ArticlesCategoryHandler)
	r.HandleFunc("/articles/{category}/{id:[0-9]+}", ArticleHandler)

Groups can be used inside patterns, as long as they are non-capturing (?:re). For example:

	r.HandleFunc("/articles/{category}/{sort:(?:asc|desc|new)}", ArticlesCategoryHandler)

The names are used to create a map of route variables which can be retrieved
calling mux.Vars():

	vars := mux.Vars(request)
	category := vars["category"]

Note that if any capturing groups are present, mux will panic() during parsing. To prevent
this, convert any capturing groups to non-capturing, e.g. change "/{sort:(asc|desc)}" to
"/{sort:(?:asc|desc)}". This is a change from prior versions which behaved unpredictably
when capturing groups were present.

And this is all you need to know about the basic usage. More advanced options
are explained below.

Routes can also be restricted to a domain or subdomain. Just define a host
pattern to be matched. They can also have variables:

	r := mux.NewRouter()
	// Only matches if domain is "www.example.com".
	r.Host("www.example.com")
	// Matches a dynamic subdomain.
	r.Host("{subdomain:[a-z]+}.domain.com")

There are several other matchers that can be added. To match path prefixes:

	r.PathPrefix("/products/")

...or HTTP methods:

	r.Methods("GET", "POST")

...or URL schemes:

	r.Schemes("https")

...or header values:

	r.Headers("X-Requested-With", "XMLHttpRequest")

...or query values:

	r.Queries("key", "value")

...or to use a custom matcher function:

	r.MatcherFunc(func(r *http.Request, rm *RouteMatch) bool {
		return r.ProtoMajor == 0
	})

...and finally, it is possible to combine several matchers in a single route:

	r.HandleFunc("/products", ProductsHandler).
	  Host("www.example.com").
	  Methods("GET").
	  Schemes("http")

Setting the same matching conditions again and again can be boring, so we have
a way to group several routes that share the same requirements.
We call it "subrouting".

For example, let's say we have several URLs that should only match when the
host is "www.example.com". Create a route for that host and get a "subrouter"
from it:

	r := mux.NewRouter()
	s := r.Host("www.example.com").Subrouter()

Then register routes in the subrouter:

	s.HandleFunc("/products/", ProductsHandler)
	s.HandleFunc("/products/{key}", ProductHandler)
	s.HandleFunc("/articles/{category}/{id:[0-9]+}"), ArticleHandler)

The three URL paths we registered above will only be tested if the domain is
"www.example.com", because the subrouter is tested first. This is not
only convenient, but also optimizes request matching. You can create
subrouters combining any attribute matchers accepted by a route.

Subrouters can be used to create domain or path "namespaces": you define
subrouters in a central place and then parts of the app can register its
paths relatively to a given subrouter.

There's one more thing about subroutes. When a subrouter has a path prefix,
the inner routes use it as base for their paths:

	r := mux.NewRouter()
	s := r.PathPrefix("/products").Subrouter()
	// "/products/"
	s.HandleFunc("/", ProductsHandler)
	// "/products/{key}/"
	s.HandleFunc("/{key}/", ProductHandler)
	// "/products/{key}/details"
	s.HandleFunc("/{key}/details", ProductDetailsHandler)

Note that the path provided to PathPrefix() represents a "wildcard": calling
PathPrefix("/static/").Handler(...) means that the handler will be passed any
request that matches "/static/*". This makes it easy to serve static files with mux:

	func main() {
		var dir string

		flag.StringVar(&dir, "dir", ".", "the directory to serve files from. Defaults to the current dir")
		flag.Parse()
		r := mux.NewRouter()

		// This will serve files under http://localhost:8000/static/<filename>
		r.PathPrefix("/static/").Handler(http.StripPrefix("/static/", http.FileServer(http.Dir(dir))))

		srv := &http.Server{
			Handler:      r,
			Addr:         "127.0.0.1:8000",
			// Good practice: enforce timeouts for servers you create!
			WriteTimeout: 15 * time.Second,
			ReadTimeout:  15 * time.Second,
		}

		log.Fatal(srv.ListenAndServe())
	}

Now let's see how to build registered URLs.

Routes can be named. All routes that define a name can have their URLs built,
or "reversed". We define a name calling Name() on a route. For example:

	r := mux.NewRouter()
	r.HandleFunc("/articles/{category}/{id:[0-9]+}", ArticleHandler).
	  Name("article")

To build a URL, get the route and call the URL() method, passing a sequence of
key/value pairs for the route variables. For the previous route, we would do:

	url, err := r.Get("article").URL("category", "technology", "id", "42")

...and the result will be a url.URL with the following path:

	"/articles/technology/42"

This also works for host and query value variables:

	r := mux.NewRouter()
	r.Host("{subdomain}.domain.com").
	  Path("/articles/{category}/{id:[0-9]+}").
	  Queries("filter", "{filter}").
	  HandlerFunc(ArticleHandler).
	  Name("article")

	// url.String() will be "http://news.domain.com/articles/technology/42?filter=gorilla"
	url, err := r.Get("article").URL("subdomain", "news",
	                                 "category", "technology",
	                                 "id", "42",
	                                 "filter", "gorilla")

All variables defined in the route are required, and their values must
conform to the corresponding patterns. These requirements guarantee that a
generated URL will always match a registered route -- the only exception is
for explicitly defined "build-only" routes which never match.

Regex support also exists for matching Headers within a route. For example, we could do:

	r.HeadersRegexp("Content-Type", "application/(text|json)")

...and the route will match both requests with a Content-Type of `application/json` as well as
`application/text`

There's also a way to build only the URL host or path for a route:
use the methods URLHost() or URLPath() instead. For the previous route,
we would do:

	// "http://news.domain.com/"
	host, err := r.Get("article").URLHost("subdomain", "news")

	// "/articles/technology/42"
	path, err := r.Get("article").URLPath("category", "technology", "id", "42")

And if you use subrouters, host and path defined separately can be built
as well:

	r := mux.NewRouter()
	s := r.Host("{subdomain}.domain.com").Subrouter()
	s.Path("/articles/{category}/{id:[0-9]+}").
	  HandlerFunc(ArticleHandler).
	  Name("article")

	// "http://news.domain.com/articles/technology/42"
	url, err := r.Get("article").URL("subdomain", "news",
	                                 "category", "technology",
	                                 "id", "42")

Since **vX.Y.Z**, mux supports the addition of middlewares to a [Router](https://godoc.org/github.com/gorilla/mux#Router), which are executed if a
match is found (including subrouters). Middlewares are defined using the de facto standard type:

	type MiddlewareFunc func(http.Handler) http.Handler

Typically, the returned handler is a closure which does something with the http.ResponseWriter and http.Request passed to it, and then calls the handler passed as parameter to the MiddlewareFunc (closures can access variables from the context where they are created).

A very basic middleware which logs the URI of the request being handled could be written as:

	func simpleMw(next http.Handler) http.Handler {
		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			// Do stuff here
			log.Println(r.RequestURI)
			// Call the next handler, which can be another middleware in the chain, or the final handler.
			next.ServeHTTP(w, r)
		})
	}

Middlewares can be added to a router using `Router.Use()`:

	r := mux.NewRouter()
	r.HandleFunc("/", handler)
	r.AddMiddleware(simpleMw)

A more complex authentication middleware, which maps session token to users, could be written as:

	// Define our struct
	type authenticationMiddleware struct {
		tokenUsers map[string]string
	}

	// Initialize it somewhere
	func (amw *authenticationMiddleware) Populate() {
		amw.tokenUsers["00000000"] = "user0"
		amw.tokenUsers["aaaaaaaa"] = "userA"
		amw.tokenUsers["05f717e5"] = "randomUser"
		amw.tokenUsers["deadbeef"] = "user0"
	}

	// Middleware function, which will be called for each request
	func (amw *authenticationMiddleware) Middleware(next http.Handler) http.Handler {
		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			token := r.Header.Get("X-Session-Token")

			if user, found := amw.tokenUsers[token]; found {
				// We found the token in our map
				log.Printf("Authenticated user %s\n", user)
				next.ServeHTTP(w, r)
			} else {
				http.Error(w, "Forbidden", 403)
			}
		})
	}

	r := mux.NewRouter()
	r.HandleFunc("/", handler)

	amw := authenticationMiddleware{}
	amw.Populate()

	r.Use(amw.Middleware)

Note: The handler chain will be stopped if your middleware doesn't call `next.ServeHTTP()` with the corresponding parameters. This can be used to abort a request if the middleware writer wants to.

*/
package mux
<commit_msg>Update doc.go: r.AddMiddleware(...) -> r.Use(...)<commit_after>// Copyright 2012 The Gorilla Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Package mux implements a request router and dispatcher.

The name mux stands for "HTTP request multiplexer". Like the standard
http.ServeMux, mux.Router matches incoming requests against a list of
registered routes and calls a handler for the route that matches the URL
or other conditions. The main features are:

	* Requests can be matched based on URL host, path, path prefix, schemes,
	  header and query values, HTTP methods or using custom matchers.
	* URL hosts, paths and query values can have variables with an optional
	  regular expression.
	* Registered URLs can be built, or "reversed", which helps maintaining
	  references to resources.
	* Routes can be used as subrouters: nested routes are only tested if the
	  parent route matches. This is useful to define groups of routes that
	  share common conditions like a host, a path prefix or other repeated
	  attributes. As a bonus, this optimizes request matching.
	* It implements the http.Handler interface so it is compatible with the
	  standard http.ServeMux.

Let's start registering a couple of URL paths and handlers:

	func main() {
		r := mux.NewRouter()
		r.HandleFunc("/", HomeHandler)
		r.HandleFunc("/products", ProductsHandler)
		r.HandleFunc("/articles", ArticlesHandler)
		http.Handle("/", r)
	}

Here we register three routes mapping URL paths to handlers. This is
equivalent to how http.HandleFunc() works: if an incoming request URL matches
one of the paths, the corresponding handler is called passing
(http.ResponseWriter, *http.Request) as parameters.

Paths can have variables. They are defined using the format {name} or
{name:pattern}. If a regular expression pattern is not defined, the matched
variable will be anything until the next slash. For example:

	r := mux.NewRouter()
	r.HandleFunc("/products/{key}", ProductHandler)
	r.HandleFunc("/articles/{category}/", ArticlesCategoryHandler)
	r.HandleFunc("/articles/{category}/{id:[0-9]+}", ArticleHandler)

Groups can be used inside patterns, as long as they are non-capturing (?:re). For example:

	r.HandleFunc("/articles/{category}/{sort:(?:asc|desc|new)}", ArticlesCategoryHandler)

The names are used to create a map of route variables which can be retrieved
calling mux.Vars():

	vars := mux.Vars(request)
	category := vars["category"]

Note that if any capturing groups are present, mux will panic() during parsing. To prevent
this, convert any capturing groups to non-capturing, e.g. change "/{sort:(asc|desc)}" to
"/{sort:(?:asc|desc)}". This is a change from prior versions which behaved unpredictably
when capturing groups were present.

And this is all you need to know about the basic usage. More advanced options
are explained below.

Routes can also be restricted to a domain or subdomain. Just define a host
pattern to be matched. They can also have variables:

	r := mux.NewRouter()
	// Only matches if domain is "www.example.com".
	r.Host("www.example.com")
	// Matches a dynamic subdomain.
	r.Host("{subdomain:[a-z]+}.domain.com")

There are several other matchers that can be added. To match path prefixes:

	r.PathPrefix("/products/")

...or HTTP methods:

	r.Methods("GET", "POST")

...or URL schemes:

	r.Schemes("https")

...or header values:

	r.Headers("X-Requested-With", "XMLHttpRequest")

...or query values:

	r.Queries("key", "value")

...or to use a custom matcher function:

	r.MatcherFunc(func(r *http.Request, rm *RouteMatch) bool {
		return r.ProtoMajor == 0
	})

...and finally, it is possible to combine several matchers in a single route:

	r.HandleFunc("/products", ProductsHandler).
	  Host("www.example.com").
	  Methods("GET").
	  Schemes("http")

Setting the same matching conditions again and again can be boring, so we have
a way to group several routes that share the same requirements.
We call it "subrouting".

For example, let's say we have several URLs that should only match when the
host is "www.example.com". Create a route for that host and get a "subrouter"
from it:

	r := mux.NewRouter()
	s := r.Host("www.example.com").Subrouter()

Then register routes in the subrouter:

	s.HandleFunc("/products/", ProductsHandler)
	s.HandleFunc("/products/{key}", ProductHandler)
	s.HandleFunc("/articles/{category}/{id:[0-9]+}"), ArticleHandler)

The three URL paths we registered above will only be tested if the domain is
"www.example.com", because the subrouter is tested first. This is not
only convenient, but also optimizes request matching. You can create
subrouters combining any attribute matchers accepted by a route.

Subrouters can be used to create domain or path "namespaces": you define
subrouters in a central place and then parts of the app can register its
paths relatively to a given subrouter.

There's one more thing about subroutes. When a subrouter has a path prefix,
the inner routes use it as base for their paths:

	r := mux.NewRouter()
	s := r.PathPrefix("/products").Subrouter()
	// "/products/"
	s.HandleFunc("/", ProductsHandler)
	// "/products/{key}/"
	s.HandleFunc("/{key}/", ProductHandler)
	// "/products/{key}/details"
	s.HandleFunc("/{key}/details", ProductDetailsHandler)

Note that the path provided to PathPrefix() represents a "wildcard": calling
PathPrefix("/static/").Handler(...) means that the handler will be passed any
request that matches "/static/*". This makes it easy to serve static files with mux:

	func main() {
		var dir string

		flag.StringVar(&dir, "dir", ".", "the directory to serve files from. Defaults to the current dir")
		flag.Parse()
		r := mux.NewRouter()

		// This will serve files under http://localhost:8000/static/<filename>
		r.PathPrefix("/static/").Handler(http.StripPrefix("/static/", http.FileServer(http.Dir(dir))))

		srv := &http.Server{
			Handler:      r,
			Addr:         "127.0.0.1:8000",
			// Good practice: enforce timeouts for servers you create!
			WriteTimeout: 15 * time.Second,
			ReadTimeout:  15 * time.Second,
		}

		log.Fatal(srv.ListenAndServe())
	}

Now let's see how to build registered URLs.

Routes can be named. All routes that define a name can have their URLs built,
or "reversed". We define a name calling Name() on a route. For example:

	r := mux.NewRouter()
	r.HandleFunc("/articles/{category}/{id:[0-9]+}", ArticleHandler).
	  Name("article")

To build a URL, get the route and call the URL() method, passing a sequence of
key/value pairs for the route variables. For the previous route, we would do:

	url, err := r.Get("article").URL("category", "technology", "id", "42")

...and the result will be a url.URL with the following path:

	"/articles/technology/42"

This also works for host and query value variables:

	r := mux.NewRouter()
	r.Host("{subdomain}.domain.com").
	  Path("/articles/{category}/{id:[0-9]+}").
	  Queries("filter", "{filter}").
	  HandlerFunc(ArticleHandler).
	  Name("article")

	// url.String() will be "http://news.domain.com/articles/technology/42?filter=gorilla"
	url, err := r.Get("article").URL("subdomain", "news",
	                                 "category", "technology",
	                                 "id", "42",
	                                 "filter", "gorilla")

All variables defined in the route are required, and their values must
conform to the corresponding patterns. These requirements guarantee that a
generated URL will always match a registered route -- the only exception is
for explicitly defined "build-only" routes which never match.

Regex support also exists for matching Headers within a route. For example, we could do:

	r.HeadersRegexp("Content-Type", "application/(text|json)")

...and the route will match both requests with a Content-Type of `application/json` as well as
`application/text`

There's also a way to build only the URL host or path for a route:
use the methods URLHost() or URLPath() instead. For the previous route,
we would do:

	// "http://news.domain.com/"
	host, err := r.Get("article").URLHost("subdomain", "news")

	// "/articles/technology/42"
	path, err := r.Get("article").URLPath("category", "technology", "id", "42")

And if you use subrouters, host and path defined separately can be built
as well:

	r := mux.NewRouter()
	s := r.Host("{subdomain}.domain.com").Subrouter()
	s.Path("/articles/{category}/{id:[0-9]+}").
	  HandlerFunc(ArticleHandler).
	  Name("article")

	// "http://news.domain.com/articles/technology/42"
	url, err := r.Get("article").URL("subdomain", "news",
	                                 "category", "technology",
	                                 "id", "42")

Since **vX.Y.Z**, mux supports the addition of middlewares to a [Router](https://godoc.org/github.com/gorilla/mux#Router), which are executed if a
match is found (including subrouters). Middlewares are defined using the de facto standard type:

	type MiddlewareFunc func(http.Handler) http.Handler

Typically, the returned handler is a closure which does something with the http.ResponseWriter and http.Request passed to it, and then calls the handler passed as parameter to the MiddlewareFunc (closures can access variables from the context where they are created).

A very basic middleware which logs the URI of the request being handled could be written as:

	func simpleMw(next http.Handler) http.Handler {
		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			// Do stuff here
			log.Println(r.RequestURI)
			// Call the next handler, which can be another middleware in the chain, or the final handler.
			next.ServeHTTP(w, r)
		})
	}

Middlewares can be added to a router using `Router.Use()`:

	r := mux.NewRouter()
	r.HandleFunc("/", handler)
	r.Use(simpleMw)

A more complex authentication middleware, which maps session token to users, could be written as:

	// Define our struct
	type authenticationMiddleware struct {
		tokenUsers map[string]string
	}

	// Initialize it somewhere
	func (amw *authenticationMiddleware) Populate() {
		amw.tokenUsers["00000000"] = "user0"
		amw.tokenUsers["aaaaaaaa"] = "userA"
		amw.tokenUsers["05f717e5"] = "randomUser"
		amw.tokenUsers["deadbeef"] = "user0"
	}

	// Middleware function, which will be called for each request
	func (amw *authenticationMiddleware) Middleware(next http.Handler) http.Handler {
		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			token := r.Header.Get("X-Session-Token")

			if user, found := amw.tokenUsers[token]; found {
				// We found the token in our map
				log.Printf("Authenticated user %s\n", user)
				next.ServeHTTP(w, r)
			} else {
				http.Error(w, "Forbidden", 403)
			}
		})
	}

	r := mux.NewRouter()
	r.HandleFunc("/", handler)

	amw := authenticationMiddleware{}
	amw.Populate()

	r.Use(amw.Middleware)

Note: The handler chain will be stopped if your middleware doesn't call `next.ServeHTTP()` with the corresponding parameters. This can be used to abort a request if the middleware writer wants to.

*/
package mux
<|endoftext|>


<commit_before>// Package jp offers a highly performant json selector in the style of the jq command line
//
package jq
<commit_msg>- improving package docs<commit_after>// Package jp offers a highly performant json selector in the style of the jq command line
//
// Usage of this package involves the concept of an Op.  An Op is a transformation that converts a []byte into a []byte.
// To get started, use the Parse function to obtain an Op.
//
//     op, err := jq.Parse(".key")
//
// This will create an Op that will accept a JSON object in []byte format and return the value associated with "key."
//
package jq
<|endoftext|>


<commit_before>// Copyright 2014 Martin Angers and Contributors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Package fetchbot provides a simple and flexible web crawler that follows the robots.txt
policies and crawl delays.

It is very much a rewrite of gocrawl (https://github.com/PuerkitoBio/gocrawl) with a
simpler API, less features built-in, but at the same time more flexibility. As for Go
itself, sometimes less is more!

Installation

To install, simply run in a terminal:

    go get github.com/PuerkitoBio/gocrawl

The package has a single external dependency, robotstxt
(https://github.com/temoto/robotstxt-go). It also integrates the `iq_slice.go`
file from the iq package (https://github.com/kylelemons/iq).

The API documentation is available on godoc.org
(http://godoc.org/github.com/PuerkitoBio/fetchbot).

Changelog

- v0.1 : initial release.

Usage

The following example shows how to create and start a Fetcher, one way to send commands,
and how to stop the fetcher once all commands have been handled.

    package main

    import (
    	"fmt"
    	"net/http"

    	"github.com/PuerkitoBio/fetchbot"
    )

    func main() {
    	f := fetchbot.New(fetchbot.HandlerFunc(handler))
    	queue := f.Start()
    	queue.SendStringHead("http://google.com", "http://golang.org", "http://golang.org/doc")
    	queue.Close()
    }

    func handler(ctx *fetchbot.Context, res *http.Response, err error) {
    	if err != nil {
    		fmt.Printf("error: %s\n", err)
    		return
    	}
    	fmt.Printf("[%d] %s %s\n", res.StatusCode, ctx.Cmd.Method(), ctx.Cmd.URL())
    }

A more complex and complete example can be found in the repository, at /example/full/.

Basically, a Fetcher is an instance of a web crawler, independent of other Fetchers.
It receives Commands via the Queue, executes the requests, and calls a Handler to
process the responses. A Command is an interface that tells the Fetcher which URL to
fetch, and which HTTP method (verb) to use (i.e. "GET", "HEAD", ...).

A call to Fetcher.Start() returns the Queue associated with this Fetcher. This is the
thread-safe object that can be used to send more commands, or to stop the crawler.

Both the Command and the Handler are interfaces, and may be implemented in various ways.
They are defined like so:

    type Command interface {
    	URL() *url.URL
    	Method() string
    }
    type Handler interface {
    	Handle(*Context, *http.Response, error)
    }

A Context is a struct that holds the Command and the Queue, so that the Handler always
knows which Command initiated this call, and has a handle on the Queue.

A Handler is similar to the net/http Handler, and middleware-style combinations can
be built on top of it. In fact, there is even a HandlerFunc type so that simple functions
with the right signature can be used as Handlers (like net/http.HandlerFunc), and there
is also a multiplexer Mux that can be used to dispatch calls based on some criteria.

The Fetcher recognizes some interfaces other than Command, for more advanced needs.
If the Command implements the BasicAuthProvider interface, a Basic Authentication
header will be put in place with the given credentials to fetch the URL.

Similarly, the CookiesProvider and HeaderProvider interfaces offer the expected
features (setting cookies and header values on the request). The ReaderProvider
and ValuesProvider interfaces are also supported, although they should be mutually
exclusive as they both set the body of the request. If both are supported, the
ReaderProvider interface is used. It sets the body of the request (eg. for a "POST")
using the given io.Reader instance. The ValuesProvider does the same, but using
the given url.Values instance, and sets the Content-Type of the body to
"application/x-www-form-urlencoded" unless it is explicitly set by a HeaderProvider.

Since the Command is an interface, it can be a custom struct that holds additional
information, such as an ID for the URL (e.g. from a database), or a depth counter
so that the crawling stops at a certain depth, etc. For basic commands that don't
require additional information, the package provides the Cmd struct that implements
the Command interface.

By default, the Fetcher uses the net/http default Client. A different client can
be set on the Fetcher.HttpClient field. It also has a CrawlDelay field that is used
only if there is no delay specified by the robots.txt of a given host. The UserAgent
field sets the user agent string to use for the requests and to validate against
the robots.txt entries. Finally, the Fetcher has a WorkerIdleTTL field that sets
the duration that a worker goroutine can wait without receiving new commands to
fetch. If the idle time-to-live is reached, the worker goroutine is stopped and its
resources are released. This can be especially useful for long-running crawlers.

What fetchbot doesn't do - especially compared to gocrawl - is that it doesn't
keep track of already visited URLs, and it doesn't normalize the URLs. This is outside
the scope of this package - all commands sent on the Queue will be fetched.
Normalization can easily be done (e.g. using
https://github.com/PuerkitoBio/purell) before sending the Command to the Fetcher.
How to keep track of visited URLs depends on the use-case of the specific crawler,
but for an example, see /example/full/main.go.

License

The BSD 3-Clause license (http://opensource.org/licenses/BSD-3-Clause), the same as
the Go language. The iq_slice.go file is under the CDDL-1.0 license (details in
the source file).
*/
package fetchbot
<commit_msg>fix doc<commit_after>// Copyright 2014 Martin Angers and Contributors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Package fetchbot provides a simple and flexible web crawler that follows the robots.txt
policies and crawl delays.

It is very much a rewrite of gocrawl (https://github.com/PuerkitoBio/gocrawl) with a
simpler API, less features built-in, but at the same time more flexibility. As for Go
itself, sometimes less is more!

Installation

To install, simply run in a terminal:

    go get github.com/PuerkitoBio/gocrawl

The package has a single external dependency, robotstxt
(https://github.com/temoto/robotstxt-go). It also integrates the `iq_slice.go`
file from the iq package (https://github.com/kylelemons/iq).

The API documentation is available on godoc.org
(http://godoc.org/github.com/PuerkitoBio/fetchbot).

Changelog

- v0.1 : initial release.

Usage

The following example shows how to create and start a Fetcher, one way to send commands,
and how to stop the fetcher once all commands have been handled.

    package main

    import (
    	"fmt"
    	"net/http"

    	"github.com/PuerkitoBio/fetchbot"
    )

    func main() {
    	f := fetchbot.New(fetchbot.HandlerFunc(handler))
    	queue := f.Start()
    	queue.SendStringHead("http://google.com", "http://golang.org", "http://golang.org/doc")
    	queue.Close()
    }

    func handler(ctx *fetchbot.Context, res *http.Response, err error) {
    	if err != nil {
    		fmt.Printf("error: %s\n", err)
    		return
    	}
    	fmt.Printf("[%d] %s %s\n", res.StatusCode, ctx.Cmd.Method(), ctx.Cmd.URL())
    }

A more complex and complete example can be found in the repository, at /example/full/.

Basically, a Fetcher is an instance of a web crawler, independent of other Fetchers.
It receives Commands via the Queue, executes the requests, and calls a Handler to
process the responses. A Command is an interface that tells the Fetcher which URL to
fetch, and which HTTP method (verb) to use (i.e. "GET", "HEAD", ...).

A call to Fetcher.Start() returns the Queue associated with this Fetcher. This is the
thread-safe object that can be used to send more commands, or to stop the crawler.

Both the Command and the Handler are interfaces, and may be implemented in various ways.
They are defined like so:

    type Command interface {
    	URL() *url.URL
    	Method() string
    }
    type Handler interface {
    	Handle(*Context, *http.Response, error)
    }

A Context is a struct that holds the Command and the Queue, so that the Handler always
knows which Command initiated this call, and has a handle on the Queue.

A Handler is similar to the net/http Handler, and middleware-style combinations can
be built on top of it. In fact, there is even a HandlerFunc type so that simple functions
with the right signature can be used as Handlers (like net/http.HandlerFunc), and there
is also a multiplexer Mux that can be used to dispatch calls based on some criteria.

The Fetcher recognizes some interfaces other than Command, for more advanced needs.
If the Command implements the BasicAuthProvider interface, a Basic Authentication
header will be put in place with the given credentials to fetch the URL.

Similarly, the CookiesProvider and HeaderProvider interfaces offer the expected
features (setting cookies and header values on the request). The ReaderProvider
and ValuesProvider interfaces are also supported, although they should be mutually
exclusive as they both set the body of the request. If both are supported, the
ReaderProvider interface is used. It sets the body of the request (eg. for a "POST")
using the given io.Reader instance. The ValuesProvider does the same, but using
the given url.Values instance, and sets the Content-Type of the body to
"application/x-www-form-urlencoded" unless it is explicitly set by a HeaderProvider.

Since the Command is an interface, it can be a custom struct that holds additional
information, such as an ID for the URL (e.g. from a database), or a depth counter
so that the crawling stops at a certain depth, etc. For basic commands that don't
require additional information, the package provides the Cmd struct that implements
the Command interface.

By default, the Fetcher uses the net/http default Client to make requests. A different client can
be set on the Fetcher.HttpClient field. The Fetcher also has a CrawlDelay field that is used
only if there is no delay specified by the robots.txt of a given host. The UserAgent
field sets the user agent string to use for the requests and to validate against
the robots.txt entries. Finally, the Fetcher has a WorkerIdleTTL field that sets
the duration that a worker goroutine can wait without receiving new commands to
fetch. If the idle time-to-live is reached, the worker goroutine is stopped and its
resources are released. This can be especially useful for long-running crawlers.

What fetchbot doesn't do - especially compared to gocrawl - is that it doesn't
keep track of already visited URLs, and it doesn't normalize the URLs. This is outside
the scope of this package - all commands sent on the Queue will be fetched.
Normalization can easily be done (e.g. using
https://github.com/PuerkitoBio/purell) before sending the Command to the Fetcher.
How to keep track of visited URLs depends on the use-case of the specific crawler,
but for an example, see /example/full/main.go.

License

The BSD 3-Clause license (http://opensource.org/licenses/BSD-3-Clause), the same as
the Go language. The iq_slice.go file is under the CDDL-1.0 license (details in
the source file).
*/
package fetchbot
<|endoftext|>


<commit_before>/*
Package goon provides an autocaching interface to the app engine datastore
similar to the python NDB package.

Goon differs from the datastore package in varios ways: it remembers the
appengine Context, which need only be specified once at creation time; kinds
need not be specified as they are computed, by default, from a type's name;
keys are inferred from specially named fields on types, removing the need to
pass key objects around.

In general, the difference is that Goon's API is identical to the datastore API,
it's just shorter.

Keys in Goon are stored in the structs themselves. Below is an example struct
with a field to specify the id. There are also optional kind and parent fields.
See docs on the KeyError() function for a full specification.
	type User struct {
		Id    string `datastore:"-" goon:"id"`
		Name  string
	}

Thus, to get a User with id 2:
	userid := 2
	g := goon.NewGoon(r)
	u := User{Id: userid}
	g.Get(&u)

Key Specifications

src must be a S or *S for some struct type S. The key is extracted based on
various fields of S. If a field of type int64 or string has a struct tag
named goon with value "id", it is used as the key's id. If a field of type
*datastore.Key has a struct tag named goon with value "parent", it is used
as the key's parent. If a field of type string has a struct tag named goon
with value "kind", it is used as the key's kind. The "kind" field supports
an optional second parameter which is the default kind name. If no kind
field exists, the struct's name is used. These fields should all have
their datastore field marked as "-".

Example, with kind User:
	type User struct {
		Id    string `datastore:"-" goon:"id"`
		Read  time.Time
	}

Example, with kind U if _kind is the empty string:
	type User struct {
		_kind string `goon:"kind,U"`
		Id    string `datastore:"-" goon:"id"`
		Read  time.Time
	}

To override kind of a single entity to UserKind:
	u := User{_kind: "UserKind"}

An example with both parent and kind:
	type UserData struct {
		Id     string         `datastore:"-" goon:"id"`
		_kind  string         `goon:"kind,UD"`
		Parent *datastore.Key `datastore:"-" goon:"parent"`
		Data   []byte
	}

Features

Datastore interaction with: Get, GetMulti, Put, PutMulti, Delete, DeleteMulti, Queries.

All key-based operations backed by memory and memcache.

Per-request, in-memory cache: fetch the same key twice, the second request is served from local memory.

Intelligent multi support: running GetMulti on correctly fetches from memory, then memcache, then the datastore; each tier only sends keys off to the next one if they were missing.

Transactions use a separate context, but locally cache any results on success.

Automatic kind naming: struct names are inferred by reflection, removing the need to manually specify key kinds.

Simpler api than appengine/datastore.

API comparison between goon and datastore

put with incomplete key

datastore:

	type Group struct {
		Name string
	}
	c := appengine.NewContext(r)
	g := &Group{Name: "name"}
	k := datastore.NewIncompleteKey(c, "Group", nil)
	err := datastore.Put(c, k, g)

goon:

	type Group struct {
		Id   int64 `datastore:"-" goon:"id"`
		Name string
	}
	n := goon.NewGoon(r)
	g := &Group{Name: "name"}
	err := n.Put(g)

get with known key

datastore:

	type Group struct {
		Name string
	}
	c := appengine.NewContext(r)
	g := &Group{}
	k := datastore.NewKey(c, "Group", "", 1, nil)
	err := datastore.Get(c, k, g)

goon:

	type Group struct {
		Id   int64 `datastore:"-" goon:"id"`
		Name string
	}
	n := goon.NewGoon(r)
	g := &Group{Id: 1}
	err := n.Get(g)

*/
package goon
<commit_msg>Correct KeyError note<commit_after>/*
Package goon provides an autocaching interface to the app engine datastore
similar to the python NDB package.

Goon differs from the datastore package in varios ways: it remembers the
appengine Context, which need only be specified once at creation time; kinds
need not be specified as they are computed, by default, from a type's name;
keys are inferred from specially named fields on types, removing the need to
pass key objects around.

In general, the difference is that Goon's API is identical to the datastore API,
it's just shorter.

Keys in Goon are stored in the structs themselves. Below is an example struct
with a field to specify the id (see the Key Specifications section below for
full documentation).
	type User struct {
		Id    string `datastore:"-" goon:"id"`
		Name  string
	}

Thus, to get a User with id 2:
	userid := 2
	g := goon.NewGoon(r)
	u := User{Id: userid}
	g.Get(&u)

Key Specifications

src must be a S or *S for some struct type S. The key is extracted based on
various fields of S. If a field of type int64 or string has a struct tag
named goon with value "id", it is used as the key's id. If a field of type
*datastore.Key has a struct tag named goon with value "parent", it is used
as the key's parent. If a field of type string has a struct tag named goon
with value "kind", it is used as the key's kind. The "kind" field supports
an optional second parameter which is the default kind name. If no kind
field exists, the struct's name is used. These fields should all have
their datastore field marked as "-".

Example, with kind User:
	type User struct {
		Id    string `datastore:"-" goon:"id"`
		Read  time.Time
	}

Example, with kind U if _kind is the empty string:
	type User struct {
		_kind string `goon:"kind,U"`
		Id    string `datastore:"-" goon:"id"`
		Read  time.Time
	}

To override kind of a single entity to UserKind:
	u := User{_kind: "UserKind"}

An example with both parent and kind:
	type UserData struct {
		Id     string         `datastore:"-" goon:"id"`
		_kind  string         `goon:"kind,UD"`
		Parent *datastore.Key `datastore:"-" goon:"parent"`
		Data   []byte
	}

Features

Datastore interaction with: Get, GetMulti, Put, PutMulti, Delete, DeleteMulti, Queries.

All key-based operations backed by memory and memcache.

Per-request, in-memory cache: fetch the same key twice, the second request is served from local memory.

Intelligent multi support: running GetMulti on correctly fetches from memory, then memcache, then the datastore; each tier only sends keys off to the next one if they were missing.

Transactions use a separate context, but locally cache any results on success.

Automatic kind naming: struct names are inferred by reflection, removing the need to manually specify key kinds.

Simpler api than appengine/datastore.

API comparison between goon and datastore

put with incomplete key

datastore:

	type Group struct {
		Name string
	}
	c := appengine.NewContext(r)
	g := &Group{Name: "name"}
	k := datastore.NewIncompleteKey(c, "Group", nil)
	err := datastore.Put(c, k, g)

goon:

	type Group struct {
		Id   int64 `datastore:"-" goon:"id"`
		Name string
	}
	n := goon.NewGoon(r)
	g := &Group{Name: "name"}
	err := n.Put(g)

get with known key

datastore:

	type Group struct {
		Name string
	}
	c := appengine.NewContext(r)
	g := &Group{}
	k := datastore.NewKey(c, "Group", "", 1, nil)
	err := datastore.Get(c, k, g)

goon:

	type Group struct {
		Id   int64 `datastore:"-" goon:"id"`
		Name string
	}
	n := goon.NewGoon(r)
	g := &Group{Id: 1}
	err := n.Get(g)

*/
package goon
<|endoftext|>


<commit_before>/*
Package dot implements Graphviz's dot language in Go.

Basic Graph creation:

	g := dot.NewGraph("G")
	g.SetType(dot.DIGRAPH)
	...
	g.AddEdge(dot.NewNode("A"), dot.NewNode("B"))
	...
	fmt.Sprint(g)
*/
package dot

import (
	"errors"
	"fmt"
	"sort"
	"strings"
)

var AttributeError = errors.New("Invalid Attribute")

var graphAttributes = []string{"Damping", "K", "URL", "aspect", "bb", "bgcolor",
	"center", "charset", "clusterrank", "colorscheme", "comment", "compound",
	"concentrate", "defaultdist", "dim", "dimen", "diredgeconstraints",
	"dpi", "epsilon", "esep", "fontcolor", "fontname", "fontnames",
	"fontpath", "fontsize", "id", "label", "labeljust", "labelloc",
	"landscape", "layers", "layersep", "layout", "levels", "levelsgap",
	"lheight", "lp", "lwidth", "margin", "maxiter", "mclimit", "mindist",
	"mode", "model", "mosek", "nodesep", "nojustify", "normalize", "nslimit",
	"nslimit1", "ordering", "orientation", "outputorder", "overlap",
	"overlap_scaling", "pack", "packmode", "pad", "page", "pagedir",
	"quadtree", "quantum", "rankdir", "ranksep", "ratio", "remincross",
	"repulsiveforce", "resolution", "root", "rotate", "searchsize", "sep",
	"showboxes", "size", "smoothing", "sortv", "splines", "start",
	"stylesheet", "target", "truecolor", "viewport", "voro_margin",
	"rank"}

var edgeAttributes = []string{"URL", "arrowhead", "arrowsize", "arrowtail",
	"color", "colorscheme", "comment", "constraint", "decorate", "dir",
	"edgeURL", "edgehref", "edgetarget", "edgetooltip", "fontcolor",
	"fontname", "fontsize", "headURL", "headclip", "headhref", "headlabel",
	"headport", "headtarget", "headtooltip", "href", "id", "label",
	"labelURL", "labelangle", "labeldistance", "labelfloat", "labelfontcolor",
	"labelfontname", "labelfontsize", "labelhref", "labeltarget",
	"labeltooltip", "layer", "len", "lhead", "lp", "ltail", "minlen",
	"nojustify", "penwidth", "pos", "samehead", "sametail", "showboxes",
	"style", "tailURL", "tailclip", "tailhref", "taillabel", "tailport",
	"tailtarget", "tailtooltip", "target", "tooltip", "weight",
	// for subgraphs
	"rank"}

var nodeAttributes = []string{"URL", "color", "colorscheme", "comment",
	"distortion", "fillcolor", "fixedsize", "fontcolor", "fontname",
	"fontsize", "group", "height", "id", "image", "imagescale", "label",
	"labelloc", "layer", "margin", "nojustify", "orientation", "penwidth",
	"peripheries", "pin", "pos", "rects", "regular", "root", "samplepoints",
	"shape", "shapefile", "showboxes", "sides", "skew", "sortv", "style",
	"target", "tooltip", "vertices", "width", "z",
	// The following are attributes dot2tex
	"texlbl", "texmode"}

var ClusterAttributes = []string{"K", "URL", "bgcolor", "color", "colorscheme",
	"fillcolor", "fontcolor", "fontname", "fontsize", "label", "labeljust",
	"labelloc", "lheight", "lp", "lwidth", "nojustify", "pencolor",
	"penwidth", "peripheries", "sortv", "style", "target", "tooltip"}

var dotKeywords = []string{"graph", "subgraph", "digraph", "node", "edge", "strict"}

type GraphType int

const (
	DIGRAPH GraphType = iota
	GRAPH
)

// Fields common to all graph object types
type common struct {
	_type       string
	name        string
	attributes  map[string]string
	sequence    int
	parentGraph *Graph
}

type GraphObject interface {
	Type() string
	Get(string) string
	Set(string, string) error
	GetParentGraph() *Graph
	SetParentGraph(g *Graph)
	Sequence() int
}

type graphObjects []GraphObject

func (gol graphObjects) Len() int {
	return len(gol)
}

func (gol graphObjects) Less(i, j int) bool {
	return gol[i].Sequence() < gol[i].Sequence()
}

func (gol graphObjects) Swap(i, j int) {
	gol[i], gol[j] = gol[j], gol[i]
}

type Graph struct {
	common
	strict               bool
	graphType            GraphType
	supressDisconnected  bool
	simplify             bool
	currentChildSequence int
	nodes                map[string][]*Node
	edges                map[string][]*Edge
	subgraphs            map[string][]*SubGraph
}

func NewGraph(name string) *Graph {
	g := &Graph{
		common: common{
			_type:      "graph",
			name:       name,
			attributes: make(map[string]string, 0),
		},
		nodes:                make(map[string][]*Node, 0),
		edges:                make(map[string][]*Edge, 0),
		subgraphs:            make(map[string][]*SubGraph, 0),
		currentChildSequence: 1,
	}
	g.SetParentGraph(g)
	return g
}

type SubGraph struct {
	Graph
}

func NewSubgraph(name string) *SubGraph {
	result := &SubGraph{
		Graph: *NewGraph(name),
	}
	result._type = "subgraph"
	return result
}

func indexInSlice(slice []string, toFind string) int {
	for i, v := range slice {
		if v == toFind {
			return i
		}
	}
	return -1
}

func needsQuotes(s string) bool {
	if indexInSlice(dotKeywords, s) != -1 {
		return false
	}

	return true
}

func quoteIfNecessary(s string) (result string) {
	if needsQuotes(s) {
		s = strings.Replace(s, "\"", "\\\"", -1)
		s = strings.Replace(s, "\n", "\\n", -1)
		s = strings.Replace(s, "\r", "\\r", -1)
		s = "\"" + s + "\""
	}
	return s
}

func validAttribute(attributeCollection []string, attributeName string) bool {
	return indexInSlice(attributeCollection, attributeName) != -1
}

func validGraphAttribute(attributeName string) bool {
	return validAttribute(graphAttributes, attributeName)
}

func validNodeAttribute(attributeName string) bool {
	return validAttribute(nodeAttributes, attributeName)
}

func sortedKeys(sourceMap map[string]string) []string {
	keys := make([]string, 0, len(sourceMap))
	for k, _ := range sourceMap {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	return keys
}

////////////////////////////////////////////////////////////////////////////////////////

func (gt GraphType) String() string {
	if gt == DIGRAPH {
		return "digraph"
	} else if gt == GRAPH {
		return "graph"
	}
	return "(invalid)"
}

func (c *common) Type() string {
	return c._type
}

func (c *common) GetParentGraph() *Graph {
	return c.parentGraph
}

func (c *common) SetParentGraph(g *Graph) {
	c.parentGraph = g
}

func (c *common) Sequence() int {
	return c.sequence
}

func (c *common) Get(attributeName string) string {
	return c.attributes[attributeName]
}

func (c *common) Set(attributeName, attributeValue string) error {
	c.attributes[attributeName] = attributeValue
	return nil
}

func setAttribute(validAttributes []string, attributes map[string]string, attributeName, attributeValue string) error {
	if validAttribute(validAttributes, attributeName) {
		attributes[attributeName] = attributeValue
		return nil
	}
	return AttributeError
}

func (g *Graph) Set(attributeName, attributeValue string) error {
	return setAttribute(graphAttributes, g.common.attributes, attributeName, attributeValue)
}

func (n *Node) Set(attributeName, attributeValue string) error {
	return setAttribute(nodeAttributes, n.common.attributes, attributeName, attributeValue)
}

func (e *Edge) Set(attributeName, attributeValue string) error {
	return setAttribute(edgeAttributes, e.common.attributes, attributeName, attributeValue)
}

func (c *common) setSequence(sequence int) {
	c.sequence = sequence
}

func (g *Graph) SetType(t GraphType) {
	g.graphType = t
}

func (c common) Name() string {
	return c.name
}

func (g *Graph) GetRoot() (result *Graph) {
	result = g
	for parent := g.GetParentGraph(); parent != result; parent = parent.GetParentGraph() {
		result = parent
	}
	return result
}

func (g *Graph) getNextSequenceNumber() (next int) {
	next = g.currentChildSequence
	g.currentChildSequence += 1
	return
}
func (g *Graph) AddNode(n *Node) {
	name := n.Name()
	if _, ok := g.nodes[name]; !ok {
		g.nodes[name] = make([]*Node, 0)
	}
	n.setSequence(g.getNextSequenceNumber())
	n.SetParentGraph(g.GetParentGraph())
	g.nodes[name] = append(g.nodes[name], n)
}

func (g *Graph) AddEdge(e *Edge) {
	name := e.Name()
	if _, ok := g.edges[name]; !ok {
		g.edges[name] = make([]*Edge, 0)
	}
	e.setSequence(g.getNextSequenceNumber())
	e.SetParentGraph(g.GetParentGraph())
	g.edges[name] = append(g.edges[name], e)
}

func (g *Graph) AddSubgraph(sg *SubGraph) {
	name := sg.Name()
	if _, ok := g.subgraphs[name]; !ok {
		g.subgraphs[name] = make([]*SubGraph, 0)
	}
	sg.setSequence(g.getNextSequenceNumber())
	g.subgraphs[name] = append(g.subgraphs[name], sg)
}

func (g *Graph) GetSubgraphs() (result []*SubGraph) {
	result = make([]*SubGraph, 0)
	for _, sgs := range g.subgraphs {
		for _, sg := range sgs {
			result = append(result, sg)
		}
	}
	return result
}

func (g Graph) String() string {
	parts := make([]string, 0)
	if g.strict {
		parts = append(parts, "strict ")
	}
	if g.name == "" {
		parts = append(parts, "{\n")
	} else {
		parts = append(parts, fmt.Sprintf("%s %s {\n", g.graphType, g.name))
	}

	attrs := make([]string, 0)
	for _, key := range sortedKeys(g.attributes) {
		attrs = append(attrs, key+"="+quoteIfNecessary(g.attributes[key]))
	}
	if len(attrs) > 0 {
		parts = append(parts, strings.Join(attrs, ";\n"))
		parts = append(parts, ";\n")
	}

	objectList := make(graphObjects, 0)

	for _, nodes := range g.nodes {
		for _, node := range nodes {
			objectList = append(objectList, node)
		}
	}
	for _, edges := range g.edges {
		for _, edge := range edges {
			objectList = append(objectList, edge)
		}
	}
	for _, subgraphs := range g.subgraphs {
		for _, subgraph := range subgraphs {
			objectList = append(objectList, subgraph)
		}
	}
	sort.Sort(objectList)

	for _, obj := range objectList {
		//@todo type-based decision making re: supressDisconnected and simplify
		//switch o := obj.(type) {
		//case *Node:
		//}
		parts = append(parts, fmt.Sprintf("%s\n", obj))
	}

	parts = append(parts, "}\n")
	return strings.Join(parts, "")
}

type Node struct {
	common
}

func NewNode(name string) *Node {
	return &Node{
		common{
			name:       name,
			attributes: make(map[string]string, 0),
		},
	}
}

func (n Node) String() string {

	name := n.name
	parts := make([]string, 0)

	attrs := make([]string, 0)
	for _, key := range sortedKeys(n.attributes) {
		attrs = append(attrs, key+"="+quoteIfNecessary(n.attributes[key]))
	}
	if len(attrs) > 0 {
		parts = append(parts, strings.Join(attrs, ", "))
	}

	//@todo don't print if node is empty
	if len(parts) > 0 {
		name += " [" + strings.Join(parts, ", ") + "]"
	}

	return name + ";"
}

type Edge struct {
	common
	points [2]*Node
}

func NewEdge(src, dst *Node) *Edge {
	return &Edge{
		common{
			_type:      "edge",
			attributes: make(map[string]string, 0),
		},
		[2]*Node{src, dst},
	}
}

func (e Edge) Source() *Node {
	return e.points[0]
}

func (e Edge) Destination() *Node {
	return e.points[1]
}

func (e Edge) String() string {
	src, dst := e.Source(), e.Destination()
	parts := make([]string, 0)

	parts = append(parts, src.Name())

	parent := e.GetParentGraph()
	if parent != nil && parent.GetRoot() != nil && parent.GetRoot().graphType == DIGRAPH {
		parts = append(parts, "->")
	} else {
		parts = append(parts, "--")
	}
	parts = append(parts, dst.Name())
	//parts = append(parts, fmt.Sprint(dst))

	attrs := make([]string, 0)
	for _, key := range sortedKeys(e.attributes) {
		attrs = append(attrs, key+"="+"\""+e.attributes[key]+"\"")
	}
	if len(attrs) > 0 {
		parts = append(parts, " [")
		parts = append(parts, strings.Join(attrs, ", "))
		parts = append(parts, "] \n")
	}

	return strings.Join(parts, " ")
}

func init() {
	sort.Strings(graphAttributes)
	sort.Strings(nodeAttributes)
	sort.Strings(edgeAttributes)
	sort.Strings(ClusterAttributes)
}
<commit_msg>Hide clusterAttributes<commit_after>/*
Package dot implements Graphviz's dot language in Go.

Basic Graph creation:

	g := dot.NewGraph("G")
	g.SetType(dot.DIGRAPH)
	...
	g.AddEdge(dot.NewNode("A"), dot.NewNode("B"))
	...
	fmt.Sprint(g)
*/
package dot

import (
	"errors"
	"fmt"
	"sort"
	"strings"
)

var AttributeError = errors.New("Invalid Attribute")

var graphAttributes = []string{"Damping", "K", "URL", "aspect", "bb", "bgcolor",
	"center", "charset", "clusterrank", "colorscheme", "comment", "compound",
	"concentrate", "defaultdist", "dim", "dimen", "diredgeconstraints",
	"dpi", "epsilon", "esep", "fontcolor", "fontname", "fontnames",
	"fontpath", "fontsize", "id", "label", "labeljust", "labelloc",
	"landscape", "layers", "layersep", "layout", "levels", "levelsgap",
	"lheight", "lp", "lwidth", "margin", "maxiter", "mclimit", "mindist",
	"mode", "model", "mosek", "nodesep", "nojustify", "normalize", "nslimit",
	"nslimit1", "ordering", "orientation", "outputorder", "overlap",
	"overlap_scaling", "pack", "packmode", "pad", "page", "pagedir",
	"quadtree", "quantum", "rankdir", "ranksep", "ratio", "remincross",
	"repulsiveforce", "resolution", "root", "rotate", "searchsize", "sep",
	"showboxes", "size", "smoothing", "sortv", "splines", "start",
	"stylesheet", "target", "truecolor", "viewport", "voro_margin",
	"rank"}

var edgeAttributes = []string{"URL", "arrowhead", "arrowsize", "arrowtail",
	"color", "colorscheme", "comment", "constraint", "decorate", "dir",
	"edgeURL", "edgehref", "edgetarget", "edgetooltip", "fontcolor",
	"fontname", "fontsize", "headURL", "headclip", "headhref", "headlabel",
	"headport", "headtarget", "headtooltip", "href", "id", "label",
	"labelURL", "labelangle", "labeldistance", "labelfloat", "labelfontcolor",
	"labelfontname", "labelfontsize", "labelhref", "labeltarget",
	"labeltooltip", "layer", "len", "lhead", "lp", "ltail", "minlen",
	"nojustify", "penwidth", "pos", "samehead", "sametail", "showboxes",
	"style", "tailURL", "tailclip", "tailhref", "taillabel", "tailport",
	"tailtarget", "tailtooltip", "target", "tooltip", "weight",
	// for subgraphs
	"rank"}

var nodeAttributes = []string{"URL", "color", "colorscheme", "comment",
	"distortion", "fillcolor", "fixedsize", "fontcolor", "fontname",
	"fontsize", "group", "height", "id", "image", "imagescale", "label",
	"labelloc", "layer", "margin", "nojustify", "orientation", "penwidth",
	"peripheries", "pin", "pos", "rects", "regular", "root", "samplepoints",
	"shape", "shapefile", "showboxes", "sides", "skew", "sortv", "style",
	"target", "tooltip", "vertices", "width", "z",
	// The following are attributes dot2tex
	"texlbl", "texmode"}

var clusterAttributes = []string{"K", "URL", "bgcolor", "color", "colorscheme",
	"fillcolor", "fontcolor", "fontname", "fontsize", "label", "labeljust",
	"labelloc", "lheight", "lp", "lwidth", "nojustify", "pencolor",
	"penwidth", "peripheries", "sortv", "style", "target", "tooltip"}

var dotKeywords = []string{"graph", "subgraph", "digraph", "node", "edge", "strict"}

type GraphType int

const (
	DIGRAPH GraphType = iota
	GRAPH
)

// Fields common to all graph object types
type common struct {
	_type       string
	name        string
	attributes  map[string]string
	sequence    int
	parentGraph *Graph
}

type GraphObject interface {
	Type() string
	Get(string) string
	Set(string, string) error
	GetParentGraph() *Graph
	SetParentGraph(g *Graph)
	Sequence() int
}

type graphObjects []GraphObject

func (gol graphObjects) Len() int {
	return len(gol)
}

func (gol graphObjects) Less(i, j int) bool {
	return gol[i].Sequence() < gol[i].Sequence()
}

func (gol graphObjects) Swap(i, j int) {
	gol[i], gol[j] = gol[j], gol[i]
}

type Graph struct {
	common
	strict               bool
	graphType            GraphType
	supressDisconnected  bool
	simplify             bool
	currentChildSequence int
	nodes                map[string][]*Node
	edges                map[string][]*Edge
	subgraphs            map[string][]*SubGraph
}

func NewGraph(name string) *Graph {
	g := &Graph{
		common: common{
			_type:      "graph",
			name:       name,
			attributes: make(map[string]string, 0),
		},
		nodes:                make(map[string][]*Node, 0),
		edges:                make(map[string][]*Edge, 0),
		subgraphs:            make(map[string][]*SubGraph, 0),
		currentChildSequence: 1,
	}
	g.SetParentGraph(g)
	return g
}

type SubGraph struct {
	Graph
}

func NewSubgraph(name string) *SubGraph {
	result := &SubGraph{
		Graph: *NewGraph(name),
	}
	result._type = "subgraph"
	return result
}

func indexInSlice(slice []string, toFind string) int {
	for i, v := range slice {
		if v == toFind {
			return i
		}
	}
	return -1
}

func needsQuotes(s string) bool {
	if indexInSlice(dotKeywords, s) != -1 {
		return false
	}

	return true
}

func quoteIfNecessary(s string) (result string) {
	if needsQuotes(s) {
		s = strings.Replace(s, "\"", "\\\"", -1)
		s = strings.Replace(s, "\n", "\\n", -1)
		s = strings.Replace(s, "\r", "\\r", -1)
		s = "\"" + s + "\""
	}
	return s
}

func validAttribute(attributeCollection []string, attributeName string) bool {
	return indexInSlice(attributeCollection, attributeName) != -1
}

func validGraphAttribute(attributeName string) bool {
	return validAttribute(graphAttributes, attributeName)
}

func validNodeAttribute(attributeName string) bool {
	return validAttribute(nodeAttributes, attributeName)
}

func sortedKeys(sourceMap map[string]string) []string {
	keys := make([]string, 0, len(sourceMap))
	for k, _ := range sourceMap {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	return keys
}

////////////////////////////////////////////////////////////////////////////////////////

func (gt GraphType) String() string {
	if gt == DIGRAPH {
		return "digraph"
	} else if gt == GRAPH {
		return "graph"
	}
	return "(invalid)"
}

func (c *common) Type() string {
	return c._type
}

func (c *common) GetParentGraph() *Graph {
	return c.parentGraph
}

func (c *common) SetParentGraph(g *Graph) {
	c.parentGraph = g
}

func (c *common) Sequence() int {
	return c.sequence
}

func (c *common) Get(attributeName string) string {
	return c.attributes[attributeName]
}

func (c *common) Set(attributeName, attributeValue string) error {
	c.attributes[attributeName] = attributeValue
	return nil
}

func setAttribute(validAttributes []string, attributes map[string]string, attributeName, attributeValue string) error {
	if validAttribute(validAttributes, attributeName) {
		attributes[attributeName] = attributeValue
		return nil
	}
	return AttributeError
}

func (g *Graph) Set(attributeName, attributeValue string) error {
	return setAttribute(graphAttributes, g.common.attributes, attributeName, attributeValue)
}

func (n *Node) Set(attributeName, attributeValue string) error {
	return setAttribute(nodeAttributes, n.common.attributes, attributeName, attributeValue)
}

func (e *Edge) Set(attributeName, attributeValue string) error {
	return setAttribute(edgeAttributes, e.common.attributes, attributeName, attributeValue)
}

func (c *common) setSequence(sequence int) {
	c.sequence = sequence
}

func (g *Graph) SetType(t GraphType) {
	g.graphType = t
}

func (c common) Name() string {
	return c.name
}

func (g *Graph) GetRoot() (result *Graph) {
	result = g
	for parent := g.GetParentGraph(); parent != result; parent = parent.GetParentGraph() {
		result = parent
	}
	return result
}

func (g *Graph) getNextSequenceNumber() (next int) {
	next = g.currentChildSequence
	g.currentChildSequence += 1
	return
}
func (g *Graph) AddNode(n *Node) {
	name := n.Name()
	if _, ok := g.nodes[name]; !ok {
		g.nodes[name] = make([]*Node, 0)
	}
	n.setSequence(g.getNextSequenceNumber())
	n.SetParentGraph(g.GetParentGraph())
	g.nodes[name] = append(g.nodes[name], n)
}

func (g *Graph) AddEdge(e *Edge) {
	name := e.Name()
	if _, ok := g.edges[name]; !ok {
		g.edges[name] = make([]*Edge, 0)
	}
	e.setSequence(g.getNextSequenceNumber())
	e.SetParentGraph(g.GetParentGraph())
	g.edges[name] = append(g.edges[name], e)
}

func (g *Graph) AddSubgraph(sg *SubGraph) {
	name := sg.Name()
	if _, ok := g.subgraphs[name]; !ok {
		g.subgraphs[name] = make([]*SubGraph, 0)
	}
	sg.setSequence(g.getNextSequenceNumber())
	g.subgraphs[name] = append(g.subgraphs[name], sg)
}

func (g *Graph) GetSubgraphs() (result []*SubGraph) {
	result = make([]*SubGraph, 0)
	for _, sgs := range g.subgraphs {
		for _, sg := range sgs {
			result = append(result, sg)
		}
	}
	return result
}

func (g Graph) String() string {
	parts := make([]string, 0)
	if g.strict {
		parts = append(parts, "strict ")
	}
	if g.name == "" {
		parts = append(parts, "{\n")
	} else {
		parts = append(parts, fmt.Sprintf("%s %s {\n", g.graphType, g.name))
	}

	attrs := make([]string, 0)
	for _, key := range sortedKeys(g.attributes) {
		attrs = append(attrs, key+"="+quoteIfNecessary(g.attributes[key]))
	}
	if len(attrs) > 0 {
		parts = append(parts, strings.Join(attrs, ";\n"))
		parts = append(parts, ";\n")
	}

	objectList := make(graphObjects, 0)

	for _, nodes := range g.nodes {
		for _, node := range nodes {
			objectList = append(objectList, node)
		}
	}
	for _, edges := range g.edges {
		for _, edge := range edges {
			objectList = append(objectList, edge)
		}
	}
	for _, subgraphs := range g.subgraphs {
		for _, subgraph := range subgraphs {
			objectList = append(objectList, subgraph)
		}
	}
	sort.Sort(objectList)

	for _, obj := range objectList {
		//@todo type-based decision making re: supressDisconnected and simplify
		//switch o := obj.(type) {
		//case *Node:
		//}
		parts = append(parts, fmt.Sprintf("%s\n", obj))
	}

	parts = append(parts, "}\n")
	return strings.Join(parts, "")
}

type Node struct {
	common
}

func NewNode(name string) *Node {
	return &Node{
		common{
			name:       name,
			attributes: make(map[string]string, 0),
		},
	}
}

func (n Node) String() string {

	name := n.name
	parts := make([]string, 0)

	attrs := make([]string, 0)
	for _, key := range sortedKeys(n.attributes) {
		attrs = append(attrs, key+"="+quoteIfNecessary(n.attributes[key]))
	}
	if len(attrs) > 0 {
		parts = append(parts, strings.Join(attrs, ", "))
	}

	//@todo don't print if node is empty
	if len(parts) > 0 {
		name += " [" + strings.Join(parts, ", ") + "]"
	}

	return name + ";"
}

type Edge struct {
	common
	points [2]*Node
}

func NewEdge(src, dst *Node) *Edge {
	return &Edge{
		common{
			_type:      "edge",
			attributes: make(map[string]string, 0),
		},
		[2]*Node{src, dst},
	}
}

func (e Edge) Source() *Node {
	return e.points[0]
}

func (e Edge) Destination() *Node {
	return e.points[1]
}

func (e Edge) String() string {
	src, dst := e.Source(), e.Destination()
	parts := make([]string, 0)

	parts = append(parts, src.Name())

	parent := e.GetParentGraph()
	if parent != nil && parent.GetRoot() != nil && parent.GetRoot().graphType == DIGRAPH {
		parts = append(parts, "->")
	} else {
		parts = append(parts, "--")
	}
	parts = append(parts, dst.Name())
	//parts = append(parts, fmt.Sprint(dst))

	attrs := make([]string, 0)
	for _, key := range sortedKeys(e.attributes) {
		attrs = append(attrs, key+"="+"\""+e.attributes[key]+"\"")
	}
	if len(attrs) > 0 {
		parts = append(parts, " [")
		parts = append(parts, strings.Join(attrs, ", "))
		parts = append(parts, "] \n")
	}

	return strings.Join(parts, " ")
}

func init() {
	sort.Strings(graphAttributes)
	sort.Strings(nodeAttributes)
	sort.Strings(edgeAttributes)
	sort.Strings(ClusterAttributes)
}
<|endoftext|>


<commit_before>package lmdb

/*
#include <stdlib.h>
#include <stdio.h>
#include "lmdb.h"
*/
import "C"

import (
	"errors"
	"fmt"
	"os"
	"runtime"
	"unsafe"
)

// success is a value returned from the LMDB API to indicate a successful call.
// The functions in this API this behavior and its use is not required.
const success = C.MDB_SUCCESS

const (
	// Flags for Env.Open.
	//
	// See mdb_env_open
	FixedMap    = C.MDB_FIXEDMAP   // Danger zone. Map memory at a fixed address.
	NoSubdir    = C.MDB_NOSUBDIR   // Argument to Open is a file, not a directory.
	Readonly    = C.MDB_RDONLY     // Used in several functions to denote an object as readonly.
	WriteMap    = C.MDB_WRITEMAP   // Use a writable memory map.
	NoMetaSync  = C.MDB_NOMETASYNC // Don't fsync metapage after commit.
	NoSync      = C.MDB_NOSYNC     // Don't fsync after commit.
	MapAsync    = C.MDB_MAPASYNC   // Flush asynchronously when using the WriteMap flag.
	NoTLS       = C.MDB_NOTLS      // Danger zone. Tie reader locktable slots to Txn objects instead of threads.
	NoLock      = C.MDB_NOLOCK     // Danger zone. LMDB does not use any locks. All transactions must serialize.
	NoReadahead = C.MDB_NORDAHEAD  // Disable readahead. Requires OS support.
	NoMemInit   = C.MDB_NOMEMINIT  // Disable LMDB memory initialization.
)

// DBI is a handle for a database in an Env.
//
// See MDB_dbi
type DBI C.MDB_dbi

// Env is opaque structure for a database environment.  A DB environment
// supports multiple databases, all residing in the same shared-memory map.
//
// See MDB_env.
type Env struct {
	_env *C.MDB_env
}

// NewEnv allocates and initializes a new Env.
//
// See mdb_env_create.
func NewEnv() (*Env, error) {
	var _env *C.MDB_env
	ret := C.mdb_env_create(&_env)
	if ret != success {
		return nil, errno(ret)
	}
	return &Env{_env}, nil
}

// Open an environment handle. If this function fails Close() must be called to
// discard the Env handle.  Open passes flags|NoTLS to mdb_env_open.
//
// See mdb_env_open.
func (env *Env) Open(path string, flags uint, mode os.FileMode) error {
	cpath := C.CString(path)
	defer C.free(unsafe.Pointer(cpath))
	ret := C.mdb_env_open(env._env, cpath, C.uint(NoTLS|flags), C.mdb_mode_t(mode))
	return errno(ret)
}

// Close shuts down the environment and releases the memory map.
//
// See mdb_env_close.
func (env *Env) Close() error {
	if env._env == nil {
		return errors.New("Environment already closed")
	}
	C.mdb_env_close(env._env)
	env._env = nil
	return nil
}

// Copy copies the data in env to an environment at path.
//
// See mdb_env_copy.
func (env *Env) Copy(path string) error {
	cpath := C.CString(path)
	defer C.free(unsafe.Pointer(cpath))
	ret := C.mdb_env_copy(env._env, cpath)
	return errno(ret)
}

// CopyFlag copies the data in env to an environment at path created with flags.
//
// See mdb_env_copy2.
func (env *Env) CopyFlag(path string, flags uint) error {
	cpath := C.CString(path)
	defer C.free(unsafe.Pointer(cpath))
	ret := C.mdb_env_copy2(env._env, cpath, C.uint(flags))
	return errno(ret)
}

// Statistics for a database in the environment
//
// See MDB_stat.
type Stat struct {
	PSize         uint   // Size of a database page. This is currently the same for all databases.
	Depth         uint   // Depth (height) of the B-tree
	BranchPages   uint64 // Number of internal (non-leaf) pages
	LeafPages     uint64 // Number of leaf pages
	OverflowPages uint64 // Number of overflow pages
	Entries       uint64 // Number of data items
}

// Stat returns statistics about the environment.
//
// See mdb_env_stat.
func (env *Env) Stat() (*Stat, error) {
	var _stat C.MDB_stat
	ret := C.mdb_env_stat(env._env, &_stat)
	if ret != success {
		return nil, errno(ret)
	}
	stat := Stat{PSize: uint(_stat.ms_psize),
		Depth:         uint(_stat.ms_depth),
		BranchPages:   uint64(_stat.ms_branch_pages),
		LeafPages:     uint64(_stat.ms_leaf_pages),
		OverflowPages: uint64(_stat.ms_overflow_pages),
		Entries:       uint64(_stat.ms_entries)}
	return &stat, nil
}

// Information about the environment.
//
// See MDB_envinfo.
type EnvInfo struct {
	MapSize    int64 // Size of the data memory map
	LastPNO    int64 // ID of the last used page
	LastTxnID  int64 // ID of the last committed transaction
	MaxReaders uint  // maximum number of threads for the environment
	NumReaders uint  // maximum number of threads used in the environment
}

// Info returns information about the environment.
//
// See mdb_env_info.
func (env *Env) Info() (*EnvInfo, error) {
	var _info C.MDB_envinfo
	ret := C.mdb_env_info(env._env, &_info)
	if ret != success {
		return nil, errno(ret)
	}
	info := EnvInfo{
		MapSize:    int64(_info.me_mapsize),
		LastPNO:    int64(_info.me_last_pgno),
		LastTxnID:  int64(_info.me_last_txnid),
		MaxReaders: uint(_info.me_maxreaders),
		NumReaders: uint(_info.me_numreaders),
	}
	return &info, nil
}

// Sync flushes buffers to disk.
//
// See mdb_env_sync.
func (env *Env) Sync(force bool) error {
	ret := C.mdb_env_sync(env._env, cbool(force))
	return errno(ret)
}

// SetFlags enables/disables flags in the environment.
//
// See mdb_env_set_flags.
func (env *Env) SetFlags(flags uint, onoff int) error {
	ret := C.mdb_env_set_flags(env._env, C.uint(flags), C.int(onoff))
	return errno(ret)
}

// Flags returns the flags set in the environment.
//
// See mdb_env_get_flags.
func (env *Env) Flags() (uint, error) {
	var _flags C.uint
	ret := C.mdb_env_get_flags(env._env, &_flags)
	if ret != success {
		return 0, errno(ret)
	}
	return uint(_flags), nil
}

// Path returns the path argument passed to Open.  Path returns an error if the
// Env was not opened previously.  Calling Path on a closed Env has undefined
// results.
//
// See mdb_env_path.
func (env *Env) Path() (string, error) {
	var path string
	cpath := C.CString(path)
	defer C.free(unsafe.Pointer(cpath))
	ret := C.mdb_env_get_path(env._env, &cpath)
	if ret != success {
		return "", errno(ret)
	}
	if cpath == nil {
		return "", fmt.Errorf("not open")
	}
	return C.GoString(cpath), nil
}

// SetMapSize sets the size of the environment memory map.
//
// See mdb_env_set_map_size.
func (env *Env) SetMapSize(size int64) error {
	if size < 0 {
		return fmt.Errorf("negative size")
	}
	ret := C.mdb_env_set_mapsize(env._env, C.size_t(size))
	return errno(ret)
}

// SetMaxReaders sets the maximum number of reader slots in the environment.
//
// See mdb_env_set_max_readers.
func (env *Env) SetMaxReaders(size int) error {
	if size < 0 {
		return fmt.Errorf("negative size")
	}
	ret := C.mdb_env_set_maxreaders(env._env, C.uint(size))
	return errno(ret)
}

// SetMaxDBs sets the maximum number of named databases for the environment.
//
// See mdb_env_set_maxdbs.
func (env *Env) SetMaxDBs(size int) error {
	if size < 0 {
		return fmt.Errorf("negative size")
	}
	ret := C.mdb_env_set_maxdbs(env._env, C.MDB_dbi(size))
	return errno(ret)
}

// BeginTxn is a low-level (potentially dangerous) method to initialize a new
// transaction on env.  BeginTxn does not attempt to serialize operations on
// write transactions to the same OS thread and its use for write transactions
// can cause undefined results without care.
//
// Instead of BeginTxn users should call the View, Update, RunTxn methods.
//
// See mdb_txn_begin.
func (env *Env) BeginTxn(parent *Txn, flags uint) (*Txn, error) {
	return beginTxn(env, parent, flags)
}

// Run creates a new Txn and calls fn with it as an argument.  Run commits the
// transaction if fn returns nil otherwise the transaction is aborted.
//
// Because Run terminates the transaction goroutines should not retain
// references to it after fn returns.  Writable transactions (without the
// Readonly flag) must not be used from any goroutines other than the one
// running fn.
//
// See mdb_txn_begin.
func (env *Env) RunTxn(flags uint, fn TxnOp) error {
	if isReadonly(flags) {
		return env.run(flags, fn)
	}
	return env.runUpdate(flags, fn)
}

// View creates a readonly transaction with a consistent view of the
// environment and passes it to fn.  View terminates its transaction after fn
// returns.  Any error encountered by View is returned.
//
// Any call to Commit, Abort, Reset or Renew on a Txn created by View will
// panic.
func (env *Env) View(fn TxnOp) error {
	return env.run(Readonly, fn)
}

// Update creates a writable transaction and passes it to fn.  Update commits
// the transaction if fn returns without error otherwise Update aborts the
// transaction and returns the error.
//
// The Txn passed to fn must not be used from multiple goroutines, even with
// synchronization.
//
// Any call to Commit, Abort, Reset or Renew on a Txn created by Update will
// panic.
func (env *Env) Update(fn TxnOp) error {
	return env.runUpdate(0, fn)
}

func (env *Env) run(flags uint, fn TxnOp) error {
	txn, err := env.BeginTxn(nil, flags)
	if err != nil {
		return err
	}
	txn.managed = true
	err = fn(txn)
	if err != nil {
		txn.abort()
		return err
	}
	return txn.commit()
}

func (env *Env) runUpdate(flags uint, fn TxnOp) error {
	errc := make(chan error)
	go func() {
		defer close(errc)
		runtime.LockOSThread()
		defer runtime.UnlockOSThread()
		err := env.run(flags, fn)
		if err != nil {
			errc <- err
		}
	}()
	return <-errc
}

// CloseDBI closes the database handle, db.  Normally calling CloseDBI
// explicitly is not necessary.
//
// It is the caller's responsibility to serialize calls to CloseDBI.
//
// See mdb_dbi_close.
func (env *Env) CloseDBI(db DBI) {
	C.mdb_dbi_close(env._env, C.MDB_dbi(db))
}

func isReadonly(flags uint) bool {
	return flags&Readonly != 0
}
<commit_msg>comment about goroutine isolation for txn<commit_after>package lmdb

/*
#include <stdlib.h>
#include <stdio.h>
#include "lmdb.h"
*/
import "C"

import (
	"errors"
	"fmt"
	"os"
	"runtime"
	"unsafe"
)

// success is a value returned from the LMDB API to indicate a successful call.
// The functions in this API this behavior and its use is not required.
const success = C.MDB_SUCCESS

const (
	// Flags for Env.Open.
	//
	// See mdb_env_open
	FixedMap    = C.MDB_FIXEDMAP   // Danger zone. Map memory at a fixed address.
	NoSubdir    = C.MDB_NOSUBDIR   // Argument to Open is a file, not a directory.
	Readonly    = C.MDB_RDONLY     // Used in several functions to denote an object as readonly.
	WriteMap    = C.MDB_WRITEMAP   // Use a writable memory map.
	NoMetaSync  = C.MDB_NOMETASYNC // Don't fsync metapage after commit.
	NoSync      = C.MDB_NOSYNC     // Don't fsync after commit.
	MapAsync    = C.MDB_MAPASYNC   // Flush asynchronously when using the WriteMap flag.
	NoTLS       = C.MDB_NOTLS      // Danger zone. Tie reader locktable slots to Txn objects instead of threads.
	NoLock      = C.MDB_NOLOCK     // Danger zone. LMDB does not use any locks. All transactions must serialize.
	NoReadahead = C.MDB_NORDAHEAD  // Disable readahead. Requires OS support.
	NoMemInit   = C.MDB_NOMEMINIT  // Disable LMDB memory initialization.
)

// DBI is a handle for a database in an Env.
//
// See MDB_dbi
type DBI C.MDB_dbi

// Env is opaque structure for a database environment.  A DB environment
// supports multiple databases, all residing in the same shared-memory map.
//
// See MDB_env.
type Env struct {
	_env *C.MDB_env
}

// NewEnv allocates and initializes a new Env.
//
// See mdb_env_create.
func NewEnv() (*Env, error) {
	var _env *C.MDB_env
	ret := C.mdb_env_create(&_env)
	if ret != success {
		return nil, errno(ret)
	}
	return &Env{_env}, nil
}

// Open an environment handle. If this function fails Close() must be called to
// discard the Env handle.  Open passes flags|NoTLS to mdb_env_open.
//
// See mdb_env_open.
func (env *Env) Open(path string, flags uint, mode os.FileMode) error {
	cpath := C.CString(path)
	defer C.free(unsafe.Pointer(cpath))
	ret := C.mdb_env_open(env._env, cpath, C.uint(NoTLS|flags), C.mdb_mode_t(mode))
	return errno(ret)
}

// Close shuts down the environment and releases the memory map.
//
// See mdb_env_close.
func (env *Env) Close() error {
	if env._env == nil {
		return errors.New("Environment already closed")
	}
	C.mdb_env_close(env._env)
	env._env = nil
	return nil
}

// Copy copies the data in env to an environment at path.
//
// See mdb_env_copy.
func (env *Env) Copy(path string) error {
	cpath := C.CString(path)
	defer C.free(unsafe.Pointer(cpath))
	ret := C.mdb_env_copy(env._env, cpath)
	return errno(ret)
}

// CopyFlag copies the data in env to an environment at path created with flags.
//
// See mdb_env_copy2.
func (env *Env) CopyFlag(path string, flags uint) error {
	cpath := C.CString(path)
	defer C.free(unsafe.Pointer(cpath))
	ret := C.mdb_env_copy2(env._env, cpath, C.uint(flags))
	return errno(ret)
}

// Statistics for a database in the environment
//
// See MDB_stat.
type Stat struct {
	PSize         uint   // Size of a database page. This is currently the same for all databases.
	Depth         uint   // Depth (height) of the B-tree
	BranchPages   uint64 // Number of internal (non-leaf) pages
	LeafPages     uint64 // Number of leaf pages
	OverflowPages uint64 // Number of overflow pages
	Entries       uint64 // Number of data items
}

// Stat returns statistics about the environment.
//
// See mdb_env_stat.
func (env *Env) Stat() (*Stat, error) {
	var _stat C.MDB_stat
	ret := C.mdb_env_stat(env._env, &_stat)
	if ret != success {
		return nil, errno(ret)
	}
	stat := Stat{PSize: uint(_stat.ms_psize),
		Depth:         uint(_stat.ms_depth),
		BranchPages:   uint64(_stat.ms_branch_pages),
		LeafPages:     uint64(_stat.ms_leaf_pages),
		OverflowPages: uint64(_stat.ms_overflow_pages),
		Entries:       uint64(_stat.ms_entries)}
	return &stat, nil
}

// Information about the environment.
//
// See MDB_envinfo.
type EnvInfo struct {
	MapSize    int64 // Size of the data memory map
	LastPNO    int64 // ID of the last used page
	LastTxnID  int64 // ID of the last committed transaction
	MaxReaders uint  // maximum number of threads for the environment
	NumReaders uint  // maximum number of threads used in the environment
}

// Info returns information about the environment.
//
// See mdb_env_info.
func (env *Env) Info() (*EnvInfo, error) {
	var _info C.MDB_envinfo
	ret := C.mdb_env_info(env._env, &_info)
	if ret != success {
		return nil, errno(ret)
	}
	info := EnvInfo{
		MapSize:    int64(_info.me_mapsize),
		LastPNO:    int64(_info.me_last_pgno),
		LastTxnID:  int64(_info.me_last_txnid),
		MaxReaders: uint(_info.me_maxreaders),
		NumReaders: uint(_info.me_numreaders),
	}
	return &info, nil
}

// Sync flushes buffers to disk.
//
// See mdb_env_sync.
func (env *Env) Sync(force bool) error {
	ret := C.mdb_env_sync(env._env, cbool(force))
	return errno(ret)
}

// SetFlags enables/disables flags in the environment.
//
// See mdb_env_set_flags.
func (env *Env) SetFlags(flags uint, onoff int) error {
	ret := C.mdb_env_set_flags(env._env, C.uint(flags), C.int(onoff))
	return errno(ret)
}

// Flags returns the flags set in the environment.
//
// See mdb_env_get_flags.
func (env *Env) Flags() (uint, error) {
	var _flags C.uint
	ret := C.mdb_env_get_flags(env._env, &_flags)
	if ret != success {
		return 0, errno(ret)
	}
	return uint(_flags), nil
}

// Path returns the path argument passed to Open.  Path returns an error if the
// Env was not opened previously.  Calling Path on a closed Env has undefined
// results.
//
// See mdb_env_path.
func (env *Env) Path() (string, error) {
	var path string
	cpath := C.CString(path)
	defer C.free(unsafe.Pointer(cpath))
	ret := C.mdb_env_get_path(env._env, &cpath)
	if ret != success {
		return "", errno(ret)
	}
	if cpath == nil {
		return "", fmt.Errorf("not open")
	}
	return C.GoString(cpath), nil
}

// SetMapSize sets the size of the environment memory map.
//
// See mdb_env_set_map_size.
func (env *Env) SetMapSize(size int64) error {
	if size < 0 {
		return fmt.Errorf("negative size")
	}
	ret := C.mdb_env_set_mapsize(env._env, C.size_t(size))
	return errno(ret)
}

// SetMaxReaders sets the maximum number of reader slots in the environment.
//
// See mdb_env_set_max_readers.
func (env *Env) SetMaxReaders(size int) error {
	if size < 0 {
		return fmt.Errorf("negative size")
	}
	ret := C.mdb_env_set_maxreaders(env._env, C.uint(size))
	return errno(ret)
}

// SetMaxDBs sets the maximum number of named databases for the environment.
//
// See mdb_env_set_maxdbs.
func (env *Env) SetMaxDBs(size int) error {
	if size < 0 {
		return fmt.Errorf("negative size")
	}
	ret := C.mdb_env_set_maxdbs(env._env, C.MDB_dbi(size))
	return errno(ret)
}

// BeginTxn is a low-level (potentially dangerous) method to initialize a new
// transaction on env.  BeginTxn does not attempt to serialize operations on
// write transactions to the same OS thread and its use for write transactions
// can cause undefined results without care.
//
// Instead of BeginTxn users should call the View, Update, RunTxn methods.
//
// See mdb_txn_begin.
func (env *Env) BeginTxn(parent *Txn, flags uint) (*Txn, error) {
	return beginTxn(env, parent, flags)
}

// Run creates a new Txn and calls fn with it as an argument.  Run commits the
// transaction if fn returns nil otherwise the transaction is aborted.
//
// Because Run terminates the transaction goroutines should not retain
// references to it after fn returns.  Writable transactions (without the
// Readonly flag) must not be used from any goroutines other than the one
// running fn.
//
// See mdb_txn_begin.
func (env *Env) RunTxn(flags uint, fn TxnOp) error {
	if isReadonly(flags) {
		return env.run(flags, fn)
	}
	return env.runUpdate(flags, fn)
}

// View creates a readonly transaction with a consistent view of the
// environment and passes it to fn.  View terminates its transaction after fn
// returns.  Any error encountered by View is returned.
//
// Any call to Commit, Abort, Reset or Renew on a Txn created by View will
// panic.
func (env *Env) View(fn TxnOp) error {
	return env.run(Readonly, fn)
}

// Update creates a writable transaction and passes it to fn.  Update commits
// the transaction if fn returns without error otherwise Update aborts the
// transaction and returns the error.
//
// The Txn passed to fn must not be used from multiple goroutines, even with
// synchronization.
//
// Any call to Commit, Abort, Reset or Renew on a Txn created by Update will
// panic.
func (env *Env) Update(fn TxnOp) error {
	return env.runUpdate(0, fn)
}

func (env *Env) run(flags uint, fn TxnOp) error {
	txn, err := env.BeginTxn(nil, flags)
	if err != nil {
		return err
	}
	txn.managed = true
	err = fn(txn)
	if err != nil {
		txn.abort()
		return err
	}
	return txn.commit()
}

// runUpdate calls fn with a managed transaction on another goroutine.  The
// transaction runs inside a goroutine so thread locking does not interact
// negatively with other objects which would like to lock their thread.
func (env *Env) runUpdate(flags uint, fn TxnOp) error {
	errc := make(chan error)
	go func() {
		defer close(errc)
		runtime.LockOSThread()
		defer runtime.UnlockOSThread()
		err := env.run(flags, fn)
		if err != nil {
			errc <- err
		}
	}()
	return <-errc
}

// CloseDBI closes the database handle, db.  Normally calling CloseDBI
// explicitly is not necessary.
//
// It is the caller's responsibility to serialize calls to CloseDBI.
//
// See mdb_dbi_close.
func (env *Env) CloseDBI(db DBI) {
	C.mdb_dbi_close(env._env, C.MDB_dbi(db))
}

func isReadonly(flags uint) bool {
	return flags&Readonly != 0
}
<|endoftext|>