<commit_before>package main

import (
	"archive/zip"
	"bytes"
	"errors"
	"flag"
	"fmt"
	"github.com/bmizerany/pat"
	"github.com/moovweb/gokogiri"
	"github.com/moovweb/gokogiri/css"
	"github.com/moovweb/gokogiri/xml"
	"html/template"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"runtime"
	"syscall"
	"time"
)

type Item struct {
	Id   string
	Href string
	Type string
}

type Epub struct {
	Zip *zip.Writer
	// TODO identifier / rights
	Title        string
	Subject      string
	Date         string
	Creator      string
	Contributors []string
	Items        []Item
}

const (
	Host           = "linuxfr.org"
	ContentType    = "application/epub+zip"
	XmlDeclaration = `<?xml version="1.0" encoding="utf-8"?>`
	Container      = XmlDeclaration + `
<container xmlns="urn:oasis:names:tc:opendocument:xmlns:container" version="1.0">
  <rootfiles>
    <rootfile full-path="EPUB/package.opf" media-type="application/oebps-package+xml"/>
  </rootfiles>
</container>`
)

// TODO embed CSS & images
// TODO cover
var PackageTemplate = template.Must(template.New("package").Parse(`
<package xmlns="http://www.idpf.org/2007/opf" unique-identifier="pub-identifier" xml:lang="fr" version="3.0">
	<metadata xmlns:dc="http://purl.org/dc/elements/1.1/">
		<dc:language id="pub-language">fr</dc:language>
		<dc:identifier id="pub-identifier">xxx</dc:identifier>
		<dc:date>{{.Date}}</dc:date>
		<meta property="dcterms:modified">{{.Date}}</meta>
		{{if .Title}}<dc:title id="pub-title">{{.Title}}</dc:title>{{end}}
		{{if .Creator}}<dc:creator id="pub-creator">{{.Creator}}</dc:creator>{{end}}
		{{range .Contributors}}<dc:contributor>{{.}}</dc:contributor>
		{{end}}
		<dc:rights>xxx</dc:rights>
	</metadata>
	<manifest>
		{{range .Items}}<item id="{{.Id}}" href="{{.Href}}" media-type="{{.Type}}"/>
		{{end}}
	</manifest>
	<spine>
		{{range .Items}}<itemref idref="{{.Id}}"/>
		{{end}}
	</spine>
</package>`))

func NewEpub(w io.Writer) (epub *Epub) {
	z := zip.NewWriter(w)
	epub = &Epub{Zip: z, Items: []Item{}}
	epub.AddMimetype()
	epub.AddFile("META-INF/container.xml", Container)
	return
}

func (epub *Epub) AddContent(article xml.Node) {
	xpath := css.Convert(".content", css.LOCAL)
	nodes, err := article.Search(xpath)
	if err != nil || len(nodes) == 0 {
		return
	}
	html := nodes[0].InnerHtml() // FIXME should be a complete HTML document
	filename := "content.html"
	epub.Items = append(epub.Items, Item{"item-content", filename, "application/xhtml+xml"})
	epub.AddFile("EPUB/" + filename, html)
}

func (epub *Epub) AddComments(article xml.Node) {
	list := article.NextSibling()
	xpath := css.Convert(".threads>li", css.LOCAL)
	threads, err := list.Search(xpath)
	if err != nil {
		return
	}
	for _, thread := range threads {
		html := thread.InnerHtml() // FIXME should be a complete HTML document
		id := thread.Attr("id")
		filename := id + ".html"
		epub.Items = append(epub.Items, Item{id, filename, "application/xhtml+xml"})
		epub.AddFile("EPUB/" + filename, html)
	}
}

func (epub *Epub) FindMeta(article xml.Node, selector string) string {
	xpath := css.Convert(selector, css.LOCAL)
	nodes, err := article.Search(xpath)
	if err != nil || len(nodes) == 0 {
		return ""
	}
	return nodes[0].Content()
}

func (epub *Epub) FindMetas(article xml.Node, selector string) []string {
	xpath := css.Convert(selector, css.LOCAL)
	nodes, err := article.Search(xpath)
	if err != nil {
		return nil
	}
	metas := make([]string, len(nodes))
	for i, node := range nodes {
		metas[i] = node.Content()
	}
	return metas
}

func (epub *Epub) FillMeta(article xml.Node) {
	epub.Title = epub.FindMeta(article, "header h1 a:last-child")
	epub.Subject = epub.FindMeta(article, "header h1 a.topic")
	meta := epub.FindMeta(article, "header time.updated")
	// FIXME ParseInLocation
	date, err := time.Parse("le 02/01/06 à 15:04", meta)
	if err != nil {
		date = time.Now()
	}
	epub.Date = date.Format(time.RFC3339)
	epub.Creator = epub.FindMeta(article, "header .meta a[rel=\"author\"]")
	epub.Contributors = epub.FindMetas(article, "header .meta .edited_by a")
}

func (epub *Epub) AddMimetype() (err error) {
	header := &zip.FileHeader{Name: "mimetype", Method: zip.Store}
	f, err := epub.Zip.CreateHeader(header)
	if err != nil {
		log.Println(err)
		return
	}

	_, err = f.Write([]byte(ContentType))
	if err != nil {
		log.Println(err)
		return
	}

	return
}

func (epub *Epub) AddFile(filename, content string) (err error) {
	f, err := epub.Zip.Create(filename)
	if err != nil {
		log.Println(err)
		return
	}

	_, err = f.Write([]byte(content))
	if err != nil {
		log.Println(err)
		return
	}

	return
}

func (epub *Epub) Close() {
	var opf bytes.Buffer
	err := PackageTemplate.Execute(&opf, epub)
	if err != nil {
		log.Println(err)
		return
	}

	epub.AddFile("EPUB/package.opf", XmlDeclaration+opf.String())
	err = epub.Zip.Close()
	if err != nil {
		log.Println(err)
	}
}

func fetchArticle(uri string) (article xml.Node, err error) {
	resp, err := http.Get(uri)
	if err != nil {
		return
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Printf("Error on ioutil.ReadAll for %s: %s\n", uri, err)
		return
	}

	doc, err := gokogiri.ParseHtml(body)
	if err != nil {
		log.Printf("Gokogiri error: %s\n", err)
		return
	}

	xpath := css.Convert("#contents article", css.LOCAL)
	articles, err := doc.Root().Search(xpath)
	if err != nil {
		log.Printf("Gokogiri error: %s\n", err)
		return
	}

	if len(articles) == 0 {
		err = errors.New("No article found in the page")
		return
	}

	article = articles[0]
	return
}

// Create an epub for a news
func News(w http.ResponseWriter, r *http.Request) {
	w.Header().Add("Content-Type", ContentType)

	slug := r.URL.Query().Get(":slug")
	uri := fmt.Sprintf("http://%s/news/%s", Host, slug)
	article, err := fetchArticle(uri)
	if err != nil {
		http.NotFound(w, r)
		return
	}

	// 	buffer := make([]byte, 4096)
	// 	buffer, _ = doc.Root().ToHtml(xml.DefaultEncodingBytes, buffer)

	epub := NewEpub(w)
	epub.FillMeta(article)
	epub.AddContent(article)
	epub.AddComments(article)
	epub.Close()
}

// Returns 200 OK if the server is running (for monitoring)
func Status(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "OK")
}

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	// Parse the command-line
	var addr string
	var logs string
	flag.StringVar(&addr, "a", "127.0.0.1:8000", "Bind to this address:port")
	flag.StringVar(&logs, "l", "-", "Use this file for logs")
	flag.Parse()

	// Logging
	if logs != "-" {
		f, err := os.OpenFile(logs, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0644)
		if err != nil {
			log.Fatal("OpenFile: ", err)
		}
		syscall.Dup2(int(f.Fd()), int(os.Stdout.Fd()))
		syscall.Dup2(int(f.Fd()), int(os.Stderr.Fd()))
	}

	// Routing
	m := pat.New()
	m.Get("/status", http.HandlerFunc(Status))
	m.Get("/news/:slug.epub", http.HandlerFunc(News))
	// TODO accept other content types
	http.Handle("/", m)

	// Start the HTTP server
	log.Printf("Listening on http://%s/\n", addr)
	err := http.ListenAndServe(addr, nil)
	if err != nil {
		log.Fatal("ListenAndServe: ", err)
	}
}
<commit_msg>Add a nav item<commit_after>package main

import (
	"archive/zip"
	"bytes"
	"errors"
	"flag"
	"fmt"
	"github.com/bmizerany/pat"
	"github.com/moovweb/gokogiri"
	"github.com/moovweb/gokogiri/css"
	"github.com/moovweb/gokogiri/xml"
	"html/template"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"runtime"
	"syscall"
	"time"
)

type Item struct {
	Id   string
	Href string
	Type string
}

type Epub struct {
	Zip *zip.Writer
	// TODO identifier / rights
	Title        string
	Subject      string
	Date         string
	Creator      string
	Contributors []string
	Items        []Item
}

const (
	Host           = "linuxfr.org"
	ContentType    = "application/epub+zip"
	XmlDeclaration = `<?xml version="1.0" encoding="utf-8"?>`
	Container      = XmlDeclaration + `
<container xmlns="urn:oasis:names:tc:opendocument:xmlns:container" version="1.0">
  <rootfiles>
    <rootfile full-path="EPUB/package.opf" media-type="application/oebps-package+xml"/>
  </rootfiles>
</container>`
	Nav = XmlDeclaration + `

    <title>LinuxFr.org</title>
    <meta charset="utf-8" />

Sommaire

Aller au contenu

`
)

// TODO embed CSS & images
// TODO cover
var PackageTemplate = template.Must(template.New("package").Parse(`
<package xmlns="http://www.idpf.org/2007/opf" unique-identifier="pub-identifier" xml:lang="fr" version="3.0">
	<metadata xmlns:dc="http://purl.org/dc/elements/1.1/">
		<dc:language id="pub-language">fr</dc:language>
		<dc:identifier id="pub-identifier">xxx</dc:identifier>
		<dc:date>{{.Date}}</dc:date>
		<meta property="dcterms:modified">{{.Date}}</meta>
		{{if .Title}}<dc:title id="pub-title">{{.Title}}</dc:title>{{end}}
		{{if .Creator}}<dc:creator id="pub-creator">{{.Creator}}</dc:creator>{{end}}
		{{range .Contributors}}<dc:contributor>{{.}}</dc:contributor>
		{{end}}
		<dc:rights>xxx</dc:rights>
	</metadata>
	<manifest>
		<item id="nav" href="nav.html" media-type="application/xhtml+xml" properties="nav"/>
		{{range .Items}}<item id="{{.Id}}" href="{{.Href}}" media-type="{{.Type}}"/>
		{{end}}
	</manifest>
	<spine>
		{{range .Items}}<itemref idref="{{.Id}}"/>
		{{end}}
	</spine>
</package>`))

func NewEpub(w io.Writer) (epub *Epub) {
	z := zip.NewWriter(w)
	epub = &Epub{Zip: z, Items: []Item{}}
	epub.AddMimetype()
	epub.AddFile("META-INF/container.xml", Container)
	epub.AddFile("EPUB/nav.html", Nav)
	return
}

func (epub *Epub) AddContent(article xml.Node) {
	xpath := css.Convert(".content", css.LOCAL)
	nodes, err := article.Search(xpath)
	if err != nil || len(nodes) == 0 {
		return
	}
	html := nodes[0].InnerHtml() // FIXME should be a complete HTML document
	filename := "content.html"
	epub.Items = append(epub.Items, Item{"item-content", filename, "application/xhtml+xml"})
	epub.AddFile("EPUB/"+filename, html)
}

func (epub *Epub) AddComments(article xml.Node) {
	list := article.NextSibling()
	xpath := css.Convert(".threads>li", css.LOCAL)
	threads, err := list.Search(xpath)
	if err != nil {
		return
	}
	for _, thread := range threads {
		html := thread.InnerHtml() // FIXME should be a complete HTML document
		id := thread.Attr("id")
		filename := id + ".html"
		epub.Items = append(epub.Items, Item{id, filename, "application/xhtml+xml"})
		epub.AddFile("EPUB/"+filename, html)
	}
}

func (epub *Epub) FindMeta(article xml.Node, selector string) string {
	xpath := css.Convert(selector, css.LOCAL)
	nodes, err := article.Search(xpath)
	if err != nil || len(nodes) == 0 {
		return ""
	}
	return nodes[0].Content()
}

func (epub *Epub) FindMetas(article xml.Node, selector string) []string {
	xpath := css.Convert(selector, css.LOCAL)
	nodes, err := article.Search(xpath)
	if err != nil {
		return nil
	}
	metas := make([]string, len(nodes))
	for i, node := range nodes {
		metas[i] = node.Content()
	}
	return metas
}

func (epub *Epub) FillMeta(article xml.Node) {
	epub.Title = epub.FindMeta(article, "header h1 a:last-child")
	epub.Subject = epub.FindMeta(article, "header h1 a.topic")
	meta := epub.FindMeta(article, "header time.updated")
	// FIXME ParseInLocation
	date, err := time.Parse("le 02/01/06 à 15:04", meta)
	if err != nil {
		date = time.Now()
	}
	epub.Date = date.Format(time.RFC3339)
	epub.Creator = epub.FindMeta(article, "header .meta a[rel=\"author\"]")
	epub.Contributors = epub.FindMetas(article, "header .meta .edited_by a")
}

func (epub *Epub) AddMimetype() (err error) {
	header := &zip.FileHeader{Name: "mimetype", Method: zip.Store}
	f, err := epub.Zip.CreateHeader(header)
	if err != nil {
		log.Println(err)
		return
	}

	_, err = f.Write([]byte(ContentType))
	if err != nil {
		log.Println(err)
		return
	}

	return
}

func (epub *Epub) AddFile(filename, content string) (err error) {
	f, err := epub.Zip.Create(filename)
	if err != nil {
		log.Println(err)
		return
	}

	_, err = f.Write([]byte(content))
	if err != nil {
		log.Println(err)
		return
	}

	return
}

func (epub *Epub) Close() {
	var opf bytes.Buffer
	err := PackageTemplate.Execute(&opf, epub)
	if err != nil {
		log.Println(err)
		return
	}

	epub.AddFile("EPUB/package.opf", XmlDeclaration+opf.String())
	err = epub.Zip.Close()
	if err != nil {
		log.Println(err)
	}
}

func fetchArticle(uri string) (article xml.Node, err error) {
	resp, err := http.Get(uri)
	if err != nil {
		return
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Printf("Error on ioutil.ReadAll for %s: %s\n", uri, err)
		return
	}

	doc, err := gokogiri.ParseHtml(body)
	if err != nil {
		log.Printf("Gokogiri error: %s\n", err)
		return
	}

	xpath := css.Convert("#contents article", css.LOCAL)
	articles, err := doc.Root().Search(xpath)
	if err != nil {
		log.Printf("Gokogiri error: %s\n", err)
		return
	}

	if len(articles) == 0 {
		err = errors.New("No article found in the page")
		return
	}

	article = articles[0]
	return
}

// Create an epub for a news
func News(w http.ResponseWriter, r *http.Request) {
	w.Header().Add("Content-Type", ContentType)

	slug := r.URL.Query().Get(":slug")
	uri := fmt.Sprintf("http://%s/news/%s", Host, slug)
	article, err := fetchArticle(uri)
	if err != nil {
		http.NotFound(w, r)
		return
	}

	// 	buffer := make([]byte, 4096)
	// 	buffer, _ = doc.Root().ToHtml(xml.DefaultEncodingBytes, buffer)

	epub := NewEpub(w)
	epub.FillMeta(article)
	epub.AddContent(article)
	epub.AddComments(article)
	epub.Close()
}

// Returns 200 OK if the server is running (for monitoring)
func Status(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "OK")
}

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	// Parse the command-line
	var addr string
	var logs string
	flag.StringVar(&addr, "a", "127.0.0.1:8000", "Bind to this address:port")
	flag.StringVar(&logs, "l", "-", "Use this file for logs")
	flag.Parse()

	// Logging
	if logs != "-" {
		f, err := os.OpenFile(logs, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0644)
		if err != nil {
			log.Fatal("OpenFile: ", err)
		}
		syscall.Dup2(int(f.Fd()), int(os.Stdout.Fd()))
		syscall.Dup2(int(f.Fd()), int(os.Stderr.Fd()))
	}

	// Routing
	m := pat.New()
	m.Get("/status", http.HandlerFunc(Status))
	m.Get("/news/:slug.epub", http.HandlerFunc(News))
	// TODO accept other content types
	http.Handle("/", m)

	// Start the HTTP server
	log.Printf("Listening on http://%s/\n", addr)
	err := http.ListenAndServe(addr, nil)
	if err != nil {
		log.Fatal("ListenAndServe: ", err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	log "github.com/Sirupsen/logrus"

	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
)

//[
//	{
//	"geometry" : {"x" : -118.15, "y" : 33.80},
//	"attributes" : {
//					"OWNER" : "Joe Smith",
//					"VALUE" : 94820.37,
//					"APPROVED" : true,
//					"LASTUPDATE" : 1227663551096
//					}
//	},
//	{
//	"geometry" : { "x" : -118.37, "y" : 34.086 },
//	"attributes" : {
//					"OWNER" : "John Doe",
//					"VALUE" : 17325.90,
//					"APPROVED" : false,
//					"LASTUPDATE" : 1227628579430
//					}
//					}
//]

type coordinates struct {
	X float64 `json:"x"`
	Y float64 `json:"y"`
}

type PlacePayload struct {
	Parkid string  `json:"parkid"` // hostemail+lat+lon
	Lat    float64 `json:"lat"`
	Lng    float64 `json:"lng"`
	Host   string  `json:"host_name"` // host(owner)
	Spaces int     `json:"spaces"`
}

type EsriFeatureNode struct {
	Geometry   coordinates  `json:"geometry"`
	Attributes PlacePayload `json:"attributes"`
}

func (h *HTTPClientHandler) addEsriNode(place HostingPlace) error {
	c := h.http

	parkid := fmt.Sprint("%s%f%f", place.Host, place.Lat, place.Long)
	payload := &PlacePayload{
		Parkid: parkid,
		Lat:    place.Lat,
		Lng:    place.Long,
		Host:   place.Host,
		Spaces: place.Space,
	}

	bts, err := json.Marshal(payload)

	if err != nil {
		log.Error(err.Error())
	}

	req, err := http.NewRequest("POST", AppConfig.ESRIEndpoint, bytes.NewBuffer(bts))
	req.Header.Set("Content-Type", "application/json")
	_, err = c.HTTPClient.Do(req)

	return err
}
<commit_msg>esri api is really terrible<commit_after>package main

import (
	log "github.com/Sirupsen/logrus"

	"bufio"
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
	//	"strconv"
)

//
//{
//"geometry": {
//	"x": 2,
//	"y": 51
//	},
//	"attributes": {
//			"parkid": "karolis@rusenas2.com0 51",
//			"lat": 4,
//			"lng": 51,
//			"host_name": "karolis@rusenas2.com",
//			"spaces": 3
//			}
//}
//

type Coordinates struct {
	X float64 `json:"x"`
	Y float64 `json:"y"`
}

type PlacePayload struct {
	Parkid string  `json:"parkid"` // hostemail+lat+lon
	Lat    float64 `json:"lat"`
	Lng    float64 `json:"lng"`
	Host   string  `json:"host_name"` // host(owner)
	Spaces int     `json:"spaces"`
}

type EsriFeatureNode struct {
	Geometry   Coordinates  `json:"geometry"`
	Attributes PlacePayload `json:"attributes"`
}

func (h *HTTPClientHandler) addEsriNode(place HostingPlace) (*http.Response, error) {
	c := h.http

	parkid := fmt.Sprintf("%s%d%d", place.Host, int(place.Lat), int(place.Long))

	payload := PlacePayload{
		Parkid: parkid,
		Lat:    place.Lat,
		Lng:    place.Long,
		Host:   place.Host,
		Spaces: place.Space,
	}

	coords := Coordinates{
		X: place.Lat,
		Y: place.Long,
	}

	finalPayload := EsriFeatureNode{Geometry: coords, Attributes: payload}

	bts, err := json.Marshal(finalPayload)

	if err != nil {
		log.Error(err.Error())
	}

	//	fullurl := fmt.Sprintf("%s%s", AppConfig.ESRIEndpoint, string(bts))

	log.WithFields(log.Fields{
		"body":     string(bts),
		"endpoint": AppConfig.ESRIEndpoint,
		//		"fullurl":  fullurl,
	}).Info("Adding esri node")

	//	req, err := http.NewRequest("POST", fullurl, nil)
	req, err := http.NewRequest("POST", AppConfig.ESRIEndpoint, bytes.NewBuffer(bts))
	req.Header.Set("Content-Type", "application/json")
	//	req.Header.Set("Content-Type", "application/html")
	resp, err := c.HTTPClient.Do(req)

	b := bufio.NewScanner(req.Body)

	bodyStr := b.Text()

	log.WithFields(log.Fields{
		"esriStatus": resp.StatusCode,
		"esriBody":   bodyStr,
	}).Info("Got response from esri")

	return resp, err
}
<|endoftext|>


<commit_before>// Copyright 2013 Jesse Allen. All rights reserved
// Released under the MIT license found in the LICENSE file.

package layouts

import (
	"bytes"
	"github.com/russross/blackfriday"
	"html/template"
	"net/http"
	"sync"
	"time"
)

type Layout struct {
	patterns     []string
	functions    template.FuncMap
	baseTemplate string
}

func New(functions template.FuncMap, baseTemplate string, patterns ...string) *Layout {
	l := new(Layout)
	l.Init(functions, baseTemplate, patterns...)
	return l
}

func (l *Layout) Init(functions template.FuncMap, baseTemplate string, patterns ...string) {
	l.functions = functions
	l.baseTemplate = baseTemplate
	l.patterns = patterns
}

type Action func(*http.Request) (map[string]interface{}, error)

func (a Action) Cache(ttl time.Duration) Action {
	var data map[string]interface{}
	lock := sync.RWMutex{}
	return func(r *http.Request) (map[string]interface{}, error) {
		lock.RLock()
		if data != nil {
			lock.RUnlock()
			return data, nil
		}
		lock.RUnlock()

		lock.Lock()
		defer lock.Unlock()
		var err error
		data, err = a(r)
		if data != nil {
			time.AfterFunc(ttl, func() {
				lock.Lock()
				data = nil
				lock.Unlock()
			})
		}
		return data, err
	}
}

type ErrorHandler func(http.ResponseWriter, *http.Request, error)

func (l *Layout) Act(respond Action, eh ErrorHandler, templates ...string) http.Handler {
	return http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) {
		t, err := l.load(templates...)
		if err != nil {
			eh(res, req, err)
			return
		}
		var data map[string]interface{}
		data, err = respond(req)
		if err != nil {
			eh(res, req, err)
			return
		}

		b := new(bytes.Buffer)
		if err = t.ExecuteTemplate(b, l.baseTemplate, data); err != nil {
			eh(res, req, err)
			return
		}
		if _, err = b.WriteTo(res); err != nil {
			eh(res, req, err)
		}
	})
}

// TODO: if performance becomes an issue, we can start caching the base templates, and cloning
func (l *Layout) load(patterns ...string) (*template.Template, error) {
	var err error
	// add some key helper functions to the templates
	b := template.New("base").Funcs(l.functions)
	for _, p := range append(l.patterns, patterns...) {
		_, err = b.ParseGlob(p)
		if err != nil {
			return nil, err
		}
	}
	return b, nil
}

func BasicFunctionMap() template.FuncMap {
	return template.FuncMap{
		"markdownCommon": func(raw string) template.HTML {
			return template.HTML(blackfriday.MarkdownCommon([]byte(raw)))
		},
		"markdownBasic": func(raw string) template.HTML {
			return template.HTML(blackfriday.MarkdownBasic([]byte(raw)))
		},
	}
}
<commit_msg>Cache templates for Act<commit_after>// Copyright 2013 Jesse Allen. All rights reserved
// Released under the MIT license found in the LICENSE file.

package layouts

import (
	"bytes"
	"github.com/russross/blackfriday"
	"html/template"
	"net/http"
	"sync"
	"time"
)

type Layout struct {
	patterns     []string
	functions    template.FuncMap
	baseTemplate string
}

func New(functions template.FuncMap, baseTemplate string, patterns ...string) *Layout {
	l := new(Layout)
	l.Init(functions, baseTemplate, patterns...)
	return l
}

func (l *Layout) Init(functions template.FuncMap, baseTemplate string, patterns ...string) {
	l.functions = functions
	l.baseTemplate = baseTemplate
	l.patterns = patterns
}

type Action func(*http.Request) (map[string]interface{}, error)

func (a Action) Cache(ttl time.Duration) Action {
	var data map[string]interface{}
	lock := sync.RWMutex{}
	return func(r *http.Request) (map[string]interface{}, error) {
		lock.RLock()
		if data != nil {
			lock.RUnlock()
			return data, nil
		}
		lock.RUnlock()

		lock.Lock()
		defer lock.Unlock()
		var err error
		data, err = a(r)
		if data != nil {
			time.AfterFunc(ttl, func() {
				lock.Lock()
				data = nil
				lock.Unlock()
			})
		}
		return data, err
	}
}

type ErrorHandler func(http.ResponseWriter, *http.Request, error)

func (l *Layout) Act(respond Action, eh ErrorHandler, templates ...string) http.Handler {
	// Load templates so that we can clone instead of loading every time
	permanentTemplates := template.Must(l.load(templates...))
	return http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) {
		t, err := permanentTemplates.Clone()
		if err != nil {
			eh(res, req, err)
			return
		}
		var data map[string]interface{}
		data, err = respond(req)
		if err != nil {
			eh(res, req, err)
			return
		}

		b := new(bytes.Buffer)
		if err = t.ExecuteTemplate(b, l.baseTemplate, data); err != nil {
			eh(res, req, err)
			return
		}
		if _, err = b.WriteTo(res); err != nil {
			eh(res, req, err)
		}
	})
}

// TODO: if performance becomes an issue, we can start caching the base templates, and cloning
func (l *Layout) load(patterns ...string) (*template.Template, error) {
	var err error
	// add some key helper functions to the templates
	b := template.New("base").Funcs(l.functions)
	for _, p := range append(l.patterns, patterns...) {
		_, err = b.ParseGlob(p)
		if err != nil {
			return nil, err
		}
	}
	return b, nil
}

func BasicFunctionMap() template.FuncMap {
	return template.FuncMap{
		"markdownCommon": func(raw string) template.HTML {
			return template.HTML(blackfriday.MarkdownCommon([]byte(raw)))
		},
		"markdownBasic": func(raw string) template.HTML {
			return template.HTML(blackfriday.MarkdownBasic([]byte(raw)))
		},
	}
}
<|endoftext|>


<commit_before>package conio

import "bufio"
import "bytes"
import "fmt"
import "os"
import "unicode"

import "github.com/mattn/go-runewidth"

var widthCache = make(map[rune]int)

func getCharWidth(n rune) int {
	width, ok := widthCache[n]
	if !ok {
		width = runewidth.RuneWidth(n)
		widthCache[n] = width
	}
	return width
	// if n > 0xFF {
	//	return 2;
	//}else{
	//	return 1;
	//}
}

var stdOut *bufio.Writer = bufio.NewWriter(os.Stdout)

func PutRep(ch rune, n int) {
	for i := 0; i < n; i++ {
		stdOut.WriteRune(ch)
	}
}

type ReadLineBuffer struct {
	Buffer    []rune
	Length    int
	Cursor    int
	Unicode   rune
	Keycode   uint16
	ViewStart int
	ViewWidth int
}

func (this *ReadLineBuffer) Insert(pos int, c []rune) bool {
	n := len(c)
	for this.Length+n >= len(this.Buffer) {
		tmp := make([]rune, len(this.Buffer)*2)
		copy(tmp, this.Buffer)
		this.Buffer = tmp
	}
	for i := this.Length; i >= pos; i-- {
		this.Buffer[i+n] = this.Buffer[i]
	}
	for i := 0; i < n; i++ {
		this.Buffer[pos+i] = c[i]
	}
	this.Length += n
	return true
}

func (this *ReadLineBuffer) InsertString(pos int, s string) int {
	list := make([]rune, 0)
	for _, r := range s {
		list = append(list, r)
	}
	if this.Insert(pos, list) {
		return len(list)
	} else {
		return -1
	}
}

func (this *ReadLineBuffer) Delete(pos int, n int) int {
	if this.Length < pos+n {
		return 0
	}
	delw := 0
	for i := pos; i < pos+n; i++ {
		delw += getCharWidth(this.Buffer[i])
	}
	for i := pos; i < this.Length-n; i++ {
		this.Buffer[i] = this.Buffer[i+n]
	}
	this.Length -= n
	return delw
}

func (this *ReadLineBuffer) ReplaceAndRepaint(pos int, str string) {
	n := this.Cursor - pos
	this.Delete(pos, n)
	this.InsertString(pos, str)
	if pos < this.ViewStart {
		PutRep('\b', this.GetWidthBetween(this.ViewStart, this.Cursor))
	} else {
		PutRep('\b', this.GetWidthBetween(pos, this.Cursor))
	}
	this.Cursor = pos
	for _, ch := range str {
		if this.Cursor >= this.ViewStart {
			PutRep(ch, 1)
		}
		this.Cursor++
	}
	this.Repaint(this.Cursor, 0)
}

func (this *ReadLineBuffer) GetWidthBetween(from int, to int) int {
	width := 0
	for i := from; i < to; i++ {
		width += getCharWidth(this.Buffer[i])
	}
	return width
}

func (this *ReadLineBuffer) Repaint(pos int, del int) {
	bs := 0
	vp := this.GetWidthBetween(this.ViewStart, pos)

	for i := pos; i < this.Length; i++ {
		w1 := getCharWidth(this.Buffer[i])
		vp += w1
		if vp >= this.ViewWidth {
			break
		}
		PutRep(this.Buffer[i], 1)
		bs += w1
	}
	PutRep(' ', del)
	PutRep('\b', bs+del)
}

func (this *ReadLineBuffer) RepaintAll(header string) {
	PutRep('\r', 1)
	for _, ch := range header {
		PutRep(ch, 1)
	}
	for i := this.ViewStart; i < this.Cursor; i++ {
		PutRep(this.Buffer[i], 1)
	}
	this.Repaint(this.Cursor, 0)
}

func (this ReadLineBuffer) String() string {
	var result bytes.Buffer
	for i := 0; i < this.Length; i++ {
		result.WriteRune(this.Buffer[i])
	}
	return result.String()
}

func (this *ReadLineBuffer) CurrentWordTop() (wordTop int) {
	wordTop = -1
	isQuoted := false
	for i := 0; i < this.Cursor; i++ {
		if this.Buffer[i] == '"' {
			isQuoted = !isQuoted
		}
		if unicode.IsSpace(this.Buffer[i]) && !isQuoted {
			wordTop = -1
		} else if wordTop < 0 {
			wordTop = i
		}
	}
	if wordTop < 0 {
		return this.Cursor
	} else {
		return wordTop
	}
}

func (this *ReadLineBuffer) CurrentWord() (string, int) {
	var buffer bytes.Buffer
	start := this.CurrentWordTop()
	for i := start; i < this.Cursor; i++ {
		if this.Buffer[i] != '"' {
			buffer.WriteRune(this.Buffer[i])
		}
	}
	return buffer.String(), start
}

type KeyFuncResult int

const (
	CONTINUE KeyFuncResult = iota
	ENTER    KeyFuncResult = iota
	ABORT    KeyFuncResult = iota
)

func KeyFuncPass(this *ReadLineBuffer) KeyFuncResult {
	return CONTINUE
}

func KeyFuncEnter(this *ReadLineBuffer) KeyFuncResult { // Ctrl-M
	return ENTER
}

func KeyFuncHead(this *ReadLineBuffer) KeyFuncResult { // Ctrl-A
	PutRep('\b', this.GetWidthBetween(this.ViewStart, this.Cursor))
	this.Cursor = 0
	this.ViewStart = 0
	this.Repaint(0, 1)
	return CONTINUE
}

func KeyFuncBackword(this *ReadLineBuffer) KeyFuncResult { // Ctrl-B
	if this.Cursor <= 0 {
		return CONTINUE
	}
	this.Cursor--
	if this.Cursor < this.ViewStart {
		this.ViewStart--
		this.Repaint(this.Cursor, 1)
	} else {
		PutRep('\b', getCharWidth(this.Buffer[this.Cursor]))
	}
	return CONTINUE
}

func KeyFuncTail(this *ReadLineBuffer) KeyFuncResult { // Ctrl-E
	allength := this.GetWidthBetween(this.ViewStart, this.Length)
	if allength < this.ViewWidth {
		for ; this.Cursor < this.Length; this.Cursor++ {
			PutRep(this.Buffer[this.Cursor], 1)
		}
	} else {
		PutRep('\a', 1)
		PutRep('\b', this.GetWidthBetween(this.ViewStart, this.Cursor))
		this.ViewStart = this.Length - 1
		w := getCharWidth(this.Buffer[this.ViewStart])
		for {
			if this.ViewStart <= 0 {
				break
			}
			w_ := w + getCharWidth(this.Buffer[this.ViewStart-1])
			if w_ >= this.ViewWidth {
				break
			}
			w = w_
			this.ViewStart--
		}
		for this.Cursor = this.ViewStart; this.Cursor < this.Length; this.Cursor++ {
			PutRep(this.Buffer[this.Cursor], 1)
		}
	}
	return CONTINUE
}

func KeyFuncForward(this *ReadLineBuffer) KeyFuncResult { // Ctrl-F
	if this.Cursor >= this.Length {
		return CONTINUE
	}
	w := this.GetWidthBetween(this.ViewStart, this.Cursor+1)
	if w < this.ViewWidth {
		// No Scroll
		PutRep(this.Buffer[this.Cursor], 1)
	} else {
		// Right Scroll
		PutRep('\b', this.GetWidthBetween(this.ViewStart, this.Cursor))
		if getCharWidth(this.Buffer[this.Cursor]) > getCharWidth(this.Buffer[this.ViewStart]) {
			this.ViewStart++
		}
		this.ViewStart++
		for i := this.ViewStart; i <= this.Cursor; i++ {
			PutRep(this.Buffer[i], 1)
		}
		PutRep(' ', 1)
		PutRep('\b', 1)
	}
	this.Cursor++
	return CONTINUE
}

func KeyFuncBackSpace(this *ReadLineBuffer) KeyFuncResult { // Backspace
	if this.Cursor > 0 {
		this.Cursor--
		delw := this.Delete(this.Cursor, 1)
		if this.Cursor >= this.ViewStart {
			PutRep('\b', delw)
		} else {
			this.ViewStart = this.Cursor
		}
		this.Repaint(this.Cursor, delw)
	}
	return CONTINUE
}

func KeyFuncDelete(this *ReadLineBuffer) KeyFuncResult { // Del
	delw := this.Delete(this.Cursor, 1)
	this.Repaint(this.Cursor, delw)
	return CONTINUE
}

func KeyFuncDeleteOrAbort(this *ReadLineBuffer) KeyFuncResult { // Ctrl-D
	if this.Length > 0 {
		return KeyFuncDelete(this)
	} else {
		return ABORT
	}
}

func KeyFuncInsertSelf(this *ReadLineBuffer) KeyFuncResult {
	ch := this.Unicode
	if ch < 0x20 || !this.Insert(this.Cursor, []rune{ch}) {
		return CONTINUE
	}
	w := this.GetWidthBetween(this.ViewStart, this.Cursor)
	w1 := getCharWidth(ch)
	if w+w1 >= this.ViewWidth {
		// scroll left
		PutRep('\b', w)
		if getCharWidth(this.Buffer[this.ViewStart]) < w1 {
			this.ViewStart++
		}
		this.ViewStart++
		for i := this.ViewStart; i <= this.Cursor; i++ {
			PutRep(this.Buffer[i], 1)
		}
		PutRep(' ', 1)
		PutRep('\b', 1)
	} else {
		this.Repaint(this.Cursor, -w1)
	}
	this.Cursor++
	return CONTINUE
}

func KeyFuncInsertReport(this *ReadLineBuffer) KeyFuncResult {
	L := this.InsertString(this.Cursor, fmt.Sprintf("[%X]", this.Unicode))
	if L >= 0 {
		this.Repaint(this.Cursor, -L)
		this.Cursor += L
	}
	return CONTINUE
}

func KeyFuncClearAfter(this *ReadLineBuffer) KeyFuncResult {
	w := this.GetWidthBetween(this.ViewStart, this.Cursor)
	i := this.Cursor
	bs := 0
	for i < this.Length && w < this.ViewWidth {
		w1 := getCharWidth(this.Buffer[i])
		PutRep(' ', w1)
		i++
		w += w1
		bs += w1
	}
	PutRep('\b', bs)
	this.Length = this.Cursor
	return CONTINUE
}

func KeyFuncClear(this *ReadLineBuffer) KeyFuncResult {
	KeyFuncClearAfter(this)
	width := this.GetWidthBetween(this.ViewStart, this.Cursor)
	PutRep('\b', width)
	PutRep(' ', width)
	PutRep('\b', width)
	this.Length = 0
	this.Cursor = 0
	this.ViewStart = 0
	return CONTINUE
}

var KeyMap = map[rune]func(*ReadLineBuffer) KeyFuncResult{
	'\r':         KeyFuncEnter,
	'\x01':       KeyFuncHead,
	'\x02':       KeyFuncBackword,
	'\x05':       KeyFuncTail,
	'\x06':       KeyFuncForward,
	'\b':         KeyFuncBackSpace,
	'\x04':       KeyFuncDeleteOrAbort,
	'\x7F':       KeyFuncDelete,
	('K' & 0x1F): KeyFuncClearAfter,
	'\x1B':       KeyFuncClear,
}

// KeyCode from
// http://msdn.microsoft.com/ja-jp/library/windows/desktop/dd375731(v=vs.85).aspx
const (
	K_LEFT  = 0x25
	K_RIGHT = 0x27
	K_DEL   = 0x2E
	K_HOME  = 0x24
	K_END   = 0x23
	K_CTRL  = 0x11
	K_SHIFT = 0x10
	K_UP    = 0x26
	K_DOWN  = 0x28
)

var ZeroMap = map[uint16]func(*ReadLineBuffer) KeyFuncResult{
	K_LEFT:  KeyFuncBackword,
	K_RIGHT: KeyFuncForward,
	K_DEL:   KeyFuncDelete,
	K_HOME:  KeyFuncHead,
	K_END:   KeyFuncTail,
	K_CTRL:  KeyFuncPass,
	K_SHIFT: KeyFuncPass,
}

func ReadLine() (string, KeyFuncResult) {
	var this ReadLineBuffer
	this.Buffer = make([]rune, 20)
	this.Length = 0
	this.Cursor = 0
	this.ViewStart = 0
	this.ViewWidth = 60
	for {
		stdOut.Flush()
		this.Unicode, this.Keycode = GetKey()
		var f func(*ReadLineBuffer) KeyFuncResult
		var ok bool
		if this.Unicode != 0 {
			f, ok = KeyMap[this.Unicode]
			if !ok {
				//f = KeyFuncInsertReport
				f = KeyFuncInsertSelf
			}
		} else {
			f, ok = ZeroMap[this.Keycode]
			if !ok {
				f = KeyFuncPass
			}
		}
		rc := f(&this)
		if rc != CONTINUE {
			stdOut.WriteRune('\n')
			stdOut.Flush()
			return this.String(), rc
		}
	}
}
<commit_msg>Cursor more hilight<commit_after>package conio

import "bufio"
import "bytes"
import "fmt"
import "os"
import "unicode"

import "github.com/mattn/go-runewidth"

var widthCache = make(map[rune]int)

func getCharWidth(n rune) int {
	width, ok := widthCache[n]
	if !ok {
		width = runewidth.RuneWidth(n)
		widthCache[n] = width
	}
	return width
	// if n > 0xFF {
	//	return 2;
	//}else{
	//	return 1;
	//}
}

var stdOut *bufio.Writer = bufio.NewWriter(os.Stdout)

func PutRep(ch rune, n int) {
	for i := 0; i < n; i++ {
		stdOut.WriteRune(ch)
	}
}

func Backspace(n int) {
	stdOut.Flush()
	x, y := GetLocate()
	Locate(x-n, y)
}

func shineCursor() {
	x, y := GetLocate()
	Locate(x, y)
}

type ReadLineBuffer struct {
	Buffer    []rune
	Length    int
	Cursor    int
	Unicode   rune
	Keycode   uint16
	ViewStart int
	ViewWidth int
}

func (this *ReadLineBuffer) Insert(pos int, c []rune) bool {
	n := len(c)
	for this.Length+n >= len(this.Buffer) {
		tmp := make([]rune, len(this.Buffer)*2)
		copy(tmp, this.Buffer)
		this.Buffer = tmp
	}
	for i := this.Length; i >= pos; i-- {
		this.Buffer[i+n] = this.Buffer[i]
	}
	for i := 0; i < n; i++ {
		this.Buffer[pos+i] = c[i]
	}
	this.Length += n
	return true
}

func (this *ReadLineBuffer) InsertString(pos int, s string) int {
	list := make([]rune, 0)
	for _, r := range s {
		list = append(list, r)
	}
	if this.Insert(pos, list) {
		return len(list)
	} else {
		return -1
	}
}

func (this *ReadLineBuffer) Delete(pos int, n int) int {
	if this.Length < pos+n {
		return 0
	}
	delw := 0
	for i := pos; i < pos+n; i++ {
		delw += getCharWidth(this.Buffer[i])
	}
	for i := pos; i < this.Length-n; i++ {
		this.Buffer[i] = this.Buffer[i+n]
	}
	this.Length -= n
	return delw
}

func (this *ReadLineBuffer) ReplaceAndRepaint(pos int, str string) {
	n := this.Cursor - pos
	this.Delete(pos, n)
	this.InsertString(pos, str)
	if pos < this.ViewStart {
		Backspace(this.GetWidthBetween(this.ViewStart, this.Cursor))
	} else {
		Backspace(this.GetWidthBetween(pos, this.Cursor))
	}
	this.Cursor = pos
	for _, ch := range str {
		if this.Cursor >= this.ViewStart {
			PutRep(ch, 1)
		}
		this.Cursor++
	}
	this.Repaint(this.Cursor, 0)
}

func (this *ReadLineBuffer) GetWidthBetween(from int, to int) int {
	width := 0
	for i := from; i < to; i++ {
		width += getCharWidth(this.Buffer[i])
	}
	return width
}

func (this *ReadLineBuffer) Repaint(pos int, del int) {
	bs := 0
	vp := this.GetWidthBetween(this.ViewStart, pos)

	for i := pos; i < this.Length; i++ {
		w1 := getCharWidth(this.Buffer[i])
		vp += w1
		if vp >= this.ViewWidth {
			break
		}
		PutRep(this.Buffer[i], 1)
		bs += w1
	}
	PutRep(' ', del)
	Backspace(bs + del)
}

func (this *ReadLineBuffer) RepaintAll(header string) {
	PutRep('\r', 1)
	for _, ch := range header {
		PutRep(ch, 1)
	}
	for i := this.ViewStart; i < this.Cursor; i++ {
		PutRep(this.Buffer[i], 1)
	}
	this.Repaint(this.Cursor, 0)
}

func (this ReadLineBuffer) String() string {
	var result bytes.Buffer
	for i := 0; i < this.Length; i++ {
		result.WriteRune(this.Buffer[i])
	}
	return result.String()
}

func (this *ReadLineBuffer) CurrentWordTop() (wordTop int) {
	wordTop = -1
	isQuoted := false
	for i := 0; i < this.Cursor; i++ {
		if this.Buffer[i] == '"' {
			isQuoted = !isQuoted
		}
		if unicode.IsSpace(this.Buffer[i]) && !isQuoted {
			wordTop = -1
		} else if wordTop < 0 {
			wordTop = i
		}
	}
	if wordTop < 0 {
		return this.Cursor
	} else {
		return wordTop
	}
}

func (this *ReadLineBuffer) CurrentWord() (string, int) {
	var buffer bytes.Buffer
	start := this.CurrentWordTop()
	for i := start; i < this.Cursor; i++ {
		if this.Buffer[i] != '"' {
			buffer.WriteRune(this.Buffer[i])
		}
	}
	return buffer.String(), start
}

type KeyFuncResult int

const (
	CONTINUE KeyFuncResult = iota
	ENTER    KeyFuncResult = iota
	ABORT    KeyFuncResult = iota
)

func KeyFuncPass(this *ReadLineBuffer) KeyFuncResult {
	return CONTINUE
}

func KeyFuncEnter(this *ReadLineBuffer) KeyFuncResult { // Ctrl-M
	return ENTER
}

func KeyFuncHead(this *ReadLineBuffer) KeyFuncResult { // Ctrl-A
	Backspace(this.GetWidthBetween(this.ViewStart, this.Cursor))
	this.Cursor = 0
	this.ViewStart = 0
	this.Repaint(0, 1)
	return CONTINUE
}

func KeyFuncBackword(this *ReadLineBuffer) KeyFuncResult { // Ctrl-B
	if this.Cursor <= 0 {
		return CONTINUE
	}
	this.Cursor--
	if this.Cursor < this.ViewStart {
		this.ViewStart--
		this.Repaint(this.Cursor, 1)
	} else {
		Backspace(getCharWidth(this.Buffer[this.Cursor]))
	}
	return CONTINUE
}

func KeyFuncTail(this *ReadLineBuffer) KeyFuncResult { // Ctrl-E
	allength := this.GetWidthBetween(this.ViewStart, this.Length)
	if allength < this.ViewWidth {
		for ; this.Cursor < this.Length; this.Cursor++ {
			PutRep(this.Buffer[this.Cursor], 1)
		}
	} else {
		PutRep('\a', 1)
		Backspace(this.GetWidthBetween(this.ViewStart, this.Cursor))
		this.ViewStart = this.Length - 1
		w := getCharWidth(this.Buffer[this.ViewStart])
		for {
			if this.ViewStart <= 0 {
				break
			}
			w_ := w + getCharWidth(this.Buffer[this.ViewStart-1])
			if w_ >= this.ViewWidth {
				break
			}
			w = w_
			this.ViewStart--
		}
		for this.Cursor = this.ViewStart; this.Cursor < this.Length; this.Cursor++ {
			PutRep(this.Buffer[this.Cursor], 1)
		}
	}
	return CONTINUE
}

func KeyFuncForward(this *ReadLineBuffer) KeyFuncResult { // Ctrl-F
	if this.Cursor >= this.Length {
		return CONTINUE
	}
	w := this.GetWidthBetween(this.ViewStart, this.Cursor+1)
	if w < this.ViewWidth {
		// No Scroll
		PutRep(this.Buffer[this.Cursor], 1)
	} else {
		// Right Scroll
		Backspace(this.GetWidthBetween(this.ViewStart, this.Cursor))
		if getCharWidth(this.Buffer[this.Cursor]) > getCharWidth(this.Buffer[this.ViewStart]) {
			this.ViewStart++
		}
		this.ViewStart++
		for i := this.ViewStart; i <= this.Cursor; i++ {
			PutRep(this.Buffer[i], 1)
		}
		PutRep(' ', 1)
		Backspace(1)
	}
	this.Cursor++
	return CONTINUE
}

func KeyFuncBackSpace(this *ReadLineBuffer) KeyFuncResult { // Backspace
	if this.Cursor > 0 {
		this.Cursor--
		delw := this.Delete(this.Cursor, 1)
		if this.Cursor >= this.ViewStart {
			Backspace(delw)
		} else {
			this.ViewStart = this.Cursor
		}
		this.Repaint(this.Cursor, delw)
	}
	return CONTINUE
}

func KeyFuncDelete(this *ReadLineBuffer) KeyFuncResult { // Del
	delw := this.Delete(this.Cursor, 1)
	this.Repaint(this.Cursor, delw)
	return CONTINUE
}

func KeyFuncDeleteOrAbort(this *ReadLineBuffer) KeyFuncResult { // Ctrl-D
	if this.Length > 0 {
		return KeyFuncDelete(this)
	} else {
		return ABORT
	}
}

func KeyFuncInsertSelf(this *ReadLineBuffer) KeyFuncResult {
	ch := this.Unicode
	if ch < 0x20 || !this.Insert(this.Cursor, []rune{ch}) {
		return CONTINUE
	}
	w := this.GetWidthBetween(this.ViewStart, this.Cursor)
	w1 := getCharWidth(ch)
	if w+w1 >= this.ViewWidth {
		// scroll left
		Backspace(w)
		if getCharWidth(this.Buffer[this.ViewStart]) < w1 {
			this.ViewStart++
		}
		this.ViewStart++
		for i := this.ViewStart; i <= this.Cursor; i++ {
			PutRep(this.Buffer[i], 1)
		}
		PutRep(' ', 1)
		Backspace(1)
	} else {
		this.Repaint(this.Cursor, -w1)
	}
	this.Cursor++
	return CONTINUE
}

func KeyFuncInsertReport(this *ReadLineBuffer) KeyFuncResult {
	L := this.InsertString(this.Cursor, fmt.Sprintf("[%X]", this.Unicode))
	if L >= 0 {
		this.Repaint(this.Cursor, -L)
		this.Cursor += L
	}
	return CONTINUE
}

func KeyFuncClearAfter(this *ReadLineBuffer) KeyFuncResult {
	w := this.GetWidthBetween(this.ViewStart, this.Cursor)
	i := this.Cursor
	bs := 0
	for i < this.Length && w < this.ViewWidth {
		w1 := getCharWidth(this.Buffer[i])
		PutRep(' ', w1)
		i++
		w += w1
		bs += w1
	}
	Backspace(bs)
	this.Length = this.Cursor
	return CONTINUE
}

func KeyFuncClear(this *ReadLineBuffer) KeyFuncResult {
	KeyFuncClearAfter(this)
	width := this.GetWidthBetween(this.ViewStart, this.Cursor)
	Backspace(width)
	PutRep(' ', width)
	Backspace(width)
	this.Length = 0
	this.Cursor = 0
	this.ViewStart = 0
	return CONTINUE
}

var KeyMap = map[rune]func(*ReadLineBuffer) KeyFuncResult{
	'\r':         KeyFuncEnter,
	'\x01':       KeyFuncHead,
	'\x02':       KeyFuncBackword,
	'\x05':       KeyFuncTail,
	'\x06':       KeyFuncForward,
	'\b':         KeyFuncBackSpace,
	'\x04':       KeyFuncDeleteOrAbort,
	'\x7F':       KeyFuncDelete,
	('K' & 0x1F): KeyFuncClearAfter,
	'\x1B':       KeyFuncClear,
}

// KeyCode from
// http://msdn.microsoft.com/ja-jp/library/windows/desktop/dd375731(v=vs.85).aspx
const (
	K_LEFT  = 0x25
	K_RIGHT = 0x27
	K_DEL   = 0x2E
	K_HOME  = 0x24
	K_END   = 0x23
	K_CTRL  = 0x11
	K_SHIFT = 0x10
	K_UP    = 0x26
	K_DOWN  = 0x28
)

var ZeroMap = map[uint16]func(*ReadLineBuffer) KeyFuncResult{
	K_LEFT:  KeyFuncBackword,
	K_RIGHT: KeyFuncForward,
	K_DEL:   KeyFuncDelete,
	K_HOME:  KeyFuncHead,
	K_END:   KeyFuncTail,
	K_CTRL:  KeyFuncPass,
	K_SHIFT: KeyFuncPass,
}

func ReadLine() (string, KeyFuncResult) {
	var this ReadLineBuffer
	this.Buffer = make([]rune, 20)
	this.Length = 0
	this.Cursor = 0
	this.ViewStart = 0
	this.ViewWidth = 60
	for {
		stdOut.Flush()
		shineCursor()
		this.Unicode, this.Keycode = GetKey()
		var f func(*ReadLineBuffer) KeyFuncResult
		var ok bool
		if this.Unicode != 0 {
			f, ok = KeyMap[this.Unicode]
			if !ok {
				//f = KeyFuncInsertReport
				f = KeyFuncInsertSelf
			}
		} else {
			f, ok = ZeroMap[this.Keycode]
			if !ok {
				f = KeyFuncPass
			}
		}
		rc := f(&this)
		if rc != CONTINUE {
			stdOut.WriteRune('\n')
			stdOut.Flush()
			return this.String(), rc
		}
	}
}
<|endoftext|>


<commit_before>package consensus

import (
	"errors"
	"math/big"
	"sort"

	"github.com/NebulousLabs/Sia/crypto"
)

// StateInfo contains basic information about the State.
type StateInfo struct {
	CurrentBlock BlockID
	Height       BlockHeight
	Target       Target
}

// blockAtHeight returns the block on the current path with the given height.
func (s *State) blockAtHeight(height BlockHeight) (b Block, exists bool) {
	bn, exists := s.blockMap[s.currentPath[height]]
	if !exists {
		return
	}
	b = bn.block
	return
}

// currentBlockNode returns the blockNode of the current block.
func (s *State) currentBlockNode() *blockNode {
	return s.blockMap[s.currentBlockID]
}

// currentBlockWeight returns the weight of the current block.
func (s *State) currentBlockWeight() *big.Rat {
	return s.currentBlockNode().target.Inverse()
}

// height returns the current height of the state.
func (s *State) height() BlockHeight {
	return s.blockMap[s.currentBlockID].height
}

// output returns the unspent SiacoinOutput associated with the given ID. If
// the output is not in the UTXO set, 'exists' will be false.
func (s *State) output(id SiacoinOutputID) (sco SiacoinOutput, exists bool) {
	sco, exists = s.siacoinOutputs[id]
	return
}

// sortedUscoSet returns all of the unspent siacoin outputs sorted
// according to the numerical value of their id.
func (s *State) sortedUscoSet() []SiacoinOutput {
	// Get all of the outputs in string form and sort the strings.
	unspentOutputs := make(crypto.HashSlice, len(s.siacoinOutputs))
	for outputID := range s.siacoinOutputs {
		unspentOutputs = append(unspentOutputs, crypto.Hash(outputID))
	}
	sort.Sort(unspentOutputs)

	// Get the outputs in order according to their sorted form.
	sortedOutputs := make([]SiacoinOutput, len(unspentOutputs))
	for i, outputID := range unspentOutputs {
		output, _ := s.output(SiacoinOutputID(outputID))
		sortedOutputs[i] = output
	}
	return sortedOutputs
}

// Sorted UsfoSet returns all of the unspent siafund outputs sorted according
// to the numerical value of their id.
func (s *State) sortedUsfoSet() []SiafundOutput {
	// Get all of the outputs in string form and sort the strings.
	outputIDs := make(crypto.HashSlice, len(s.siafundOutputs))
	for outputID := range s.siafundOutputs {
		outputIDs = append(outputIDs, crypto.Hash(outputID))
	}
	sort.Sort(outputIDs)

	// Get the outputs in order according to their sorted string form.
	sortedOutputs := make([]SiafundOutput, len(outputIDs))
	for i, outputID := range outputIDs {
		// Sanity check - the output should exist.
		output, exists := s.siafundOutputs[SiafundOutputID(outputID)]
		if DEBUG {
			if !exists {
				panic("output doesn't exist")
			}
		}

		sortedOutputs[i] = output
	}
	return sortedOutputs
}

// BlockAtHeight returns the block on the current path with the given height.
func (s *State) BlockAtHeight(height BlockHeight) (b Block, exists bool) {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.blockAtHeight(height)
}

// Block returns the block associated with the given id.
func (s *State) Block(id BlockID) (b Block, exists bool) {
	s.mu.RLock()
	defer s.mu.RUnlock()

	node, exists := s.blockMap[id]
	if !exists {
		return
	}
	b = node.block
	return
}

// BlockOutputDiffs returns the SiacoinOutputDiffs for a given block.
func (s *State) BlockOutputDiffs(id BlockID) (scods []SiacoinOutputDiff, err error) {
	node, exists := s.blockMap[id]
	if !exists {
		err = errors.New("requested an unknown block")
		return
	}
	if !node.diffsGenerated {
		err = errors.New("diffs have not been generated for the requested block")
		return
	}
	scods = node.siacoinOutputDiffs
	return
}

// BlocksSince returns a set of output diffs representing how the state
// has changed since block 'id'. OutputDiffsSince will flip the `new` value for
// diffs that got reversed.
func (s *State) BlocksSince(id BlockID) (removedBlocks, addedBlocks []BlockID, err error) {
	s.mu.RLock()
	defer s.mu.RUnlock()

	node, exists := s.blockMap[id]
	if !exists {
		err = errors.New("block is unknown")
		return
	}

	// Get all the IDs from the blockchain to the current path.
	path := s.backtrackToCurrentPath(node)
	for i := len(path) - 1; i > 0; i-- {
		removedBlocks = append(removedBlocks, path[i].block.ID())
	}

	// Get all the IDs going forward from the common parent.
	for height := path[0].height + 1; ; height++ {
		if _, exists := s.currentPath[height]; !exists {
			break
		}

		node := s.blockMap[s.currentPath[height]]
		addedBlocks = append(addedBlocks, node.block.ID())
	}

	return
}

// FileContract returns the file contract associated with the 'id'. If the
// contract does not exist, exists will be false.
func (s *State) FileContract(id FileContractID) (fc FileContract, exists bool) {
	s.mu.RLock()
	defer s.mu.RUnlock()

	fc, exists = s.fileContracts[id]
	return
}

// CurrentBlock returns the highest block on the tallest fork.
func (s *State) CurrentBlock() Block {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.currentBlockNode().block
}

// CurrentTarget returns the target of the next block that needs to be
// submitted to the state.
func (s *State) CurrentTarget() Target {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.currentBlockNode().target
}

// EarliestTimestamp returns the earliest timestamp that the next block can
// have in order for it to be considered valid.
func (s *State) EarliestTimestamp() Timestamp {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.currentBlockNode().earliestChildTimestamp()
}

// Height returns the height of the current blockchain (the longest fork).
func (s *State) Height() BlockHeight {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.height()
}

// HeightOfBlock returns the height of the block with the given ID.
func (s *State) HeightOfBlock(bid BlockID) (height BlockHeight, exists bool) {
	s.mu.RLock()
	defer s.mu.RUnlock()

	bn, exists := s.blockMap[bid]
	if !exists {
		return
	}
	height = bn.height
	return
}

// SiacoinOutput returns the siacoin output associated with the given ID.
func (s *State) SiacoinOutput(id SiacoinOutputID) (output SiacoinOutput, exists bool) {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.output(id)
}

// SiafundOutput returns the siafund output associated with the given ID.
func (s *State) SiafundOutput(id SiafundOutputID) (output SiafundOutput, exists bool) {
	s.mu.RLock()
	defer s.mu.RUnlock()
	output, exists = s.siafundOutputs[id]
	return
}

// SortedUtxoSet returns all of the unspent transaction outputs sorted
// according to the numerical value of their id.
func (s *State) SortedUtxoSet() []SiacoinOutput {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.sortedUscoSet()
}

// StorageProofSegment returns the segment to be used in the storage proof for
// a given file contract.
func (s *State) StorageProofSegment(fcid FileContractID) (index uint64, err error) {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.storageProofSegment(fcid)
}

// ValidTransaction checks that a transaction is valid within the context of
// the current consensus set.
func (s *State) ValidTransaction(t Transaction) (err error) {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.validTransaction(t)
}

// ValidTransactionComponents checks that a transaction follows basic rules,
// such as the storage proof rules, and it checks that all of the signatures
// are valid, but it does not check that all of the inputs, storage proofs, and
// terminations act on existing outputs and contracts. This function is
// primarily for the transaction pool, which has access to unconfirmed
// transactions. ValidTransactionComponents will not return an error simply
// because there are missing inputs. ValidTransactionComponenets will return an
// error if the state height is not sufficient to fulfill all of the
// requirements of the transaction.
func (s *State) ValidTransactionComponents(t Transaction) (err error) {
	s.mu.RLock()
	defer s.mu.RUnlock()

	err = t.FollowsStorageProofRules()
	if err != nil {
		return
	}
	err = s.validFileContracts(t)
	if err != nil {
		return
	}
	err = s.validStorageProofs(t)
	if err != nil {
		return
	}
	err = s.validSignatures(t)
	if err != nil {
		return
	}

	return
}

// ValidUnlockConditions checks that the conditions of uc have been met.
func (s *State) ValidUnlockConditions(uc UnlockConditions, uh UnlockHash) (err error) {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.validUnlockConditions(uc, uh)
}
<commit_msg>add check for max transaction size in ValidTransactionComponenets<commit_after>package consensus

import (
	"errors"
	"math/big"
	"sort"

	"github.com/NebulousLabs/Sia/crypto"
)

// StateInfo contains basic information about the State.
type StateInfo struct {
	CurrentBlock BlockID
	Height       BlockHeight
	Target       Target
}

// blockAtHeight returns the block on the current path with the given height.
func (s *State) blockAtHeight(height BlockHeight) (b Block, exists bool) {
	bn, exists := s.blockMap[s.currentPath[height]]
	if !exists {
		return
	}
	b = bn.block
	return
}

// currentBlockNode returns the blockNode of the current block.
func (s *State) currentBlockNode() *blockNode {
	return s.blockMap[s.currentBlockID]
}

// currentBlockWeight returns the weight of the current block.
func (s *State) currentBlockWeight() *big.Rat {
	return s.currentBlockNode().target.Inverse()
}

// height returns the current height of the state.
func (s *State) height() BlockHeight {
	return s.blockMap[s.currentBlockID].height
}

// output returns the unspent SiacoinOutput associated with the given ID. If
// the output is not in the UTXO set, 'exists' will be false.
func (s *State) output(id SiacoinOutputID) (sco SiacoinOutput, exists bool) {
	sco, exists = s.siacoinOutputs[id]
	return
}

// sortedUscoSet returns all of the unspent siacoin outputs sorted
// according to the numerical value of their id.
func (s *State) sortedUscoSet() []SiacoinOutput {
	// Get all of the outputs in string form and sort the strings.
	unspentOutputs := make(crypto.HashSlice, len(s.siacoinOutputs))
	for outputID := range s.siacoinOutputs {
		unspentOutputs = append(unspentOutputs, crypto.Hash(outputID))
	}
	sort.Sort(unspentOutputs)

	// Get the outputs in order according to their sorted form.
	sortedOutputs := make([]SiacoinOutput, len(unspentOutputs))
	for i, outputID := range unspentOutputs {
		output, _ := s.output(SiacoinOutputID(outputID))
		sortedOutputs[i] = output
	}
	return sortedOutputs
}

// Sorted UsfoSet returns all of the unspent siafund outputs sorted according
// to the numerical value of their id.
func (s *State) sortedUsfoSet() []SiafundOutput {
	// Get all of the outputs in string form and sort the strings.
	outputIDs := make(crypto.HashSlice, len(s.siafundOutputs))
	for outputID := range s.siafundOutputs {
		outputIDs = append(outputIDs, crypto.Hash(outputID))
	}
	sort.Sort(outputIDs)

	// Get the outputs in order according to their sorted string form.
	sortedOutputs := make([]SiafundOutput, len(outputIDs))
	for i, outputID := range outputIDs {
		// Sanity check - the output should exist.
		output, exists := s.siafundOutputs[SiafundOutputID(outputID)]
		if DEBUG {
			if !exists {
				panic("output doesn't exist")
			}
		}

		sortedOutputs[i] = output
	}
	return sortedOutputs
}

// BlockAtHeight returns the block on the current path with the given height.
func (s *State) BlockAtHeight(height BlockHeight) (b Block, exists bool) {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.blockAtHeight(height)
}

// Block returns the block associated with the given id.
func (s *State) Block(id BlockID) (b Block, exists bool) {
	s.mu.RLock()
	defer s.mu.RUnlock()

	node, exists := s.blockMap[id]
	if !exists {
		return
	}
	b = node.block
	return
}

// BlockOutputDiffs returns the SiacoinOutputDiffs for a given block.
func (s *State) BlockOutputDiffs(id BlockID) (scods []SiacoinOutputDiff, err error) {
	node, exists := s.blockMap[id]
	if !exists {
		err = errors.New("requested an unknown block")
		return
	}
	if !node.diffsGenerated {
		err = errors.New("diffs have not been generated for the requested block")
		return
	}
	scods = node.siacoinOutputDiffs
	return
}

// BlocksSince returns a set of output diffs representing how the state
// has changed since block 'id'. OutputDiffsSince will flip the `new` value for
// diffs that got reversed.
func (s *State) BlocksSince(id BlockID) (removedBlocks, addedBlocks []BlockID, err error) {
	s.mu.RLock()
	defer s.mu.RUnlock()

	node, exists := s.blockMap[id]
	if !exists {
		err = errors.New("block is unknown")
		return
	}

	// Get all the IDs from the blockchain to the current path.
	path := s.backtrackToCurrentPath(node)
	for i := len(path) - 1; i > 0; i-- {
		removedBlocks = append(removedBlocks, path[i].block.ID())
	}

	// Get all the IDs going forward from the common parent.
	for height := path[0].height + 1; ; height++ {
		if _, exists := s.currentPath[height]; !exists {
			break
		}

		node := s.blockMap[s.currentPath[height]]
		addedBlocks = append(addedBlocks, node.block.ID())
	}

	return
}

// FileContract returns the file contract associated with the 'id'. If the
// contract does not exist, exists will be false.
func (s *State) FileContract(id FileContractID) (fc FileContract, exists bool) {
	s.mu.RLock()
	defer s.mu.RUnlock()

	fc, exists = s.fileContracts[id]
	return
}

// CurrentBlock returns the highest block on the tallest fork.
func (s *State) CurrentBlock() Block {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.currentBlockNode().block
}

// CurrentTarget returns the target of the next block that needs to be
// submitted to the state.
func (s *State) CurrentTarget() Target {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.currentBlockNode().target
}

// EarliestTimestamp returns the earliest timestamp that the next block can
// have in order for it to be considered valid.
func (s *State) EarliestTimestamp() Timestamp {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.currentBlockNode().earliestChildTimestamp()
}

// Height returns the height of the current blockchain (the longest fork).
func (s *State) Height() BlockHeight {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.height()
}

// HeightOfBlock returns the height of the block with the given ID.
func (s *State) HeightOfBlock(bid BlockID) (height BlockHeight, exists bool) {
	s.mu.RLock()
	defer s.mu.RUnlock()

	bn, exists := s.blockMap[bid]
	if !exists {
		return
	}
	height = bn.height
	return
}

// SiacoinOutput returns the siacoin output associated with the given ID.
func (s *State) SiacoinOutput(id SiacoinOutputID) (output SiacoinOutput, exists bool) {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.output(id)
}

// SiafundOutput returns the siafund output associated with the given ID.
func (s *State) SiafundOutput(id SiafundOutputID) (output SiafundOutput, exists bool) {
	s.mu.RLock()
	defer s.mu.RUnlock()
	output, exists = s.siafundOutputs[id]
	return
}

// SortedUtxoSet returns all of the unspent transaction outputs sorted
// according to the numerical value of their id.
func (s *State) SortedUtxoSet() []SiacoinOutput {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.sortedUscoSet()
}

// StorageProofSegment returns the segment to be used in the storage proof for
// a given file contract.
func (s *State) StorageProofSegment(fcid FileContractID) (index uint64, err error) {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.storageProofSegment(fcid)
}

// ValidTransaction checks that a transaction is valid within the context of
// the current consensus set.
func (s *State) ValidTransaction(t Transaction) (err error) {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.validTransaction(t)
}

// ValidTransactionComponents checks that a transaction follows basic rules,
// such as the storage proof rules, and it checks that all of the signatures
// are valid, but it does not check that all of the inputs, storage proofs, and
// terminations act on existing outputs and contracts. This function is
// primarily for the transaction pool, which has access to unconfirmed
// transactions. ValidTransactionComponents will not return an error simply
// because there are missing inputs. ValidTransactionComponenets will return an
// error if the state height is not sufficient to fulfill all of the
// requirements of the transaction.
func (s *State) ValidTransactionComponents(t Transaction) (err error) {
	s.mu.RLock()
	defer s.mu.RUnlock()

	// This will stop too-large transactions from accidentally being validated.
	// This check doesn't happen when checking blocks, because the size of the
	// block was already checked.
	if len(enconding.Marshal(t)) < MaxBlockSize - 5e3 {
		return errors.New("transaction is too large")
	}

	err = t.FollowsStorageProofRules()
	if err != nil {
		return
	}
	err = s.validFileContracts(t)
	if err != nil {
		return
	}
	err = s.validStorageProofs(t)
	if err != nil {
		return
	}
	err = s.validSignatures(t)
	if err != nil {
		return
	}

	return
}

// ValidUnlockConditions checks that the conditions of uc have been met.
func (s *State) ValidUnlockConditions(uc UnlockConditions, uh UnlockHash) (err error) {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.validUnlockConditions(uc, uh)
}
<|endoftext|>


<commit_before>package main

import (
	d "dsbldr"
	"encoding/json"
	"fmt"
)

builder := d.Builder{
	BaseURL: "localhost:8080",
	RequestHeaders: map[string]string{
		"Authorization": BasicOAuthHeader(
			"OAUTH_CONSUMER_KEY",
			"OAUTH_NONCE",
			"OAUTH_SIGNATURE",
			"OAUTH_SIGNATURE_METHOD", "OAUTH_TIMESTAMP",
			"OAUTH_TOKEN",
		)
	}
}

builder.AddFeatures(
	&d.Feature{
		Name: "item_ids",
		Endpoint: "/items/",
		RunFunc: func(response []string) []string {
			responseMap = (make[map]string)
			json.Unmarshal(response, &responseMap)
		}
	},
	&d.Feature{
		Name: "item_prices",
		Endpoint: "/items/prices/{{item_ids}}/",
		RunFunc: func(response []string) []string {
			// blah blah
		}
	},
	&d.Feature{
		Name: "item_category",
		Endpoint: "/items/category/{{item_ids}}/",
		RunFunc: func(response string) []string {
			// blah blah
		}
	},
)

func main() {
	fmt.Print(err)
}<commit_msg>made minor modifications in demo.go<commit_after>package main

import (
	d "dsbldr"
	"encoding/json"
	"fmt"
)

builder d.Builder := d.Builder{
	BaseURL: "localhost:8080",
	RequestHeaders: map[string]string{
		"Authorization": BasicOAuthHeader(
			"OAUTH_CONSUMER_KEY",
			"OAUTH_NONCE",
			"OAUTH_SIGNATURE",
			"OAUTH_SIGNATURE_METHOD", "OAUTH_TIMESTAMP",
			"OAUTH_TOKEN",
		),
	},
}

builder.AddFeatures(
	&d.Feature{
		Name: "item_ids",
		Endpoint: "/items/",
		RunFunc: func(response []string) []string {
			responseMap = (make[map]string)
			json.Unmarshal(response, &responseMap)
		},
	},
	&d.Feature{
		Name: "item_prices",
		Endpoint: "/items/prices/{{item_ids}}/",
		RunFunc: func(response []string) []string {
			// blah blah
		},
	},
	&d.Feature{
		Name: "item_category",
		Endpoint: "/items/category/{{item_ids}}/",
		RunFunc: func(response []string) []string {
			// blah blah
		},
	},
)

//func main() {
//}
<|endoftext|>


<commit_before>package rootio

import (
	"bufio"
	B "encoding/binary"
	"fmt"
	"io"
	"os"
)

const LargeFileBoundary = 0x7FFFFFFF

var E = B.BigEndian

type Reader interface {
	io.Reader
	io.ReaderAt
	io.Seeker
	io.Closer
}

// A ROOT file is a suite of consecutive data records (TKey's) with
// the following format (see also the TKey class). If the key is
// located past the 32 bit file limit (> 2 GB) then some fields will
// be 8 instead of 4 bytes:
//    1->4            Nbytes    = Length of compressed object (in bytes)
//    5->6            Version   = TKey version identifier
//    7->10           ObjLen    = Length of uncompressed object
//    11->14          Datime    = Date and time when object was written to file
//    15->16          KeyLen    = Length of the key structure (in bytes)
//    17->18          Cycle     = Cycle of key
//    19->22 [19->26] SeekKey   = Pointer to record itself (consistency check)
//    23->26 [27->34] SeekPdir  = Pointer to directory header
//    27->27 [35->35] lname     = Number of bytes in the class name
//    28->.. [36->..] ClassName = Object Class Name
//    ..->..          lname     = Number of bytes in the object name
//    ..->..          Name      = lName bytes with the name of the object
//    ..->..          lTitle    = Number of bytes in the object title
//    ..->..          Title     = Title of the object
//    ----->          DATA      = Data bytes associated to the object
//
// The first data record starts at byte fBEGIN (currently set to kBEGIN).
// Bytes 1->kBEGIN contain the file description, when fVersion >= 1000000
// it is a large file (> 2 GB) and the offsets will be 8 bytes long and
// fUnits will be set to 8:
//    1->4            "root"      = Root file identifier
//    5->8            fVersion    = File format version
//    9->12           fBEGIN      = Pointer to first data record
//    13->16 [13->20] fEND        = Pointer to first free word at the EOF
//    17->20 [21->28] fSeekFree   = Pointer to FREE data record
//    21->24 [29->32] fNbytesFree = Number of bytes in FREE data record
//    25->28 [33->36] nfree       = Number of free data records
//    29->32 [37->40] fNbytesName = Number of bytes in TNamed at creation time
//    33->33 [41->41] fUnits      = Number of bytes for file pointers
//    34->37 [42->45] fCompress   = Compression level and algorithm
//    38->41 [46->53] fSeekInfo   = Pointer to TStreamerInfo record
//    42->45 [54->57] fNbytesInfo = Number of bytes in TStreamerInfo record
//    46->63 [58->75] fUUID       = Universal Unique ID
type File struct {
	Reader
	id string //non-root, identifies filename, etc.

	magic   [4]byte
	version int32
	begin   int64

	// Remainder of record is variable length, 4 or 8 bytes per pointer
	end         int64
	seekfree    int64 // first available record
	nbytesfree  int32 // total bytes available
	nfree       int32 // total free bytes
	nbytesname  int32 // number of bytes in TNamed at creation time
	units       byte
	compression int32
	seekinfo    int64 // pointer to TStreamerInfo
	nbytesinfo  int32 // sizeof(TStreamerInfo)
	uuid        [18]byte

	root directory // root directory of this file
}

// Open opens the named ROOT file for reading. If successful, methods on the
// returned file can be used for reading; the associated file descriptor
// has mode os.O_RDONLY.
func Open(path string) (*File, error) {
	fd, err := os.Open(path)
	if err != nil {
		return nil, fmt.Errorf("Unable to open %q (%q)", path, err.Error())
	}

	f := &File{
		Reader: fd,
		id:     path,
	}
	f.root = directory{file: f}

	err = f.readHeader()
	if err != nil {
		return nil, err
	}

	return f, nil
}

func (f *File) readHeader() (err error) {

	var stage string

	defer func() {
		if r := recover(); r != nil {
			err = fmt.Errorf("Error reading file named %q while %s (%q)",
				f.id, stage, r.(error).Error())
		}
	}()

	stage = "reading header"

	dec := rootDecoder{r: bufio.NewReader(f)}

	// Header

	err = dec.readBin(&f.magic)
	if err != nil {
		return err
	}

	if string(f.magic[:]) != "root" {
		return fmt.Errorf("%q is not a root file", f.id)
	}

	err = dec.readInt32(&f.version)
	if err != nil {
		return err
	}

	err = dec.readInt32(&f.begin)
	if err != nil {
		return err
	}

	if f.version < 1000000 { // small file
		err = dec.readInt32(&f.end)
		if err != nil {
			return err
		}

		err = dec.readInt32(&f.seekfree)
		if err != nil {
			return err
		}

		err = dec.readInt32(&f.nbytesfree)
		if err != nil {
			return err
		}

		err = dec.readInt32(&f.nfree)
		if err != nil {
			return err
		}

		err = dec.readInt32(&f.nbytesname)
		if err != nil {
			return err
		}

		err = dec.readBin(&f.units)
		if err != nil {
			return err
		}

		err = dec.readInt32(&f.compression)
		if err != nil {
			return err
		}

		err = dec.readInt32(&f.seekinfo)
		if err != nil {
			return err
		}

		err = dec.readInt32(&f.nbytesinfo)
		if err != nil {
			return err
		}

	} else { // large files
		err = dec.readInt64(&f.end)
		if err != nil {
			return err
		}

		err = dec.readInt64(&f.seekfree)
		if err != nil {
			return err
		}

		err = dec.readInt32(&f.nbytesfree)
		if err != nil {
			return err
		}

		err = dec.readInt32(&f.nfree)
		if err != nil {
			return err
		}

		err = dec.readInt32(&f.nbytesname)
		if err != nil {
			return err
		}

		err = dec.readBin(&f.units)
		if err != nil {
			return err
		}

		err = dec.readInt32(&f.compression)
		if err != nil {
			return err
		}

		err = dec.readInt64(&f.seekinfo)
		if err != nil {
			return err
		}

		err = dec.readInt32(&f.nbytesinfo)
		if err != nil {
			return err
		}
	}

	err = dec.readBin(&f.uuid)
	if err != nil {
		return err
	}

	stage = "read directory info"
	err = f.root.readDirInfo()
	if err != nil {
		return err
	}

	stage = "read keys of top directory"
	err = f.root.readKeys()
	if err != nil {
		return err
	}

	stage = "reading streamerinfos"

	stage = "reading keys"

	// Contents of file

	// fmt.Printf("==================================>>>\n")
	// _, err = f.Seek(int64(f.begin), os.SEEK_SET)
	// for f.Tell() < f.end {
	// 	err = f.root.readKey()
	// 	if err != nil {
	// 		return err
	// 	}
	// }
	// fmt.Printf("<<<==================================\n")

	return err
}

func (f *File) Map() {
	for _, k := range f.root.keys {
		if k.classname == "TBasket" {
			//b := k.AsBasket()
			fmt.Printf("%8s %60s %6v %6v %f\n", k.classname, k.name, k.bytes-k.keylen, k.objlen, float64(k.objlen)/float64(k.bytes-k.keylen))
		} else {
			//println(k.classname, k.name, k.title)
			fmt.Printf("%8s %60s %6v %6v %f\n", k.classname, k.name, k.bytes-k.keylen, k.objlen, float64(k.objlen)/float64(k.bytes-k.keylen))
		}
	}

}

func (f *File) Tell() int64 {
	where, err := f.Seek(0, os.SEEK_CUR)
	if err != nil {
		panic(err)
	}
	return where
}

// Close closes the File, rendering it unusable for I/O. It returns an
// error, if any.
func (f *File) Close() error {
	for _, k := range f.root.keys {
		k.f = nil
	}
	f.root.keys = nil
	f.root.file = nil
	return f.Reader.Close()
}

// Keys returns the list of keys this File contains
func (f *File) Keys() []Key {
	return f.root.keys
}

func (f *File) Name() string {
	return f.root.Name()
}

func (f *File) Title() string {
	return f.root.Title()
}

func (f *File) Class() string {
	return "TFile"
}

// Get returns the object identified by namecycle
//   namecycle has the format name;cycle
//   name  = * is illegal, cycle = * is illegal
//   cycle = "" or cycle = 9999 ==> apply to a memory object
//
//   examples:
//     foo   : get object named foo in memory
//             if object is not in memory, try with highest cycle from file
//     foo;1 : get cycle 1 of foo on file
func (f *File) Get(namecycle string) (Object, bool) {
	return f.root.Get(namecycle)
}

// testing interfaces
var _ Object = (*File)(nil)
var _ Directory = (*File)(nil)

// EOF
<commit_msg>file: use NewDecoder<commit_after>package rootio

import (
	"bytes"
	B "encoding/binary"
	"fmt"
	"io"
	"os"
)

const LargeFileBoundary = 0x7FFFFFFF

var E = B.BigEndian

type Reader interface {
	io.Reader
	io.ReaderAt
	io.Seeker
	io.Closer
}

// A ROOT file is a suite of consecutive data records (TKey's) with
// the following format (see also the TKey class). If the key is
// located past the 32 bit file limit (> 2 GB) then some fields will
// be 8 instead of 4 bytes:
//    1->4            Nbytes    = Length of compressed object (in bytes)
//    5->6            Version   = TKey version identifier
//    7->10           ObjLen    = Length of uncompressed object
//    11->14          Datime    = Date and time when object was written to file
//    15->16          KeyLen    = Length of the key structure (in bytes)
//    17->18          Cycle     = Cycle of key
//    19->22 [19->26] SeekKey   = Pointer to record itself (consistency check)
//    23->26 [27->34] SeekPdir  = Pointer to directory header
//    27->27 [35->35] lname     = Number of bytes in the class name
//    28->.. [36->..] ClassName = Object Class Name
//    ..->..          lname     = Number of bytes in the object name
//    ..->..          Name      = lName bytes with the name of the object
//    ..->..          lTitle    = Number of bytes in the object title
//    ..->..          Title     = Title of the object
//    ----->          DATA      = Data bytes associated to the object
//
// The first data record starts at byte fBEGIN (currently set to kBEGIN).
// Bytes 1->kBEGIN contain the file description, when fVersion >= 1000000
// it is a large file (> 2 GB) and the offsets will be 8 bytes long and
// fUnits will be set to 8:
//    1->4            "root"      = Root file identifier
//    5->8            fVersion    = File format version
//    9->12           fBEGIN      = Pointer to first data record
//    13->16 [13->20] fEND        = Pointer to first free word at the EOF
//    17->20 [21->28] fSeekFree   = Pointer to FREE data record
//    21->24 [29->32] fNbytesFree = Number of bytes in FREE data record
//    25->28 [33->36] nfree       = Number of free data records
//    29->32 [37->40] fNbytesName = Number of bytes in TNamed at creation time
//    33->33 [41->41] fUnits      = Number of bytes for file pointers
//    34->37 [42->45] fCompress   = Compression level and algorithm
//    38->41 [46->53] fSeekInfo   = Pointer to TStreamerInfo record
//    42->45 [54->57] fNbytesInfo = Number of bytes in TStreamerInfo record
//    46->63 [58->75] fUUID       = Universal Unique ID
type File struct {
	Reader
	id string //non-root, identifies filename, etc.

	magic   [4]byte
	version int32
	begin   int64

	// Remainder of record is variable length, 4 or 8 bytes per pointer
	end         int64
	seekfree    int64 // first available record
	nbytesfree  int32 // total bytes available
	nfree       int32 // total free bytes
	nbytesname  int32 // number of bytes in TNamed at creation time
	units       byte
	compression int32
	seekinfo    int64 // pointer to TStreamerInfo
	nbytesinfo  int32 // sizeof(TStreamerInfo)
	uuid        [18]byte

	root directory // root directory of this file
}

// Open opens the named ROOT file for reading. If successful, methods on the
// returned file can be used for reading; the associated file descriptor
// has mode os.O_RDONLY.
func Open(path string) (*File, error) {
	fd, err := os.Open(path)
	if err != nil {
		return nil, fmt.Errorf("Unable to open %q (%q)", path, err.Error())
	}

	f := &File{
		Reader: fd,
		id:     path,
	}
	f.root = directory{file: f}

	err = f.readHeader()
	if err != nil {
		return nil, err
	}

	return f, nil
}

func (f *File) readHeader() (err error) {

	var stage string

	defer func() {
		if r := recover(); r != nil {
			err = fmt.Errorf("Error reading file named %q while %s (%q)",
				f.id, stage, r.(error).Error())
		}
	}()

	stage = "reading header"

	buf := make([]byte, 64)
	_, err = f.ReadAt(buf, 0)
	if err != nil {
		return err
	}

	dec := NewDecoder(bytes.NewBuffer(buf))

	// Header

	err = dec.readBin(&f.magic)
	if err != nil {
		return err
	}

	if string(f.magic[:]) != "root" {
		return fmt.Errorf("%q is not a root file", f.id)
	}

	err = dec.readInt32(&f.version)
	if err != nil {
		return err
	}

	err = dec.readInt32(&f.begin)
	if err != nil {
		return err
	}

	if f.version < 1000000 { // small file
		err = dec.readInt32(&f.end)
		if err != nil {
			return err
		}

		err = dec.readInt32(&f.seekfree)
		if err != nil {
			return err
		}

		err = dec.readInt32(&f.nbytesfree)
		if err != nil {
			return err
		}

		err = dec.readInt32(&f.nfree)
		if err != nil {
			return err
		}

		err = dec.readInt32(&f.nbytesname)
		if err != nil {
			return err
		}

		err = dec.readBin(&f.units)
		if err != nil {
			return err
		}

		err = dec.readInt32(&f.compression)
		if err != nil {
			return err
		}

		err = dec.readInt32(&f.seekinfo)
		if err != nil {
			return err
		}

		err = dec.readInt32(&f.nbytesinfo)
		if err != nil {
			return err
		}

	} else { // large files
		err = dec.readInt64(&f.end)
		if err != nil {
			return err
		}

		err = dec.readInt64(&f.seekfree)
		if err != nil {
			return err
		}

		err = dec.readInt32(&f.nbytesfree)
		if err != nil {
			return err
		}

		err = dec.readInt32(&f.nfree)
		if err != nil {
			return err
		}

		err = dec.readInt32(&f.nbytesname)
		if err != nil {
			return err
		}

		err = dec.readBin(&f.units)
		if err != nil {
			return err
		}

		err = dec.readInt32(&f.compression)
		if err != nil {
			return err
		}

		err = dec.readInt64(&f.seekinfo)
		if err != nil {
			return err
		}

		err = dec.readInt32(&f.nbytesinfo)
		if err != nil {
			return err
		}
	}

	err = dec.readBin(&f.uuid)
	if err != nil {
		return err
	}

	stage = "read directory info"
	err = f.root.readDirInfo()
	if err != nil {
		return err
	}

	stage = "read streamerinfos"
	err = f.readStreamerInfo()
	if err != nil {
		return err
	}

	stage = "read keys"
	err = f.root.readKeys()
	if err != nil {
		return err
	}

	return err
}

func (f *File) Map() {
	for _, k := range f.root.keys {
		if k.classname == "TBasket" {
			//b := k.AsBasket()
			fmt.Printf("%8s %60s %6v %6v %f\n", k.classname, k.name, k.bytes-k.keylen, k.objlen, float64(k.objlen)/float64(k.bytes-k.keylen))
		} else {
			//println(k.classname, k.name, k.title)
			fmt.Printf("%8s %60s %6v %6v %f\n", k.classname, k.name, k.bytes-k.keylen, k.objlen, float64(k.objlen)/float64(k.bytes-k.keylen))
		}
	}

}

func (f *File) Tell() int64 {
	where, err := f.Seek(0, os.SEEK_CUR)
	if err != nil {
		panic(err)
	}
	return where
}

// Close closes the File, rendering it unusable for I/O. It returns an
// error, if any.
func (f *File) Close() error {
	for _, k := range f.root.keys {
		k.f = nil
	}
	f.root.keys = nil
	f.root.file = nil
	return f.Reader.Close()
}

// Keys returns the list of keys this File contains
func (f *File) Keys() []Key {
	return f.root.keys
}

func (f *File) Name() string {
	return f.root.Name()
}

func (f *File) Title() string {
	return f.root.Title()
}

func (f *File) Class() string {
	return "TFile"
}

// Get returns the object identified by namecycle
//   namecycle has the format name;cycle
//   name  = * is illegal, cycle = * is illegal
//   cycle = "" or cycle = 9999 ==> apply to a memory object
//
//   examples:
//     foo   : get object named foo in memory
//             if object is not in memory, try with highest cycle from file
//     foo;1 : get cycle 1 of foo on file
func (f *File) Get(namecycle string) (Object, bool) {
	return f.root.Get(namecycle)
}

// testing interfaces
var _ Object = (*File)(nil)
var _ Directory = (*File)(nil)

// EOF
<|endoftext|>


<commit_before>package main

import (
	"io/ioutil"
	"os"
	"fmt"
	"net/http"
	"io"
	"path/filepath"
	"bytes"
	"archive/zip"
	"strings"
)

// Download the zip file at the given URL to a temporary local directory.
// Returns the asbolute path to the downloaded zip file.
// IMPORTANT: You must call "defer os.RemoveAll(dir)" in the calling function when done with the downloaded zip file!
func downloadGithubZipFile(githubRelease gitHubCommit, githubToken string) (string, *fetchError) {

	// Create a temp directory
	// Note that ioutil.TempDir has a peculiar interface. We need not specify any meaningful values to achieve our
	// goal of getting a temporary directory.
	tempDir, err := ioutil.TempDir("", "")
	if err != nil {
		return "", wrapError(err)
	}

	// Create an empty file to write to
	file, err := os.Create(filepath.Join(tempDir, "repo.zip"))
	if err != nil {
		return "", wrapError(err)
	}
	defer file.Close()

	// Define the url
	url := fmt.Sprintf("https://api.github.com/repos/%s/%s/zipball/%s", githubRelease.repo.Owner, githubRelease.repo.Name, githubRelease.gitTag)

	// Download the file, possibly using the GitHub oAuth Token
	httpClient := &http.Client{}
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return "", wrapError(err)
	}

	if githubToken != "" {
		req.Header.Set("Authorization", fmt.Sprintf("token %s", githubToken))
	}

	resp, err := httpClient.Do(req)
	if err != nil {
		return "", wrapError(err)
	}
	if resp.StatusCode != 200 {
		// Convert the resp.Body to a string
		buf := new(bytes.Buffer)
		buf.ReadFrom(resp.Body)
		respBody := buf.String()

		return "", newError(500, fmt.Sprintf("Failed to download file at the url %s. Received HTTP Response %d. Body: %s", url, resp.StatusCode, respBody))
	}
	if resp.Header.Get("Content-Type") != "application/zip" {
		return "", newError(500, fmt.Sprintf("Failed to download file at the url %s. Expected HTTP Response's \"Content-Type\" header to be \"application/zip\", but was \"%s\"", url, resp.Header.Get("Content-Type")))
	}

	// Copy the contents of the downloaded file to our empty file
	_, err = io.Copy(file, resp.Body)
	if err != nil {
		return "", wrapError(err)
	}

	return filepath.Join(tempDir, "repo.zip"), nil
}

// extractFiles decompresses the file at zipFileAbsPath and moves only those files under filesToExtractFromZipPath to localPath
func extractFiles(zipFilePath, filesToExtractFromZipPath, localPath string) error {

	// Open the zip file for reading.
	r, err := zip.OpenReader(zipFilePath)
	if err != nil {
		return err
	}
	defer r.Close()

	// pathPrefix represents the portion of the local file path we will ignore when copying the file to localPath
	// E.g. full path = fetch-test-public-0.0.3/folder/file1.txt
	//      path prefix = fetch-test-public-0.0.3
	//      file that will eventually get written = <localPath>/folder/file1.txt

	// By convention, the first file in the zip file is the top-level directory
	pathPrefix := r.File[0].Name

	// Add the path from which we will extract files to the path prefix so we can exclude the appropriate files
	pathPrefix = filepath.Join(pathPrefix, filesToExtractFromZipPath)

	// Iterate through the files in the archive,
	// printing some of their contents.
	for _, f := range r.File {

		// If the given file is in the filesToExtractFromZipPath, proceed
		if strings.Index(f.Name, pathPrefix) == 0 {

			// Read the contents of the file in the .zip file
			readCloser, err := f.Open()
			if err != nil {
				return fmt.Errorf("Failed to open file %s: %s", f.Name, err)
			}
			defer readCloser.Close()

			if f.FileInfo().IsDir() {
				// Create a directory
				os.MkdirAll(filepath.Join(localPath, strings.TrimPrefix(f.Name, pathPrefix)), 0777)
			} else {
				// Create a new empty file
				fmt.Printf("Writing file %s\n", filepath.Join(localPath, strings.TrimPrefix(f.Name, pathPrefix)))
				file, err := os.Create(filepath.Join(localPath, strings.TrimPrefix(f.Name, pathPrefix)))
				if err != nil {
					return fmt.Errorf("Failed to create new file: %s", err)
				}
				defer file.Close()

				// Copy the contents to it
				_, err = io.Copy(file, readCloser)
				if err != nil {
					return fmt.Errorf("Failed to copy file: %s", err)
				}
			}
		}
	}

	return nil
}

// getZipFileName extracts the zip file name from the absolute file path of the zip file
// It returns the name without the .zip suffix
func getZipFileName(zipFilePath string) string {
	exploded := strings.Split(zipFilePath, "/")
	return strings.TrimSuffix(exploded[len(exploded)-1], ".zip")
}<commit_msg>Replace extraneous create empty file with ioutil.WriteFile<commit_after>package main

import (
	"io/ioutil"
	"os"
	"fmt"
	"net/http"
	"path/filepath"
	"bytes"
	"archive/zip"
	"strings"
)

// Download the zip file at the given URL to a temporary local directory.
// Returns the asbolute path to the downloaded zip file.
// IMPORTANT: You must call "defer os.RemoveAll(dir)" in the calling function when done with the downloaded zip file!
func downloadGithubZipFile(githubRelease gitHubCommit, githubToken string) (string, *fetchError) {

	// Create a temp directory
	// Note that ioutil.TempDir has a peculiar interface. We need not specify any meaningful values to achieve our
	// goal of getting a temporary directory.
	tempDir, err := ioutil.TempDir("", "")
	if err != nil {
		return "", wrapError(err)
	}

	// Define the url
	url := fmt.Sprintf("https://api.github.com/repos/%s/%s/zipball/%s", githubRelease.repo.Owner, githubRelease.repo.Name, githubRelease.gitTag)

	// Download the file, possibly using the GitHub oAuth Token
	httpClient := &http.Client{}
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return "", wrapError(err)
	}

	if githubToken != "" {
		req.Header.Set("Authorization", fmt.Sprintf("token %s", githubToken))
	}

	resp, err := httpClient.Do(req)
	if err != nil {
		return "", wrapError(err)
	}

	// Load the resp.Body into a buffer so we can convert it to a string or []bytes as necessary
	respBodyBuffer := new(bytes.Buffer)
	respBodyBuffer.ReadFrom(resp.Body)

	if resp.StatusCode != 200 {
		return "", newError(500, fmt.Sprintf("Failed to download file at the url %s. Received HTTP Response %d. Body: %s", url, resp.StatusCode, respBodyBuffer.String()))
	}
	if resp.Header.Get("Content-Type") != "application/zip" {
		return "", newError(500, fmt.Sprintf("Failed to download file at the url %s. Expected HTTP Response's \"Content-Type\" header to be \"application/zip\", but was \"%s\"", url, resp.Header.Get("Content-Type")))
	}

	// Copy the contents of the downloaded file to our empty file
	err = ioutil.WriteFile(filepath.Join(tempDir, "repo.zip"), respBodyBuffer.Bytes(), 0644)
	if err != nil {
		return "", wrapError(err)
	}

	return filepath.Join(tempDir, "repo.zip"), nil
}

// Decompresse the file at zipFileAbsPath and move only those files under filesToExtractFromZipPath to localPath
func extractFiles(zipFilePath, filesToExtractFromZipPath, localPath string) error {

	// Open the zip file for reading.
	r, err := zip.OpenReader(zipFilePath)
	if err != nil {
		return err
	}
	defer r.Close()

	// pathPrefix represents the portion of the local file path we will ignore when copying the file to localPath
	// E.g. full path = fetch-test-public-0.0.3/folder/file1.txt
	//      path prefix = fetch-test-public-0.0.3
	//      file that will eventually get written = <localPath>/folder/file1.txt

	// By convention, the first file in the zip file is the top-level directory
	pathPrefix := r.File[0].Name

	// Add the path from which we will extract files to the path prefix so we can exclude the appropriate files
	pathPrefix = filepath.Join(pathPrefix, filesToExtractFromZipPath)

	// Iterate through the files in the archive,
	// printing some of their contents.
	for _, f := range r.File {

		// If the given file is in the filesToExtractFromZipPath, proceed
		if strings.Index(f.Name, pathPrefix) == 0 {

			// Read the contents of the file in the .zip file
			readCloser, err := f.Open()
			if err != nil {
				return fmt.Errorf("Failed to open file %s: %s", f.Name, err)
			}
			defer readCloser.Close()

			if f.FileInfo().IsDir() {
				// Create a directory
				os.MkdirAll(filepath.Join(localPath, strings.TrimPrefix(f.Name, pathPrefix)), 0777)
			} else {
				// Read the file into a byte array
				var bytesBuffer []byte
				readCloser.Read(bytesBuffer)

				// Write the file
				err = ioutil.WriteFile(filepath.Join(localPath, strings.TrimPrefix(f.Name, pathPrefix)), bytesBuffer, 0644)
				if err != nil {
					return fmt.Errorf("Failed to write file: %s", err)
				}
			}
		}
	}

	return nil
}<|endoftext|>


<commit_before>package the_platinum_searcher

import (
	"io/ioutil"
	"os"

	"path/filepath"
	"strings"
)

type find struct {
	Out    chan *GrepParams
	Option *Option
}

func Find(root string, pattern *Pattern, out chan *GrepParams, option *Option) {
	find := find{
		Out:    out,
		Option: option,
	}
	find.Start(root, pattern)
}

func (f *find) Start(root string, pattern *Pattern) {
	if f.Option.SearchStream {
		f.findStream(pattern)
	} else {
		f.findFile(root, pattern)
	}
}

func (f *find) findStream(pattern *Pattern) {
	// TODO: File type is fixed in ASCII because it can not determine the character code.
	f.Out <- &GrepParams{"", ASCII, pattern}
	close(f.Out)
}

func (f *find) findFile(root string, pattern *Pattern) {

	var ignores ignoreMatchers
	if f.Option.NoPtIgnore == false {
		if homePtIgnore := homePtIgnore(); homePtIgnore != nil {
			ignores = append(ignores, homePtIgnore)
		}
	}

	if f.Option.NoGlobalGitIgnore == false {
		ignores = append(ignores, globalGitIgnore())
	}

	ignores = append(ignores, genericIgnore(f.Option.Ignore))
	Walk(root, ignores, f.Option.Follow, func(path string, info *FileInfo, depth int, ignores ignoreMatchers, err error) (error, ignoreMatchers) {
		if info.IsDir() {
			if depth > f.Option.Depth+1 {
				return filepath.SkipDir, ignores
			}
			//Current Directory skipping should be checked first before loading ignores
			//within this directory
			if !isRoot(depth) && isHidden(info.Name()) {
				return filepath.SkipDir, ignores
			} else {
				if ignores.Match(path, info.IsDir(), depth) {
					return filepath.SkipDir, ignores
				}
			}
			ignores = append(ignores, newIgnoreMatchers(path, f.Option.VcsIgnores(), depth+1)...)
			return nil, ignores
		}
		if !info.follow && info.IsSymlink() {
			return nil, ignores
		}
		if !isRoot(depth) && isHidden(info.Name()) {
			return nil, ignores
		}

		if ignores.Match(path, info.IsDir(), depth) {
			return nil, ignores
		}

		if pattern.FileRegexp != nil && !pattern.FileRegexp.MatchString(path) {
			return nil, ignores
		}
		fileType := UNKNOWN
		if f.Option.FilesWithRegexp == "" {
			fileType = IdentifyType(path)
			if fileType == ERROR || fileType == BINARY {
				return nil, ignores
			}
		}
		f.Out <- &GrepParams{path, fileType, pattern}
		return nil, ignores
	})
	close(f.Out)
}

type WalkFunc func(path string, info *FileInfo, depth int, ignores ignoreMatchers, err error) (error, ignoreMatchers)

func Walk(root string, ignores ignoreMatchers, follow bool, walkFn WalkFunc) error {
	info, err := os.Lstat(root)
	fileInfo := newFileInfo(root, info, follow)
	if err != nil {
		walkError, _ := walkFn(root, fileInfo, 1, nil, err)
		return walkError
	}
	return walk(root, fileInfo, 1, ignores, walkFn)
}

func walkOnGoRoutine(path string, info *FileInfo, notify chan int, depth int, parentIgnore ignoreMatchers, walkFn WalkFunc) {
	walk(path, info, depth, parentIgnore, walkFn)
	notify <- 0
}

func walk(path string, info *FileInfo, depth int, parentIgnores ignoreMatchers, walkFn WalkFunc) error {
	err, ig := walkFn(path, info, depth, parentIgnores, nil)
	if err != nil {
		if info.IsDir() && err == filepath.SkipDir {
			return nil
		}
		return err
	}

	if !info.IsDir() {
		return nil
	}

	list, err := ioutil.ReadDir(path)
	if err != nil {
		walkError, _ := walkFn(path, info, depth, ig, err)
		return walkError
	}

	depth++
	notify := make(chan int, len(list))
	for _, l := range list {
		fileInfo := newFileInfo(path, l, info.follow)
		if isDirectRoot(depth) {
			go walkOnGoRoutine(filepath.Join(path, fileInfo.Name()), fileInfo, notify, depth, ig, walkFn)

		} else {
			walk(filepath.Join(path, fileInfo.Name()), fileInfo, depth, ig, walkFn)
		}
	}
	if isDirectRoot(depth) {
		for i := 0; i < cap(notify); i++ {
			<-notify
		}
	}
	return nil
}

func isRoot(depth int) bool {
	return depth == 1
}

func isDirectRoot(depth int) bool {
	return depth == 2
}

func isHidden(name string) bool {
	return strings.HasPrefix(name, ".") && len(name) > 1
}

func contains(path string, patterns *[]string) bool {
	for _, p := range *patterns {
		if p == path {
			return true
		}
	}
	return false
}
<commit_msg>Fixed a bug that dosen't work absolute ignore pattern.<commit_after>package the_platinum_searcher

import (
	"io/ioutil"
	"os"

	"path/filepath"
	"strings"
)

type find struct {
	Out    chan *GrepParams
	Option *Option
}

func Find(root string, pattern *Pattern, out chan *GrepParams, option *Option) {
	find := find{
		Out:    out,
		Option: option,
	}
	find.Start(root, pattern)
}

func (f *find) Start(root string, pattern *Pattern) {
	if f.Option.SearchStream {
		f.findStream(pattern)
	} else {
		f.findFile(root, pattern)
	}
}

func (f *find) findStream(pattern *Pattern) {
	// TODO: File type is fixed in ASCII because it can not determine the character code.
	f.Out <- &GrepParams{"", ASCII, pattern}
	close(f.Out)
}

func (f *find) findFile(root string, pattern *Pattern) {

	var ignores ignoreMatchers
	if f.Option.NoPtIgnore == false {
		if homePtIgnore := homePtIgnore(); homePtIgnore != nil {
			ignores = append(ignores, homePtIgnore)
		}
	}

	if f.Option.NoGlobalGitIgnore == false {
		ignores = append(ignores, globalGitIgnore())
	}

	ignores = append(ignores, genericIgnore(f.Option.Ignore))
	Walk(root, ignores, f.Option.Follow, func(path string, info *FileInfo, depth int, ignores ignoreMatchers, err error) (error, ignoreMatchers) {
		if info.IsDir() {
			if depth > f.Option.Depth+1 {
				return filepath.SkipDir, ignores
			}
			//Current Directory skipping should be checked first before loading ignores
			//within this directory
			if !isRoot(depth) && isHidden(info.Name()) {
				return filepath.SkipDir, ignores
			} else {
				if ignores.Match(path, info.IsDir(), depth) {
					return filepath.SkipDir, ignores
				}
			}
			ignores = append(ignores, newIgnoreMatchers(path, f.Option.VcsIgnores(), depth+2)...)
			return nil, ignores
		}
		if !info.follow && info.IsSymlink() {
			return nil, ignores
		}
		if !isRoot(depth) && isHidden(info.Name()) {
			return nil, ignores
		}

		if ignores.Match(path, info.IsDir(), depth) {
			return nil, ignores
		}

		if pattern.FileRegexp != nil && !pattern.FileRegexp.MatchString(path) {
			return nil, ignores
		}
		fileType := UNKNOWN
		if f.Option.FilesWithRegexp == "" {
			fileType = IdentifyType(path)
			if fileType == ERROR || fileType == BINARY {
				return nil, ignores
			}
		}
		f.Out <- &GrepParams{path, fileType, pattern}
		return nil, ignores
	})
	close(f.Out)
}

type WalkFunc func(path string, info *FileInfo, depth int, ignores ignoreMatchers, err error) (error, ignoreMatchers)

func Walk(root string, ignores ignoreMatchers, follow bool, walkFn WalkFunc) error {
	info, err := os.Lstat(root)
	fileInfo := newFileInfo(root, info, follow)
	if err != nil {
		walkError, _ := walkFn(root, fileInfo, 1, nil, err)
		return walkError
	}
	return walk(root, fileInfo, 1, ignores, walkFn)
}

func walkOnGoRoutine(path string, info *FileInfo, notify chan int, depth int, parentIgnore ignoreMatchers, walkFn WalkFunc) {
	walk(path, info, depth, parentIgnore, walkFn)
	notify <- 0
}

func walk(path string, info *FileInfo, depth int, parentIgnores ignoreMatchers, walkFn WalkFunc) error {
	err, ig := walkFn(path, info, depth, parentIgnores, nil)
	if err != nil {
		if info.IsDir() && err == filepath.SkipDir {
			return nil
		}
		return err
	}

	if !info.IsDir() {
		return nil
	}

	list, err := ioutil.ReadDir(path)
	if err != nil {
		walkError, _ := walkFn(path, info, depth, ig, err)
		return walkError
	}

	depth++
	notify := make(chan int, len(list))
	for _, l := range list {
		fileInfo := newFileInfo(path, l, info.follow)
		if isDirectRoot(depth) {
			go walkOnGoRoutine(filepath.Join(path, fileInfo.Name()), fileInfo, notify, depth, ig, walkFn)

		} else {
			walk(filepath.Join(path, fileInfo.Name()), fileInfo, depth, ig, walkFn)
		}
	}
	if isDirectRoot(depth) {
		for i := 0; i < cap(notify); i++ {
			<-notify
		}
	}
	return nil
}

func isRoot(depth int) bool {
	return depth == 1
}

func isDirectRoot(depth int) bool {
	return depth == 2
}

func isHidden(name string) bool {
	return strings.HasPrefix(name, ".") && len(name) > 1
}

func contains(path string, patterns *[]string) bool {
	for _, p := range *patterns {
		if p == path {
			return true
		}
	}
	return false
}
<|endoftext|>