<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"strings"
)

type Score struct {
	FileName                        string
	WordCount, LineCount, ByteCount int
}

func (self *Score) Add(other *Score) *Score {
	self.WordCount += other.WordCount
	self.LineCount += other.LineCount
	self.ByteCount += other.ByteCount
	return self
}

var w, l, c bool

func init() {
	flag.BoolVar(&l, "l", true, "Count the number of lines")
	flag.BoolVar(&w, "w", true, "Count the number of words")
	flag.BoolVar(&c, "c", true, "Count the number of bytes")
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: wc [-clmw] file\n\n")
		flag.VisitAll(usage)
	}
}

func usage(f *flag.Flag) {
	fmt.Printf("  -%s\t%s\n", f.Name, f.Usage)
}

func words(str *string) int {
	scanner := bufio.NewScanner(strings.NewReader(*str))
	// Set the split function for the scanning operation.
	scanner.Split(bufio.ScanWords)
	// Count the words.
	count := 0
	for scanner.Scan() {
		count++
	}
	return count
}

func lines(str *string) int {
	scanner := bufio.NewScanner(strings.NewReader(*str))
	// Set the split function for the scanning operation.
	scanner.Split(bufio.ScanLines)
	// Count the words.
	count := 0
	for scanner.Scan() {
		count++
	}
	return count
}

func count_file(fileName string) *Score {
	bytes, _ := ioutil.ReadFile(fileName)
	score := count(bytes)
	score.FileName = fileName
	return score
}

func count(bytes []byte) *Score {
	str := string(bytes)
	return &Score{
		LineCount: lines(&str),
		WordCount: words(&str),
		ByteCount: len(bytes)}
}

func Printout(score *Score) {
	if !(w && l && c) {
		fmt.Printf("%d\t%d\t%d ", score.LineCount, score.WordCount, score.ByteCount)
	} else {
		if l {
			fmt.Printf("%d\t ", score.LineCount)
		}
		if w {
			fmt.Printf("%d\t ", score.WordCount)
		}
		if c {
			fmt.Printf("%d\t ", score.ByteCount)
		}
		fmt.Printf("%s\n", score.FileName)
	}
}

func main() {
	flag.Parse()
	fileCount := len(flag.Args())
	var score *Score
	if fileCount == 1 {
		score = count_file(flag.Args()[0])
	} else if fileCount > 1 {
		score = &Score{WordCount: 0, LineCount: 0, ByteCount: 0, FileName: "total"}
		for i := 0; i < fileCount; i++ {
			s := count_file(flag.Args()[i])
			score.Add(s)
			Printout(s)
		}
	} else {
		bytes, _ := ioutil.ReadAll(os.Stdin)
		score = count(bytes)
	}
	Printout(score)
}
<commit_msg>Nicer wc output with tabwriter<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"strings"
	"text/tabwriter"
)

type Score struct {
	FileName                        string
	WordCount, LineCount, ByteCount int
}

func (self *Score) Add(other *Score) *Score {
	self.WordCount += other.WordCount
	self.LineCount += other.LineCount
	self.ByteCount += other.ByteCount
	return self
}

var w, l, c bool

func init() {
	flag.BoolVar(&l, "l", true, "Count the number of lines")
	flag.BoolVar(&w, "w", true, "Count the number of words")
	flag.BoolVar(&c, "c", true, "Count the number of bytes")
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: wc [-clmw] file\n\n")
		flag.VisitAll(usage)
	}
}

func usage(f *flag.Flag) {
	fmt.Printf("  -%s\t%s\n", f.Name, f.Usage)
}

func words(str *string) int {
	scanner := bufio.NewScanner(strings.NewReader(*str))
	// Set the split function for the scanning operation.
	scanner.Split(bufio.ScanWords)
	// Count the words.
	count := 0
	for scanner.Scan() {
		count++
	}
	return count
}

func lines(str *string) int {
	scanner := bufio.NewScanner(strings.NewReader(*str))
	// Set the split function for the scanning operation.
	scanner.Split(bufio.ScanLines)
	// Count the words.
	count := 0
	for scanner.Scan() {
		count++
	}
	return count
}

func count_file(fileName string) *Score {
	bytes, _ := ioutil.ReadFile(fileName)
	score := count(bytes)
	score.FileName = fileName
	return score
}

func count(bytes []byte) *Score {
	str := string(bytes)
	return &Score{
		LineCount: lines(&str),
		WordCount: words(&str),
		ByteCount: len(bytes)}
}

func Printout(score *Score) {
	out := ""
	if !(w && l && c) {
		out = fmt.Sprintf("%d\t%d\t%d ", score.LineCount, score.WordCount, score.ByteCount)
	} else {
		if l {
			out = fmt.Sprintf("%d\t ", score.LineCount)
		}
		if w {
			out = fmt.Sprintf("%s%d\t ", out, score.WordCount)
		}
		if c {
			out = fmt.Sprintf("%s%d\t ", out, score.ByteCount)
		}
		out = fmt.Sprintf("%s%s", out, score.FileName)
	}
	w := new(tabwriter.Writer)

	w.Init(os.Stdout, 8, 8, 0, ' ', tabwriter.AlignRight)
	fmt.Fprintln(w, out)
	w.Flush()
}

func main() {
	flag.Parse()
	fileCount := len(flag.Args())
	var score *Score
	if fileCount == 1 {
		score = count_file(flag.Args()[0])
	} else if fileCount > 1 {
		score = &Score{WordCount: 0, LineCount: 0, ByteCount: 0, FileName: "total"}
		for i := 0; i < fileCount; i++ {
			s := count_file(flag.Args()[i])
			score.Add(s)
			Printout(s)
		}
	} else {
		bytes, _ := ioutil.ReadAll(os.Stdin)
		score = count(bytes)
	}
	Printout(score)
}
<|endoftext|>


<commit_before>// weedo.go
package weedo

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"mime/multipart"
	"net/http"
	"net/textproto"
	"net/url"
	"strconv"
	"strings"
)

var defaultClient *Client

func init() {
	log.SetFlags(log.LstdFlags | log.Lshortfile)
	defaultClient = NewClient("localhost:9333")
}

type Fid struct {
	Id, Key, Cookie uint64
}

type Client struct {
	master  *Master
	volumes map[uint64]*Volume
	filers  map[string]*Filer
}

func NewClient(masterUrl string, filerUrls ...string) *Client {
	filers := make(map[string]*Filer)
	for _, url := range filerUrls {
		filer := NewFiler(url)
		filers[filer.Url] = filer
	}
	return &Client{
		master:  NewMaster(masterUrl),
		volumes: make(map[uint64]*Volume),
		filers:  filers,
	}
}

func (c *Client) Master() *Master {
	return c.master
}

func (c *Client) Volume(volumeId, collection string) (*Volume, error) {
	vid, _ := strconv.ParseUint(volumeId, 10, 32)
	if vid == 0 {
		fid, _ := ParseFid(volumeId)
		vid = fid.Id
	}

	if vid == 0 {
		return nil, errors.New("id malformed")
	}

	if v, ok := c.volumes[vid]; ok {
		return v, nil
	}
	vol, err := c.Master().lookup(volumeId, collection)
	if err != nil {
		return nil, err
	}

	c.volumes[vid] = vol

	return vol, nil
}

func (c *Client) Filer(url string) *Filer {
	filer := NewFiler(url)
	if v, ok := c.filers[filer.Url]; ok {
		return v
	}

	c.filers[filer.Url] = filer
	return filer
}

func ParseFid(s string) (fid Fid, err error) {
	a := strings.Split(s, ",")
	if len(a) != 2 || len(a[1]) <= 8 {
		return fid, errors.New("Fid format invalid")
	}
	if fid.Id, err = strconv.ParseUint(a[0], 10, 32); err != nil {
		return
	}
	index := len(a[1]) - 8
	if fid.Key, err = strconv.ParseUint(a[1][:index], 16, 64); err != nil {
		return
	}
	if fid.Cookie, err = strconv.ParseUint(a[1][index:], 16, 32); err != nil {
		return
	}

	return
}

// First, contact with master server and assign a fid, then upload to volume server
// It is same as the follow steps
// curl http://localhost:9333/dir/assign
// curl -F file=@example.jpg http://127.0.0.1:8080/3,01637037d6
func AssignUpload(filename, mimeType string, file io.Reader) (fid string, size int64, err error) {
	return AssignUploadArgs(filename, mimeType, file, url.Values{})
}

func AssignUploadArgs(filename, mimeType string, file io.Reader, args url.Values) (fid string, size int64, err error) {
	return defaultClient.AssignUploadArgs(filename, mimeType, file, args)
}

func Delete(fid string, count int) (err error) {
	return defaultClient.Delete(fid, count)
}

func (c *Client) GetUrl(fid string, collection ...string) (publicUrl, url string, err error) {
	col := ""
	if len(collection) > 0 {
		col = collection[0]
	}
	vol, err := c.Volume(fid, col)
	if err != nil {
		return
	}

	publicUrl = fmt.Sprintf("%s/%s", vol.PublicUrl(), fid)
	url = fmt.Sprintf("%s/%s", vol.Url(), fid)
	return
}

func (c *Client) GetUrls(fid string, collection ...string) (locations []Location, err error) {
	col := ""
	if len(collection) > 0 {
		col = collection[0]
	}
	vol, err := c.Volume(fid, col)
	if err != nil {
		return
	}
	for _, loc := range vol.Locations {
		loc.PublicUrl = fmt.Sprintf("%s/%s", loc.PublicUrl, fid)
		loc.Url = fmt.Sprintf("%s/%s", loc.Url, fid)
		locations = append(locations, loc)
	}
	return
}

func (c *Client) AssignUpload(filename, mimeType string, file io.Reader) (fid string, size int64, err error) {
	return c.AssignUploadArgs(filename, mimeType, file, url.Values{})
}

func (c *Client) AssignUploadArgs(filename, mimeType string, file io.Reader, args url.Values) (fid string, size int64, err error) {
	fid, err = c.Master().AssignArgs(args)
	if err != nil {
		return
	}

	vol, err := c.Volume(fid, args.Get("collection"))
	if err != nil {
		return
	}
	size, err = vol.Upload(fid, 0, filename, mimeType, file)

	return
}

func (c *Client) Delete(fid string, count int, collection ...string) (err error) {
	col := ""
	if len(collection) > 0 {
		col = collection[0]
	}
	vol, err := c.Volume(fid, col)
	if err != nil {
		return
	}
	return vol.Delete(fid, count)
}

var quoteEscaper = strings.NewReplacer("\\", "\\\\", `"`, "\\\"")

func escapeQuotes(s string) string {
	return quoteEscaper.Replace(s)
}

func createFormFile(writer *multipart.Writer, fieldname, filename, mime string) (io.Writer, error) {
	h := make(textproto.MIMEHeader)
	h.Set("Content-Disposition",
		fmt.Sprintf(`form-data; name="%s"; filename="%s"`,
			escapeQuotes(fieldname), escapeQuotes(filename)))
	if len(mime) == 0 {
		mime = "application/octet-stream"
	}
	h.Set("Content-Type", mime)
	return writer.CreatePart(h)
}

func makeFormData(filename, mimeType string, content io.Reader) (formData io.Reader, contentType string, err error) {
	buf := new(bytes.Buffer)
	writer := multipart.NewWriter(buf)

	part, err := createFormFile(writer, "file", filename, mimeType)
	//log.Println(filename, mimeType)
	if err != nil {
		log.Println(err)
		return
	}
	_, err = io.Copy(part, content)
	if err != nil {
		log.Println(err)
		return
	}

	formData = buf
	contentType = writer.FormDataContentType()
	//log.Println(contentType)
	writer.Close()

	return
}

type uploadResp struct {
	Fid      string
	FileName string
	FileUrl  string
	Size     int64
	Error    string
}

func upload(url string, contentType string, formData io.Reader) (r *uploadResp, err error) {
	resp, err := http.Post(url, contentType, formData)
	if err != nil {
		log.Println(err)
		return
	}
	defer resp.Body.Close()

	upload := new(uploadResp)
	if err = decodeJson(resp.Body, upload); err != nil {
		return
	}

	if upload.Error != "" {
		err = errors.New(upload.Error)
		return
	}

	r = upload

	return
}

func del(url string) error {
	client := http.Client{}
	request, err := http.NewRequest("DELETE", url, nil)
	if err != nil {
		return err
	}
	resp, err := client.Do(request)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		txt, _ := ioutil.ReadAll(resp.Body)
		return errors.New(string(txt))
	}
	return err
}

func decodeJson(r io.Reader, v interface{}) error {
	return json.NewDecoder(r).Decode(v)
}
<commit_msg>fix #6<commit_after>// weedo.go
package weedo

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"mime/multipart"
	"net/http"
	"net/textproto"
	"net/url"
	"strconv"
	"strings"
)

var defaultClient *Client

func init() {
	log.SetFlags(log.LstdFlags | log.Lshortfile)
	defaultClient = NewClient("localhost:9333")
}

type Fid struct {
	Id, Key, Cookie uint64
}

type Client struct {
	master  *Master
	volumes map[uint64]*Volume
	filers  map[string]*Filer
}

func NewClient(masterUrl string, filerUrls ...string) *Client {
	filers := make(map[string]*Filer)
	for _, url := range filerUrls {
		filer := NewFiler(url)
		filers[filer.Url] = filer
	}
	return &Client{
		master:  NewMaster(masterUrl),
		volumes: make(map[uint64]*Volume),
		filers:  filers,
	}
}

func (c *Client) Master() *Master {
	return c.master
}

func (c *Client) Volume(volumeId, collection string) (*Volume, error) {
	vid, _ := strconv.ParseUint(volumeId, 10, 32)
	if vid == 0 {
		fid, _ := ParseFid(volumeId)
		vid = fid.Id
	}

	if vid == 0 {
		return nil, errors.New("id malformed")
	}

	if v, ok := c.volumes[vid]; ok {
		return v, nil
	}
	vol, err := c.Master().lookup(volumeId, collection)
	if err != nil {
		return nil, err
	}

	c.volumes[vid] = vol

	return vol, nil
}

func (c *Client) Filer(url string) *Filer {
	filer := NewFiler(url)
	if v, ok := c.filers[filer.Url]; ok {
		return v
	}

	c.filers[filer.Url] = filer
	return filer
}

func ParseFid(s string) (fid Fid, err error) {
	a := strings.Split(s, ",")
	if len(a) != 2 || len(a[1]) <= 8 {
		return fid, errors.New("Fid format invalid")
	}
	if fid.Id, err = strconv.ParseUint(a[0], 10, 32); err != nil {
		return
	}
	index := len(a[1]) - 8
	if fid.Key, err = strconv.ParseUint(a[1][:index], 16, 64); err != nil {
		return
	}
	if fid.Cookie, err = strconv.ParseUint(a[1][index:], 16, 32); err != nil {
		return
	}

	return
}

// First, contact with master server and assign a fid, then upload to volume server
// It is same as the follow steps
// curl http://localhost:9333/dir/assign
// curl -F file=@example.jpg http://127.0.0.1:8080/3,01637037d6
func AssignUpload(filename, mimeType string, file io.Reader) (fid string, size int64, err error) {
	return AssignUploadArgs(filename, mimeType, file, url.Values{})
}

func AssignUploadArgs(filename, mimeType string, file io.Reader, args url.Values) (fid string, size int64, err error) {
	return defaultClient.AssignUploadArgs(filename, mimeType, file, args)
}

func Delete(fid string, count int) (err error) {
	return defaultClient.Delete(fid, count)
}

func (c *Client) GetUrl(fid string, collection ...string) (publicUrl, url string, err error) {
	col := ""
	if len(collection) > 0 {
		col = collection[0]
	}
	vol, err := c.Volume(fid, col)
	if err != nil {
		return
	}

	publicUrl = fmt.Sprintf("%s/%s", vol.PublicUrl(), fid)
	url = fmt.Sprintf("%s/%s", vol.Url(), fid)
	return
}

func (c *Client) GetUrls(fid string, collection ...string) (locations []Location, err error) {
	col := ""
	if len(collection) > 0 {
		col = collection[0]
	}
	vol, err := c.Volume(fid, col)
	if err != nil {
		return
	}
	for _, loc := range vol.Locations {
		loc.PublicUrl = fmt.Sprintf("%s/%s", loc.PublicUrl, fid)
		loc.Url = fmt.Sprintf("%s/%s", loc.Url, fid)
		locations = append(locations, loc)
	}
	return
}

func (c *Client) AssignUpload(filename, mimeType string, file io.Reader) (fid string, size int64, err error) {
	return c.AssignUploadArgs(filename, mimeType, file, url.Values{})
}

func (c *Client) AssignUploadArgs(filename, mimeType string, file io.Reader, args url.Values) (fid string, size int64, err error) {
	fid, err = c.Master().AssignArgs(args)
	if err != nil {
		return
	}

	vol, err := c.Volume(fid, args.Get("collection"))
	if err != nil {
		return
	}
	size, err = vol.Upload(fid, 0, filename, mimeType, file)

	return
}

func (c *Client) Delete(fid string, count int, collection ...string) (err error) {
	col := ""
	if len(collection) > 0 {
		col = collection[0]
	}
	vol, err := c.Volume(fid, col)
	if err != nil {
		return
	}
	return vol.Delete(fid, count)
}

var quoteEscaper = strings.NewReplacer("\\", "\\\\", `"`, "\\\"")

func escapeQuotes(s string) string {
	return quoteEscaper.Replace(s)
}

func createFormFile(writer *multipart.Writer, fieldname, filename, mime string) (io.Writer, error) {
	h := make(textproto.MIMEHeader)
	h.Set("Content-Disposition",
		fmt.Sprintf(`form-data; name="%s"; filename="%s"`,
			escapeQuotes(fieldname), escapeQuotes(filename)))
	if len(mime) == 0 {
		mime = "application/octet-stream"
	}
	h.Set("Content-Type", mime)
	return writer.CreatePart(h)
}

func makeFormData(filename, mimeType string, content io.Reader) (formData io.Reader, contentType string, err error) {
	buf := new(bytes.Buffer)
	writer := multipart.NewWriter(buf)

	part, err := createFormFile(writer, "file", filename, mimeType)
	//log.Println(filename, mimeType)
	if err != nil {
		log.Println(err)
		return
	}
	_, err = io.Copy(part, content)
	if err != nil {
		log.Println(err)
		return
	}

	formData = buf
	contentType = writer.FormDataContentType()
	//log.Println(contentType)
	writer.Close()

	return
}

type uploadResp struct {
	Fid      string
	FileName string
	FileUrl  string
	Size     int64
	Error    string
}

func upload(url string, contentType string, formData io.Reader) (r *uploadResp, err error) {
	resp, err := http.Post(url, contentType, formData)
	if err != nil {
		log.Println(err)
		return
	}
	defer resp.Body.Close()

	upload := new(uploadResp)
	if err = decodeJson(resp.Body, upload); err != nil {
		return
	}

	if upload.Error != "" {
		err = errors.New(upload.Error)
		return
	}

	r = upload

	return
}

func del(url string) error {
	client := http.Client{}
	request, err := http.NewRequest("DELETE", url, nil)
	if err != nil {
		return err
	}
	resp, err := client.Do(request)
	if err != nil {
		return err
	}
	resp.Body.Close()

	if resp.StatusCode != http.StatusOK &&
		resp.StatusCode != http.StatusAccepted {
		txt, _ := ioutil.ReadAll(resp.Body)
		return errors.New(string(txt))
	}
	return nil
}

func decodeJson(r io.Reader, v interface{}) error {
	return json.NewDecoder(r).Decode(v)
}
<|endoftext|>


<commit_before>package epub

import (
	"archive/zip"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
)

const (
	containerFilename     = "container.xml"
	containerFileTemplate = `<?xml version="1.0" encoding="UTF-8"?>
<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">
  <rootfiles>
    <rootfile full-path="%s/%s" media-type="application/oebps-package+xml" />
  </rootfiles>
</container>
`
	contentFolderName = "EPUB"
	// Permissions for any new directories we create
	dirPermissions = 0755
	// Permissions for any new files we create
	filePermissions   = 0644
	imageFolderName   = "img"
	mediaTypeNcx      = "application/x-dtbncx+xml"
	mediaTypeEpub     = "application/epub+zip"
	mediaTypeXhtml    = "application/xhtml+xml"
	metaInfFolderName = "META-INF"
	mimetypeFilename  = "mimetype"
	pkgFilename       = "package.opf"
	sectionFileFormat = "section%04d.xhtml"
	tempDirPrefix     = "go-epub"
	xhtmlFolderName   = "xhtml"
)

// Write writes the EPUB file. The destination path must be the full path to
// the resulting file, including filename and extension.
func (e *Epub) Write(destFilePath string) error {
	tempDir, err := ioutil.TempDir("", tempDirPrefix)
	defer os.Remove(tempDir)
	if err != nil {
		log.Fatalf("os.Remove error: %s", err)
	}

	// Must be called first
	err = createEpubFolders(tempDir)
	if err != nil {
		return err
	}

	// Must be called after:
	// createEpubFolders()
	err = e.writeImages(tempDir)
	if err != nil {
		return err
	}

	// Must be called after:
	// createEpubFolders()
	err = writeMimetype(tempDir)
	if err != nil {
		return err
	}

	// Must be called after:
	// createEpubFolders()
	err = e.writeSections(tempDir)
	if err != nil {
		return err
	}

	// Must be called after:
	// createEpubFolders()
	// writeSections()
	err = e.writeToc(tempDir)
	if err != nil {
		return err
	}

	// Must be called after:
	// createEpubFolders()
	err = writeContainerFile(tempDir)
	if err != nil {
		return err
	}

	// Must be called after:
	// createEpubFolders()
	// writeImages()
	// writeSections()
	// writeToc()
	err = e.writePackageFile(tempDir)
	if err != nil {
		return err
	}

	// Must be called last
	err = e.writeEpub(tempDir, destFilePath)
	if err != nil {
		return err
	}

	// TODO

	//	output, err := xml.MarshalIndent(e.toc.navDoc.xml, "", "  ")
	//	output = append([]byte(xhtmlDoctype), output...)

	//	output, err := xml.MarshalIndent(e.pkg.xml, "", "  ")

	//  output, err := xml.MarshalIndent(e.toc.ncxXML, "", "  ")
	//	output = append([]byte(xml.Header), output...)
	//	fmt.Println(string(output))

	err = os.RemoveAll(tempDir)
	if err != nil {
		return err
	}

	return nil
}

// Create the EPUB folder structure in a temp directory
func createEpubFolders(tempDir string) error {
	if err := os.Mkdir(
		filepath.Join(
			tempDir,
			contentFolderName,
		),
		dirPermissions); err != nil {
		return err
	}

	if err := os.Mkdir(
		filepath.Join(
			tempDir,
			contentFolderName,
			xhtmlFolderName,
		),
		dirPermissions); err != nil {
		return err
	}

	if err := os.Mkdir(
		filepath.Join(
			tempDir,
			metaInfFolderName,
		),
		dirPermissions); err != nil {
		return err
	}

	return nil
}

// Write the contatiner file (container.xml), which mostly just points to the
// package file (package.opf)
//
// Sample: https://github.com/bmaupin/epub-samples/blob/master/minimal-v32/META-INF/container.xml
// Spec: http://www.idpf.org/epub/301/spec/epub-ocf.html#sec-container-metainf-container.xml
func writeContainerFile(tempDir string) error {
	containerFilePath := filepath.Join(tempDir, metaInfFolderName, containerFilename)
	if err := ioutil.WriteFile(
		containerFilePath,
		[]byte(
			fmt.Sprintf(
				containerFileTemplate,
				contentFolderName,
				pkgFilename,
			),
		),
		filePermissions,
	); err != nil {
		return err
	}

	return nil
}

// Write the EPUB file itself by zipping up everything from a temp directory
func (e *Epub) writeEpub(tempDir string, destFilePath string) error {
	f, err := os.Create(destFilePath)
	if err != nil {
		log.Fatalf("os.Create error: %s", err)
	}
	defer func() {
		if err := f.Close(); err != nil {
			log.Fatalf("os.File.Close error: %s", err)
		}
	}()

	z := zip.NewWriter(f)
	defer func() {
		if err := z.Close(); err != nil {
			log.Fatalf("zip.Writer.Close error: %s", err)
		}
	}()

	skipMimetypeFile := false

	var addFileToZip = func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		// Get the path of the file relative to the folder we're zipping
		relativePath, err := filepath.Rel(tempDir, path)
		if err != nil {
			log.Fatalf("filepath.Rel error: %s", err)
		}

		// Only include regular files, not directories
		if !info.Mode().IsRegular() {
			return nil
		}

		var w io.Writer
		if path == filepath.Join(tempDir, mimetypeFilename) {
			// Skip the mimetype file if it's already been written
			if skipMimetypeFile == true {
				return nil
			}
			// The mimetype file must be uncompressed according to the EPUB spec
			w, err = z.CreateHeader(&zip.FileHeader{
				Name:   relativePath,
				Method: zip.Store,
			})
		} else {
			w, err = z.Create(relativePath)
		}
		if err != nil {
			log.Fatalf("zip.Writer.Create error: %s", err)
		}

		r, err := os.Open(path)
		if err != nil {
			log.Fatalf("os.Open error: %s", err)
		}
		defer func() {
			if err := r.Close(); err != nil {
				log.Fatalf("os.File.Close error: %s", err)
			}
		}()

		_, err = io.Copy(w, r)
		if err != nil {
			log.Fatalf("io.Copy error: %s", err)
		}

		return nil
	}

	// Add the mimetype file first
	mimetypeFilePath := filepath.Join(tempDir, mimetypeFilename)
	mimetypeInfo, err := os.Lstat(mimetypeFilePath)
	if err != nil {
		log.Fatalf("os.Lstat error: %s", err)
	}
	addFileToZip(mimetypeFilePath, mimetypeInfo, nil)

	skipMimetypeFile = true

	err = filepath.Walk(tempDir, addFileToZip)
	if err != nil {
		log.Fatalf("os.Lstat error: %s", err)
	}

	return nil
}

// Get images from their source and save them in the temporary directory
func (e *Epub) writeImages(tempDir string) error {
	imageFolderPath := filepath.Join(tempDir, contentFolderName, imageFolderName)
	if err := os.Mkdir(imageFolderPath, dirPermissions); err != nil {
		return err
	}

	for imageFilename, imageSource := range e.images {
		u, err := url.Parse(imageSource)
		if err != nil {
			return err
		}

		var r io.ReadCloser
		var resp *http.Response
		// If it's a URL
		if u.Scheme == "http" || u.Scheme == "https" {
			resp, err = http.Get(imageSource)
			r = resp.Body

			// Otherwise, assume it's a local file
		} else {
			r, err = os.Open(imageSource)
		}
		if err != nil {
			return err
		}
		defer func() {
			err = r.Close()
		}()

		imageFilePath := filepath.Join(
			imageFolderPath,
			imageFilename,
		)

		w, err := os.Create(imageFilePath)
		if err != nil {
			return err
		}
		defer func() {
			err = w.Close()
		}()

		_, err = io.Copy(w, r)
		if err != nil {
			return err
		}
	}

	return nil
}

// Write the mimetype file
//
// Sample: https://github.com/bmaupin/epub-samples/blob/master/minimal-v32/mimetype
// Spec: http://www.idpf.org/epub/301/spec/epub-ocf.html#sec-zip-container-mime
func writeMimetype(tempDir string) error {
	mimetypeFilePath := filepath.Join(tempDir, mimetypeFilename)

	if err := ioutil.WriteFile(mimetypeFilePath, []byte(mediaTypeEpub), filePermissions); err != nil {
		return err
	}

	return nil
}

func (e *Epub) writePackageFile(tempDir string) error {
	err := e.pkg.write(tempDir)
	if err != nil {
		return err
	}

	return nil
}

// Write the section files to the temporary directory and add the sections to
// the TOC and package files
func (e *Epub) writeSections(tempDir string) error {
	for i, section := range e.sections {
		sectionIndex := i + 1
		sectionFilename := fmt.Sprintf(sectionFileFormat, sectionIndex)
		sectionFilePath := filepath.Join(tempDir, contentFolderName, xhtmlFolderName, sectionFilename)
		section.write(sectionFilePath)

		relativePath := filepath.Join(xhtmlFolderName, sectionFilename)
		e.toc.addSection(sectionIndex, section.Title(), relativePath)
		e.pkg.addToManifest(sectionFilename, relativePath, mediaTypeXhtml, "")
		e.pkg.addToSpine(sectionFilename)
	}

	return nil
}

// Write the TOC file to the temporary directory and add the TOC entries to the
// package file
func (e *Epub) writeToc(tempDir string) error {
	e.pkg.addToManifest(tocNavItemID, tocNavFilename, mediaTypeXhtml, tocNavItemProperties)
	e.pkg.addToManifest(tocNcxItemID, tocNcxFilename, mediaTypeNcx, "")

	err := e.toc.write(tempDir)
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>Add images to the package manifest<commit_after>package epub

import (
	"archive/zip"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
)

const (
	containerFilename     = "container.xml"
	containerFileTemplate = `<?xml version="1.0" encoding="UTF-8"?>
<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">
  <rootfiles>
    <rootfile full-path="%s/%s" media-type="application/oebps-package+xml" />
  </rootfiles>
</container>
`
	contentFolderName = "EPUB"
	// Permissions for any new directories we create
	dirPermissions = 0755
	// Permissions for any new files we create
	filePermissions   = 0644
	imageFolderName   = "img"
	mediaTypeNcx      = "application/x-dtbncx+xml"
	mediaTypeEpub     = "application/epub+zip"
	mediaTypeGif      = "image/gif"
	mediaTypeJpeg     = "image/jpeg"
	mediaTypePng      = "image/png"
	mediaTypeSvg      = "image/svg+xml"
	mediaTypeXhtml    = "application/xhtml+xml"
	metaInfFolderName = "META-INF"
	mimetypeFilename  = "mimetype"
	pkgFilename       = "package.opf"
	sectionFileFormat = "section%04d.xhtml"
	tempDirPrefix     = "go-epub"
	xhtmlFolderName   = "xhtml"
)

// Write writes the EPUB file. The destination path must be the full path to
// the resulting file, including filename and extension.
func (e *Epub) Write(destFilePath string) error {
	tempDir, err := ioutil.TempDir("", tempDirPrefix)
	defer os.Remove(tempDir)
	if err != nil {
		log.Fatalf("os.Remove error: %s", err)
	}

	// Must be called first
	err = createEpubFolders(tempDir)
	if err != nil {
		return err
	}

	// Must be called after:
	// createEpubFolders()
	err = e.writeImages(tempDir)
	if err != nil {
		return err
	}

	// Must be called after:
	// createEpubFolders()
	err = writeMimetype(tempDir)
	if err != nil {
		return err
	}

	// Must be called after:
	// createEpubFolders()
	err = e.writeSections(tempDir)
	if err != nil {
		return err
	}

	// Must be called after:
	// createEpubFolders()
	// writeSections()
	err = e.writeToc(tempDir)
	if err != nil {
		return err
	}

	// Must be called after:
	// createEpubFolders()
	err = writeContainerFile(tempDir)
	if err != nil {
		return err
	}

	// Must be called after:
	// createEpubFolders()
	// writeImages()
	// writeSections()
	// writeToc()
	err = e.writePackageFile(tempDir)
	if err != nil {
		return err
	}

	// Must be called last
	err = e.writeEpub(tempDir, destFilePath)
	if err != nil {
		return err
	}

	// TODO

	//	output, err := xml.MarshalIndent(e.toc.navDoc.xml, "", "  ")
	//	output = append([]byte(xhtmlDoctype), output...)

	//	output, err := xml.MarshalIndent(e.pkg.xml, "", "  ")

	//  output, err := xml.MarshalIndent(e.toc.ncxXML, "", "  ")
	//	output = append([]byte(xml.Header), output...)
	//	fmt.Println(string(output))

	err = os.RemoveAll(tempDir)
	if err != nil {
		return err
	}

	return nil
}

// Create the EPUB folder structure in a temp directory
func createEpubFolders(tempDir string) error {
	if err := os.Mkdir(
		filepath.Join(
			tempDir,
			contentFolderName,
		),
		dirPermissions); err != nil {
		return err
	}

	if err := os.Mkdir(
		filepath.Join(
			tempDir,
			contentFolderName,
			xhtmlFolderName,
		),
		dirPermissions); err != nil {
		return err
	}

	if err := os.Mkdir(
		filepath.Join(
			tempDir,
			metaInfFolderName,
		),
		dirPermissions); err != nil {
		return err
	}

	return nil
}

// Return the relative path to the image within the EPUB file
func getImageRelPath(imageFilename string) string {
	return filepath.Join(
		"..",
		imageFolderName,
		imageFilename,
	)
}

// Write the contatiner file (container.xml), which mostly just points to the
// package file (package.opf)
//
// Sample: https://github.com/bmaupin/epub-samples/blob/master/minimal-v32/META-INF/container.xml
// Spec: http://www.idpf.org/epub/301/spec/epub-ocf.html#sec-container-metainf-container.xml
func writeContainerFile(tempDir string) error {
	containerFilePath := filepath.Join(tempDir, metaInfFolderName, containerFilename)
	if err := ioutil.WriteFile(
		containerFilePath,
		[]byte(
			fmt.Sprintf(
				containerFileTemplate,
				contentFolderName,
				pkgFilename,
			),
		),
		filePermissions,
	); err != nil {
		return err
	}

	return nil
}

// Write the EPUB file itself by zipping up everything from a temp directory
func (e *Epub) writeEpub(tempDir string, destFilePath string) error {
	f, err := os.Create(destFilePath)
	if err != nil {
		log.Fatalf("os.Create error: %s", err)
	}
	defer func() {
		if err := f.Close(); err != nil {
			log.Fatalf("os.File.Close error: %s", err)
		}
	}()

	z := zip.NewWriter(f)
	defer func() {
		if err := z.Close(); err != nil {
			log.Fatalf("zip.Writer.Close error: %s", err)
		}
	}()

	skipMimetypeFile := false

	var addFileToZip = func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		// Get the path of the file relative to the folder we're zipping
		relativePath, err := filepath.Rel(tempDir, path)
		if err != nil {
			log.Fatalf("filepath.Rel error: %s", err)
		}

		// Only include regular files, not directories
		if !info.Mode().IsRegular() {
			return nil
		}

		var w io.Writer
		if path == filepath.Join(tempDir, mimetypeFilename) {
			// Skip the mimetype file if it's already been written
			if skipMimetypeFile == true {
				return nil
			}
			// The mimetype file must be uncompressed according to the EPUB spec
			w, err = z.CreateHeader(&zip.FileHeader{
				Name:   relativePath,
				Method: zip.Store,
			})
		} else {
			w, err = z.Create(relativePath)
		}
		if err != nil {
			log.Fatalf("zip.Writer.Create error: %s", err)
		}

		r, err := os.Open(path)
		if err != nil {
			log.Fatalf("os.Open error: %s", err)
		}
		defer func() {
			if err := r.Close(); err != nil {
				log.Fatalf("os.File.Close error: %s", err)
			}
		}()

		_, err = io.Copy(w, r)
		if err != nil {
			log.Fatalf("io.Copy error: %s", err)
		}

		return nil
	}

	// Add the mimetype file first
	mimetypeFilePath := filepath.Join(tempDir, mimetypeFilename)
	mimetypeInfo, err := os.Lstat(mimetypeFilePath)
	if err != nil {
		log.Fatalf("os.Lstat error: %s", err)
	}
	addFileToZip(mimetypeFilePath, mimetypeInfo, nil)

	skipMimetypeFile = true

	err = filepath.Walk(tempDir, addFileToZip)
	if err != nil {
		log.Fatalf("os.Lstat error: %s", err)
	}

	return nil
}

// Get images from their source and save them in the temporary directory
func (e *Epub) writeImages(tempDir string) error {
	imageFolderPath := filepath.Join(tempDir, contentFolderName, imageFolderName)
	if err := os.Mkdir(imageFolderPath, dirPermissions); err != nil {
		return err
	}

	for imageFilename, imageSource := range e.images {
		// Get the image from the source
		u, err := url.Parse(imageSource)
		if err != nil {
			return err
		}

		var r io.ReadCloser
		var resp *http.Response
		// If it's a URL
		if u.Scheme == "http" || u.Scheme == "https" {
			resp, err = http.Get(imageSource)
			r = resp.Body

			// Otherwise, assume it's a local file
		} else {
			r, err = os.Open(imageSource)
		}
		if err != nil {
			return err
		}
		defer func() {
			err = r.Close()
		}()

		imageFilePath := filepath.Join(
			imageFolderPath,
			imageFilename,
		)

		// Add the image to the EPUB temp directory
		w, err := os.Create(imageFilePath)
		if err != nil {
			return err
		}
		defer func() {
			err = w.Close()
		}()

		_, err = io.Copy(w, r)
		if err != nil {
			return err
		}

		// Determine the media type
		imageMediaType := ""
		if filepath.Ext(imageFilename) == ".gif" {
			imageMediaType = mediaTypeGif
		} else if filepath.Ext(imageFilename) == ".jpg" || filepath.Ext(imageFilename) == ".jpeg" {
			imageMediaType = mediaTypeJpeg
		} else if filepath.Ext(imageFilename) == ".png" {
			imageMediaType = mediaTypePng
		} else if filepath.Ext(imageFilename) == ".svg" {
			imageMediaType = mediaTypeSvg
		}

		// Add the image to the package file manifest
		e.pkg.addToManifest(imageFilename, filepath.Join(imageFolderName, imageFilename), imageMediaType, "")
	}

	return nil
}

// Write the mimetype file
//
// Sample: https://github.com/bmaupin/epub-samples/blob/master/minimal-v32/mimetype
// Spec: http://www.idpf.org/epub/301/spec/epub-ocf.html#sec-zip-container-mime
func writeMimetype(tempDir string) error {
	mimetypeFilePath := filepath.Join(tempDir, mimetypeFilename)

	if err := ioutil.WriteFile(mimetypeFilePath, []byte(mediaTypeEpub), filePermissions); err != nil {
		return err
	}

	return nil
}

func (e *Epub) writePackageFile(tempDir string) error {
	err := e.pkg.write(tempDir)
	if err != nil {
		return err
	}

	return nil
}

// Write the section files to the temporary directory and add the sections to
// the TOC and package files
func (e *Epub) writeSections(tempDir string) error {
	for i, section := range e.sections {
		sectionIndex := i + 1
		sectionFilename := fmt.Sprintf(sectionFileFormat, sectionIndex)
		sectionFilePath := filepath.Join(tempDir, contentFolderName, xhtmlFolderName, sectionFilename)
		section.write(sectionFilePath)

		relativePath := filepath.Join(xhtmlFolderName, sectionFilename)
		e.toc.addSection(sectionIndex, section.Title(), relativePath)
		e.pkg.addToManifest(sectionFilename, relativePath, mediaTypeXhtml, "")
		e.pkg.addToSpine(sectionFilename)
	}

	return nil
}

// Write the TOC file to the temporary directory and add the TOC entries to the
// package file
func (e *Epub) writeToc(tempDir string) error {
	e.pkg.addToManifest(tocNavItemID, tocNavFilename, mediaTypeXhtml, tocNavItemProperties)
	e.pkg.addToManifest(tocNcxItemID, tocNcxFilename, mediaTypeNcx, "")

	err := e.toc.write(tempDir)
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2015, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

// Package xurls extracts urls from plain text using regular expressions.
package xurls

import "regexp"

//go:generate go run generate/tldsgen/main.go
//go:generate go run generate/regexgen/main.go

const (
	letter    = `\p{L}`
	number    = `\p{N}`
	iriChar   = letter + number
	pathChar  = iriChar + `/\-+_&~*%=#@.,:;'?!|` + `\p{Sc}\p{Sk}\p{So}`
	endChar   = iriChar + `/\-+_&~*%=#` + `\p{Sc}`
	wellParen = `\([` + pathChar + `]*(\([` + pathChar + `]*\)[` + pathChar + `]*)*\)`
	wellBrack = `\[[` + pathChar + `]*(\[[` + pathChar + `]*\][` + pathChar + `]*)*\]`
	wellAll   = wellParen + `|` + wellBrack
	pathCont  = `([` + pathChar + `]*(` + wellAll + `|[` + endChar + `])+)+`
	comScheme = `[a-zA-Z][a-zA-Z.\-+]*://`
	scheme    = `(` + comScheme + `|` + otherScheme + `)`

	iri      = `[` + iriChar + `]([` + iriChar + `\-]*[` + iriChar + `])?`
	domain   = `(` + iri + `\.)+`
	octet    = `(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9][0-9]|[0-9])`
	ipv4Addr = `\b` + octet + `\.` + octet + `\.` + octet + `\.` + octet + `\b`
	ipv6Addr = `([0-9a-fA-F]{1,4}:([0-9a-fA-F]{1,4}:([0-9a-fA-F]{1,4}:([0-9a-fA-F]{1,4}:([0-9a-fA-F]{1,4}:[0-9a-fA-F]{0,4}|:[0-9a-fA-F]{1,4})?|(:[0-9a-fA-F]{1,4}){0,2})|(:[0-9a-fA-F]{1,4}){0,3})|(:[0-9a-fA-F]{1,4}){0,4})|:(:[0-9a-fA-F]{1,4}){0,5})((:[0-9a-fA-F]{1,4}){2}|:(25[0-5]|(2[0-4]|1[0-9]|[1-9])?[0-9])(\.(25[0-5]|(2[0-4]|1[0-9]|[1-9])?[0-9])){3})|(([0-9a-fA-F]{1,4}:){1,6}|:):[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){7}:`
	ipAddr   = `(` + ipv4Addr + `|` + ipv6Addr + `)`
	hostName = `(` + domain + gtld + `|` + ipAddr + `)`
	port     = `(:[0-9]*)?`
	path     = `(/|/` + pathCont + `?|\b|$)`
	webURL   = hostName + port + path
	email    = `[a-zA-Z0-9._%\-+]+@` + hostName

	strict  = `(\b` + scheme + pathCont + `)`
	relaxed = `(` + strict + `|` + webURL + `|` + email + `)`
)

var (
	// Relaxed matches all the urls it can find.
	Relaxed = regexp.MustCompile(relaxed)
	// Strict only matches urls with a scheme to avoid false positives.
	Strict = regexp.MustCompile(strict)
)

func init() {
	Relaxed.Longest()
	Strict.Longest()
}

// StrictMatchingScheme produces a regexp that matches urls like Strict but
// whose scheme matches the given regular expression.
func StrictMatchingScheme(exp string) (*regexp.Regexp, error) {
	strictMatching := `(\b(?i)(` + exp + `)(?-i)` + pathCont + `)`
	re, err := regexp.Compile(strictMatching)
	if err != nil {
		return nil, err
	}
	re.Longest()
	return re, nil
}
<commit_msg>Be explicit about what each unicode category means<commit_after>// Copyright (c) 2015, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

// Package xurls extracts urls from plain text using regular expressions.
package xurls

import "regexp"

//go:generate go run generate/tldsgen/main.go
//go:generate go run generate/regexgen/main.go

const (
	letter    = `\p{L}`
	number    = `\p{N}`
	iriChar   = letter + number
	currency  = `\p{Sc}`
	modifier  = `\p{Sk}`
	otherSymb = `\p{So}`
	pathChar  = iriChar + `/\-+_&~*%=#@.,:;'?!|` + currency + modifier + otherSymb
	endChar   = iriChar + `/\-+_&~*%=#` + currency
	wellParen = `\([` + pathChar + `]*(\([` + pathChar + `]*\)[` + pathChar + `]*)*\)`
	wellBrack = `\[[` + pathChar + `]*(\[[` + pathChar + `]*\][` + pathChar + `]*)*\]`
	wellAll   = wellParen + `|` + wellBrack
	pathCont  = `([` + pathChar + `]*(` + wellAll + `|[` + endChar + `])+)+`
	comScheme = `[a-zA-Z][a-zA-Z.\-+]*://`
	scheme    = `(` + comScheme + `|` + otherScheme + `)`

	iri      = `[` + iriChar + `]([` + iriChar + `\-]*[` + iriChar + `])?`
	domain   = `(` + iri + `\.)+`
	octet    = `(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9][0-9]|[0-9])`
	ipv4Addr = `\b` + octet + `\.` + octet + `\.` + octet + `\.` + octet + `\b`
	ipv6Addr = `([0-9a-fA-F]{1,4}:([0-9a-fA-F]{1,4}:([0-9a-fA-F]{1,4}:([0-9a-fA-F]{1,4}:([0-9a-fA-F]{1,4}:[0-9a-fA-F]{0,4}|:[0-9a-fA-F]{1,4})?|(:[0-9a-fA-F]{1,4}){0,2})|(:[0-9a-fA-F]{1,4}){0,3})|(:[0-9a-fA-F]{1,4}){0,4})|:(:[0-9a-fA-F]{1,4}){0,5})((:[0-9a-fA-F]{1,4}){2}|:(25[0-5]|(2[0-4]|1[0-9]|[1-9])?[0-9])(\.(25[0-5]|(2[0-4]|1[0-9]|[1-9])?[0-9])){3})|(([0-9a-fA-F]{1,4}:){1,6}|:):[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){7}:`
	ipAddr   = `(` + ipv4Addr + `|` + ipv6Addr + `)`
	hostName = `(` + domain + gtld + `|` + ipAddr + `)`
	port     = `(:[0-9]*)?`
	path     = `(/|/` + pathCont + `?|\b|$)`
	webURL   = hostName + port + path
	email    = `[a-zA-Z0-9._%\-+]+@` + hostName

	strict  = `(\b` + scheme + pathCont + `)`
	relaxed = `(` + strict + `|` + webURL + `|` + email + `)`
)

var (
	// Relaxed matches all the urls it can find.
	Relaxed = regexp.MustCompile(relaxed)
	// Strict only matches urls with a scheme to avoid false positives.
	Strict = regexp.MustCompile(strict)
)

func init() {
	Relaxed.Longest()
	Strict.Longest()
}

// StrictMatchingScheme produces a regexp that matches urls like Strict but
// whose scheme matches the given regular expression.
func StrictMatchingScheme(exp string) (*regexp.Regexp, error) {
	strictMatching := `(\b(?i)(` + exp + `)(?-i)` + pathCont + `)`
	re, err := regexp.Compile(strictMatching)
	if err != nil {
		return nil, err
	}
	re.Longest()
	return re, nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2015, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

// Package xurls extracts urls from plain text using regular expressions.
package xurls

import (
	"regexp"
	"strings"
	"unicode/utf8"
)

//go:generate go run ./generate/tldsgen
//go:generate go run ./generate/schemesgen
//go:generate go run ./generate/unicodegen

const (
	letter    = `\p{L}`
	mark      = `\p{M}`
	number    = `\p{N}`
	iriChar   = letter + mark + number
	currency  = `\p{Sc}`
	otherSymb = `\p{So}`
	endChar   = iriChar + `/\-_+&~%=#` + currency + otherSymb
	midChar   = endChar + "_*" + otherPuncMinusDoubleQuote
	wellParen = `\([` + midChar + `]*(\([` + midChar + `]*\)[` + midChar + `]*)*\)`
	wellBrack = `\[[` + midChar + `]*(\[[` + midChar + `]*\][` + midChar + `]*)*\]`
	wellBrace = `\{[` + midChar + `]*(\{[` + midChar + `]*\}[` + midChar + `]*)*\}`
	wellAll   = wellParen + `|` + wellBrack + `|` + wellBrace
	pathCont  = `([` + midChar + `]*(` + wellAll + `|[` + endChar + `])+)+`

	iri      = `[` + iriChar + `]([` + iriChar + `\-]*[` + iriChar + `])?`
	domain   = `(` + iri + `\.)+`
	octet    = `(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9][0-9]|[0-9])`
	ipv4Addr = `\b` + octet + `\.` + octet + `\.` + octet + `\.` + octet + `\b`
	ipv6Addr = `([0-9a-fA-F]{1,4}:([0-9a-fA-F]{1,4}:([0-9a-fA-F]{1,4}:([0-9a-fA-F]{1,4}:([0-9a-fA-F]{1,4}:[0-9a-fA-F]{0,4}|:[0-9a-fA-F]{1,4})?|(:[0-9a-fA-F]{1,4}){0,2})|(:[0-9a-fA-F]{1,4}){0,3})|(:[0-9a-fA-F]{1,4}){0,4})|:(:[0-9a-fA-F]{1,4}){0,5})((:[0-9a-fA-F]{1,4}){2}|:(25[0-5]|(2[0-4]|1[0-9]|[1-9])?[0-9])(\.(25[0-5]|(2[0-4]|1[0-9]|[1-9])?[0-9])){3})|(([0-9a-fA-F]{1,4}:){1,6}|:):[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){7}:`
	ipAddr   = `(` + ipv4Addr + `|` + ipv6Addr + `)`
	port     = `(:[0-9]*)?`
)

// AnyScheme can be passed to StrictMatchingScheme to match any possibly valid
// scheme, and not just the known ones.
var AnyScheme = `([a-zA-Z][a-zA-Z.\-+]*://|` + anyOf(SchemesNoAuthority...) + `:)`

// SchemesNoAuthority is a sorted list of some well-known url schemes that are
// followed by ":" instead of "://". The list includes both officially
// registered and unofficial schemes.
var SchemesNoAuthority = []string{
	`bitcoin`, // Bitcoin
	`cid`,     // Content-ID
	`file`,    // Files
	`magnet`,  // Torrent magnets
	`mailto`,  // Mail
	`mid`,     // Message-ID
	`sms`,     // SMS
	`tel`,     // Telephone
	`xmpp`,    // XMPP
}

// SchemesUnofficial is a sorted list of some well-known url schemes which
// aren't officially registered just yet. They tend to correspond to software.
//
// Mostly collected from https://en.wikipedia.org/wiki/List_of_URI_schemes#Unofficial_but_common_URI_schemes.
var SchemesUnofficial = []string{
	`jdbc`,       // Java database Connectivity
	`postgres`,   // PostgreSQL (short form)
	`postgresql`, // PostgreSQL
	`slack`,      // Slack
	`zoommtg`,    // Zoom (desktop)
	`zoomus`,     // Zoom (mobile)
}

func anyOf(strs ...string) string {
	var b strings.Builder
	b.WriteByte('(')
	for i, s := range strs {
		if i != 0 {
			b.WriteByte('|')
		}
		b.WriteString(regexp.QuoteMeta(s))
	}
	b.WriteByte(')')
	return b.String()
}

func strictExp() string {
	schemes := `((` + anyOf(Schemes...) + `|` + anyOf(SchemesUnofficial...) + `)://|` + anyOf(SchemesNoAuthority...) + `:)`
	return `(?i)` + schemes + `(?-i)` + pathCont
}

func relaxedExp() string {
	var asciiTLDs, unicodeTLDs []string
	for i, tld := range TLDs {
		if tld[0] >= utf8.RuneSelf {
			asciiTLDs = TLDs[:i:i]
			unicodeTLDs = TLDs[i:]
			break
		}
	}
	punycode := `xn--[a-z0-9-]+`

	// Use \b to make sure ASCII TLDs are immediately followed by a word break.
	// We can't do that with unicode TLDs, as they don't see following
	// whitespace as a word break.
	tlds := `(?i)(` + punycode + `|` + anyOf(append(asciiTLDs, PseudoTLDs...)...) + `\b|` + anyOf(unicodeTLDs...) + `)(?-i)`
	site := domain + tlds

	hostName := `(` + site + `|` + ipAddr + `)`
	webURL := hostName + port + `(/|/` + pathCont + `)?`
	email := `[a-zA-Z0-9._%\-+]+@` + site
	return strictExp() + `|` + webURL + `|` + email
}

// Strict produces a regexp that matches any URL with a scheme in either the
// Schemes or SchemesNoAuthority lists.
func Strict() *regexp.Regexp {
	re := regexp.MustCompile(strictExp())
	re.Longest()
	return re
}

// Relaxed produces a regexp that matches any URL matched by Strict, plus any
// URL with no scheme or email address.
func Relaxed() *regexp.Regexp {
	re := regexp.MustCompile(relaxedExp())
	re.Longest()
	return re
}

// StrictMatchingScheme produces a regexp similar to Strict, but requiring that
// the scheme match the given regular expression. See AnyScheme too.
func StrictMatchingScheme(exp string) (*regexp.Regexp, error) {
	strictMatching := `(?i)(` + exp + `)(?-i)` + pathCont
	re, err := regexp.Compile(strictMatching)
	if err != nil {
		return nil, err
	}
	re.Longest()
	return re, nil
}
<commit_msg>Update schemes<commit_after>// Copyright (c) 2015, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

// Package xurls extracts urls from plain text using regular expressions.
package xurls

import (
	"regexp"
	"strings"
	"unicode/utf8"
)

//go:generate go run ./generate/tldsgen
//go:generate go run ./generate/schemesgen
//go:generate go run ./generate/unicodegen

const (
	letter    = `\p{L}`
	mark      = `\p{M}`
	number    = `\p{N}`
	iriChar   = letter + mark + number
	currency  = `\p{Sc}`
	otherSymb = `\p{So}`
	endChar   = iriChar + `/\-_+&~%=#` + currency + otherSymb
	midChar   = endChar + "_*" + otherPuncMinusDoubleQuote
	wellParen = `\([` + midChar + `]*(\([` + midChar + `]*\)[` + midChar + `]*)*\)`
	wellBrack = `\[[` + midChar + `]*(\[[` + midChar + `]*\][` + midChar + `]*)*\]`
	wellBrace = `\{[` + midChar + `]*(\{[` + midChar + `]*\}[` + midChar + `]*)*\}`
	wellAll   = wellParen + `|` + wellBrack + `|` + wellBrace
	pathCont  = `([` + midChar + `]*(` + wellAll + `|[` + endChar + `])+)+`

	iri      = `[` + iriChar + `]([` + iriChar + `\-]*[` + iriChar + `])?`
	domain   = `(` + iri + `\.)+`
	octet    = `(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9][0-9]|[0-9])`
	ipv4Addr = `\b` + octet + `\.` + octet + `\.` + octet + `\.` + octet + `\b`
	ipv6Addr = `([0-9a-fA-F]{1,4}:([0-9a-fA-F]{1,4}:([0-9a-fA-F]{1,4}:([0-9a-fA-F]{1,4}:([0-9a-fA-F]{1,4}:[0-9a-fA-F]{0,4}|:[0-9a-fA-F]{1,4})?|(:[0-9a-fA-F]{1,4}){0,2})|(:[0-9a-fA-F]{1,4}){0,3})|(:[0-9a-fA-F]{1,4}){0,4})|:(:[0-9a-fA-F]{1,4}){0,5})((:[0-9a-fA-F]{1,4}){2}|:(25[0-5]|(2[0-4]|1[0-9]|[1-9])?[0-9])(\.(25[0-5]|(2[0-4]|1[0-9]|[1-9])?[0-9])){3})|(([0-9a-fA-F]{1,4}:){1,6}|:):[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){7}:`
	ipAddr   = `(` + ipv4Addr + `|` + ipv6Addr + `)`
	port     = `(:[0-9]*)?`
)

// AnyScheme can be passed to StrictMatchingScheme to match any possibly valid
// scheme, and not just the known ones.
var AnyScheme = `([a-zA-Z][a-zA-Z.\-+]*://|` + anyOf(SchemesNoAuthority...) + `:)`

// SchemesNoAuthority is a sorted list of some well-known url schemes that are
// followed by ":" instead of "://". The list includes both officially
// registered and unofficial schemes.
var SchemesNoAuthority = []string{
	`bitcoin`, // Bitcoin
	`cid`,     // Content-ID
	`file`,    // Files
	`magnet`,  // Torrent magnets
	`mailto`,  // Mail
	`mid`,     // Message-ID
	`sms`,     // SMS
	`tel`,     // Telephone
	`xmpp`,    // XMPP
}

// SchemesUnofficial is a sorted list of some well-known url schemes which
// aren't officially registered just yet. They tend to correspond to software.
//
// Mostly collected from https://en.wikipedia.org/wiki/List_of_URI_schemes#Unofficial_but_common_URI_schemes.
var SchemesUnofficial = []string{
	`gemini`,        // gemini
	`jdbc`,          // Java database Connectivity
	`moz-extension`, // Firefox extension
	`postgres`,      // PostgreSQL (short form)
	`postgresql`,    // PostgreSQL
	`slack`,         // Slack
	`zoommtg`,       // Zoom (desktop)
	`zoomus`,        // Zoom (mobile)
}

func anyOf(strs ...string) string {
	var b strings.Builder
	b.WriteByte('(')
	for i, s := range strs {
		if i != 0 {
			b.WriteByte('|')
		}
		b.WriteString(regexp.QuoteMeta(s))
	}
	b.WriteByte(')')
	return b.String()
}

func strictExp() string {
	schemes := `((` + anyOf(Schemes...) + `|` + anyOf(SchemesUnofficial...) + `)://|` + anyOf(SchemesNoAuthority...) + `:)`
	return `(?i)` + schemes + `(?-i)` + pathCont
}

func relaxedExp() string {
	var asciiTLDs, unicodeTLDs []string
	for i, tld := range TLDs {
		if tld[0] >= utf8.RuneSelf {
			asciiTLDs = TLDs[:i:i]
			unicodeTLDs = TLDs[i:]
			break
		}
	}
	punycode := `xn--[a-z0-9-]+`

	// Use \b to make sure ASCII TLDs are immediately followed by a word break.
	// We can't do that with unicode TLDs, as they don't see following
	// whitespace as a word break.
	tlds := `(?i)(` + punycode + `|` + anyOf(append(asciiTLDs, PseudoTLDs...)...) + `\b|` + anyOf(unicodeTLDs...) + `)(?-i)`
	site := domain + tlds

	hostName := `(` + site + `|` + ipAddr + `)`
	webURL := hostName + port + `(/|/` + pathCont + `)?`
	email := `[a-zA-Z0-9._%\-+]+@` + site
	return strictExp() + `|` + webURL + `|` + email
}

// Strict produces a regexp that matches any URL with a scheme in either the
// Schemes or SchemesNoAuthority lists.
func Strict() *regexp.Regexp {
	re := regexp.MustCompile(strictExp())
	re.Longest()
	return re
}

// Relaxed produces a regexp that matches any URL matched by Strict, plus any
// URL with no scheme or email address.
func Relaxed() *regexp.Regexp {
	re := regexp.MustCompile(relaxedExp())
	re.Longest()
	return re
}

// StrictMatchingScheme produces a regexp similar to Strict, but requiring that
// the scheme match the given regular expression. See AnyScheme too.
func StrictMatchingScheme(exp string) (*regexp.Regexp, error) {
	strictMatching := `(?i)(` + exp + `)(?-i)` + pathCont
	re, err := regexp.Compile(strictMatching)
	if err != nil {
		return nil, err
	}
	re.Longest()
	return re, nil
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"encoding/csv"
	"fmt"
	"log"
	"os"
	"strings"

	// Register hdb driver.
	_ "github.com/SAP/go-hdb/driver"
	// ini config
	"github.com/go-ini/ini"
	// internal
	"github.com/morxs/go-hana/utils"
	// cli
	"github.com/urfave/cli"
)

const (
	zstxlSQL = `select
MANDT,
TDOBJECT,
TDNAME,
TDID,
TDSPRAS,
LINNO,
TDLINE
from z_wilmar1.zstxl
where tdname in
(
	select
	concat(a.ebeln,a.ebelp) as "TDNAME"
	from z_wilmar1.ekpo a
	left join z_wilmar1.ekko b
	on a.mandt = b.mandt
	and a.ebeln = b.ebeln
	where b.aedat between ? and ?
	and b.bstyp = 'F'
	and (b.bsart like '%20' or b.bsart like '%25')
	and b.loekz = ''
	and a.loekz = ''
	and b.bukrs in ('BM', 'BO', 'CL', 'DE', 'EB', 'EC', 'EE', 'EL', 'EP', 'ES', 'FB', 'FM', 'GM', 'GU', 'HM', 'JW', 'KI', 'KM', 'NE', 'NO', 'NS', 'NX', 'OE', 'PB', 'PE', 'PO', 'RB', 'RH', 'RM', 'SE', 'SF', 'SG', 'SH', 'SO', 'SU', 'VI', 'WH',
	'AA', 'AD', 'AG', 'AJ', 'AN', 'AP', 'BN', 'BV', 'BW', 'BX', 'BY', 'CA', 'CC', 'CX', 'DA',
	'DB', 'DC', 'DG', 'DI', 'GA', 'GK', 'IA', 'ID', 'IE', 'IF', 'KD', 'KF', 'KG', 'MD', 'MF', 'MH',
	'MJ', 'MO', 'NI', 'PA', 'PF', 'PR', 'PT', 'PV', 'PX', 'RA', 'RJ',
	'SB', 'SJ', 'SN', 'SV', 'SX', 'TB', 'TC', 'TM', 'TN', 'UD', 'UI', 'WJ')
)`
)

const (
	cFile = "zstxl.csv"
)

func init() {
}

func main() {
	var sCfg, sStartDate, sEndDate string
	var bLog bool

	app := cli.NewApp()
	app.Name = "ZSTXL"
	app.Usage = "Get table ZSTXL"
	app.Version = "0.1.1"
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:        "config, c",
			Value:       "config.ini",
			Usage:       "Custom config file",
			Destination: &sCfg,
		},
		cli.StringFlag{
			Name:        "start, s",
			Usage:       "Start Date (SAP format)",
			Destination: &sStartDate,
		},
		cli.StringFlag{
			Name:        "end, e",
			Usage:       "End Date (SAP format)",
			Destination: &sEndDate,
		},
		cli.BoolFlag{
			Name:        "log, l",
			Hidden:      true,
			Usage:       "Enable logging. Log filename will be <query_filename>+.log",
			Destination: &bLog,
		},
	}

	app.Action = func(c *cli.Context) error {
		if sStartDate == "" || sEndDate == "" {
			log.Fatal("You need to enter Start and End Date")
		}
		// read config file
		utils.WriteMsg("READ CONFIG")
		iniCfg, err := ini.Load(sCfg)
		if err != nil {
			utils.WriteMsg("CONFIG")
			log.Fatal(err)
		}
		iniSection := iniCfg.Section("server")
		iniKeyUsername := iniSection.Key("uid").String()
		iniKeyPassword := iniSection.Key("pwd").String()
		iniKeyHost := iniSection.Key("host").String()
		iniKeyPort := iniSection.Key("port").String()
		hdbDsn := "hdb://" + iniKeyUsername + ":" + iniKeyPassword + "@" + iniKeyHost + ":" + iniKeyPort

		utils.WriteMsg("OPEN HDB")
		db, err := sql.Open(utils.DriverName, hdbDsn)
		if err != nil {
			log.Fatal(err)
		}
		defer db.Close()

		if err := db.Ping(); err != nil {
			log.Fatal(err)
		}

		// create file
		utils.WriteMsg("CREATE FILE: " + cFile)
		file, err := os.Create(cFile)
		if err != nil {
			log.Fatal(err)
		}
		defer file.Close()

		// try to query
		utils.WriteMsg("QUERY")
		rows, err := db.Query(zstxlSQL, sStartDate, sEndDate)
		if err != nil {
			log.Fatal(err)
		}
		defer rows.Close()

		// prepare file
		utils.WriteMsg("WRITE CSV")
		w := csv.NewWriter(file)
		w.Comma = ';'

		fmt.Println(rows.Columns())

		// add header to file
		rs, _ := rows.Columns()
		var rec []string
		for i := 0; i < len(rs); i++ {
			rec = append(rec, rs[i])
		}
		w.Write(rec)

		for rows.Next() {
			var mandt, tdobject, tdname, tdid, tdspras, linno, tdline string
			if err := rows.Scan(&mandt, &tdobject, &tdname, &tdid, &tdspras, &linno, &tdline); err != nil {
				utils.WriteMsg("SCAN")
				log.Fatal(err)
			}

			/*
				var bi big.Int
				var z float64
				var neg bool
				var i int
			*/

			var record []string

			record = append(record, mandt)
			record = append(record, tdobject)
			record = append(record, tdname)
			record = append(record, tdid)
			record = append(record, tdspras)
			record = append(record, linno)
			// delete all \n
			tdline = strings.Replace(tdline, "\n", " ", -1)
			tdline = strings.Replace(tdline, "\r", " ", -1)
			record = append(record, tdline)
			w.Write(record)
			//fmt.Println(record)
		}
		w.Flush()

		if err := rows.Err(); err != nil {
			utils.WriteMsg("ROWS")
			log.Fatal(err)
		}
		return nil
	}

	// init the program
	err := app.Run(os.Args)
	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>- add vgo module for starting test vgo - but it not working yet (at all)<commit_after>package main // import "github.com/morxs/go-hana"

import (
	"database/sql"
	"encoding/csv"
	"fmt"
	"log"
	"os"
	"strings"

	// Register hdb driver.
	_ "github.com/SAP/go-hdb/driver"
	// ini config
	"github.com/go-ini/ini"
	// internal
	"github.com/morxs/go-hana/utils"
	// cli
	"github.com/urfave/cli"
)

const (
	zstxlSQL = `select
MANDT,
TDOBJECT,
TDNAME,
TDID,
TDSPRAS,
LINNO,
TDLINE
from z_wilmar1.zstxl
where tdname in
(
	select
	concat(a.ebeln,a.ebelp) as "TDNAME"
	from z_wilmar1.ekpo a
	left join z_wilmar1.ekko b
	on a.mandt = b.mandt
	and a.ebeln = b.ebeln
	where b.aedat between ? and ?
	and b.bstyp = 'F'
	and (b.bsart like '%20' or b.bsart like '%25')
	and b.loekz = ''
	and a.loekz = ''
	and b.bukrs in ('BM', 'BO', 'CL', 'DE', 'EB', 'EC', 'EE', 'EL', 'EP', 'ES', 'FB', 'FM', 'GM', 'GU', 'HM', 'JW', 'KI', 'KM', 'NE', 'NO', 'NS', 'NX', 'OE', 'PB', 'PE', 'PO', 'RB', 'RH', 'RM', 'SE', 'SF', 'SG', 'SH', 'SO', 'SU', 'VI', 'WH',
	'AA', 'AD', 'AG', 'AJ', 'AN', 'AP', 'BN', 'BV', 'BW', 'BX', 'BY', 'CA', 'CC', 'CX', 'DA',
	'DB', 'DC', 'DG', 'DI', 'GA', 'GK', 'IA', 'ID', 'IE', 'IF', 'KD', 'KF', 'KG', 'MD', 'MF', 'MH',
	'MJ', 'MO', 'NI', 'PA', 'PF', 'PR', 'PT', 'PV', 'PX', 'RA', 'RJ',
	'SB', 'SJ', 'SN', 'SV', 'SX', 'TB', 'TC', 'TM', 'TN', 'UD', 'UI', 'WJ')
)`
)

const (
	cFile = "zstxl.csv"
)

func init() {
}

func main() {
	var sCfg, sStartDate, sEndDate string
	var bLog bool

	app := cli.NewApp()
	app.Name = "ZSTXL"
	app.Usage = "Get table ZSTXL"
	app.Version = "0.1.1"
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:        "config, c",
			Value:       "config.ini",
			Usage:       "Custom config file",
			Destination: &sCfg,
		},
		cli.StringFlag{
			Name:        "start, s",
			Usage:       "Start Date (SAP format)",
			Destination: &sStartDate,
		},
		cli.StringFlag{
			Name:        "end, e",
			Usage:       "End Date (SAP format)",
			Destination: &sEndDate,
		},
		cli.BoolFlag{
			Name:        "log, l",
			Hidden:      true,
			Usage:       "Enable logging. Log filename will be <query_filename>+.log",
			Destination: &bLog,
		},
	}

	app.Action = func(c *cli.Context) error {
		if sStartDate == "" || sEndDate == "" {
			log.Fatal("You need to enter Start and End Date")
		}
		// read config file
		utils.WriteMsg("READ CONFIG")
		iniCfg, err := ini.Load(sCfg)
		if err != nil {
			utils.WriteMsg("CONFIG")
			log.Fatal(err)
		}
		iniSection := iniCfg.Section("server")
		iniKeyUsername := iniSection.Key("uid").String()
		iniKeyPassword := iniSection.Key("pwd").String()
		iniKeyHost := iniSection.Key("host").String()
		iniKeyPort := iniSection.Key("port").String()
		hdbDsn := "hdb://" + iniKeyUsername + ":" + iniKeyPassword + "@" + iniKeyHost + ":" + iniKeyPort

		utils.WriteMsg("OPEN HDB")
		db, err := sql.Open(utils.DriverName, hdbDsn)
		if err != nil {
			log.Fatal(err)
		}
		defer db.Close()

		if err := db.Ping(); err != nil {
			log.Fatal(err)
		}

		// create file
		utils.WriteMsg("CREATE FILE: " + cFile)
		file, err := os.Create(cFile)
		if err != nil {
			log.Fatal(err)
		}
		defer file.Close()

		// try to query
		utils.WriteMsg("QUERY")
		rows, err := db.Query(zstxlSQL, sStartDate, sEndDate)
		if err != nil {
			log.Fatal(err)
		}
		defer rows.Close()

		// prepare file
		utils.WriteMsg("WRITE CSV")
		w := csv.NewWriter(file)
		w.Comma = ';'

		fmt.Println(rows.Columns())

		// add header to file
		rs, _ := rows.Columns()
		var rec []string
		for i := 0; i < len(rs); i++ {
			rec = append(rec, rs[i])
		}
		w.Write(rec)

		for rows.Next() {
			var mandt, tdobject, tdname, tdid, tdspras, linno, tdline string
			if err := rows.Scan(&mandt, &tdobject, &tdname, &tdid, &tdspras, &linno, &tdline); err != nil {
				utils.WriteMsg("SCAN")
				log.Fatal(err)
			}

			/*
				var bi big.Int
				var z float64
				var neg bool
				var i int
			*/

			var record []string

			record = append(record, mandt)
			record = append(record, tdobject)
			record = append(record, tdname)
			record = append(record, tdid)
			record = append(record, tdspras)
			record = append(record, linno)
			// delete all \n
			tdline = strings.Replace(tdline, "\n", " ", -1)
			tdline = strings.Replace(tdline, "\r", " ", -1)
			record = append(record, tdline)
			w.Write(record)
			//fmt.Println(record)
		}
		w.Flush()

		if err := rows.Err(); err != nil {
			utils.WriteMsg("ROWS")
			log.Fatal(err)
		}
		return nil
	}

	// init the program
	err := app.Run(os.Args)
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package equalfile

import (
	"bytes"
	"encoding/hex"
	"fmt"
	"hash"
	"io"
	"os"
)

const defaultMaxSize = 10000000000 // Only the first 10^10 bytes of io.Reader are compared.
const defaultBufSize = 20000

type Options struct {
	Debug         bool  // enable debugging to stdout
	ForceFileRead bool  // prevent shortcut at filesystem level (link, pathname, etc)
	MaxSize       int64 // prevent forever reading from an infinite reader
}

type Cmp struct {
	Opt Options

	readCount int
	readMin   int
	readMax   int
	readSum   int64

	hashType         hash.Hash
	hashMatchCompare bool
	hashTable        map[string]hashSum

	buf []byte
}

type hashSum struct {
	result []byte
	err    error
}

// New creates Cmp for multiple comparison mode.
func NewMultiple(buf []byte, options Options, h hash.Hash, compareOnMatch bool) *Cmp {
	c := &Cmp{
		Opt:              options,
		hashType:         h,
		hashMatchCompare: compareOnMatch,
		hashTable:        map[string]hashSum{},
		buf:              buf,
	}
	if c.Opt.MaxSize == 0 {
		c.Opt.MaxSize = defaultMaxSize
	}
	if c.buf == nil || len(c.buf) == 0 {
		c.buf = make([]byte, defaultBufSize)
	}
	return c
}

// New creates Cmp for single comparison mode.
func New(buf []byte, options Options) *Cmp {
	return NewMultiple(buf, options, nil, true)
}

func (c *Cmp) getHash(path string) ([]byte, error) {
	h, found := c.hashTable[path]
	if found {
		return h.result, h.err
	}

	f, openErr := os.Open(path)
	if openErr != nil {
		return nil, openErr
	}
	defer f.Close()

	sum := make([]byte, c.hashType.Size())
	c.hashType.Reset()
	n, copyErr := io.CopyN(c.hashType, f, c.Opt.MaxSize)
	copy(sum, c.hashType.Sum(nil))

	if copyErr == io.EOF && n < c.Opt.MaxSize {
		copyErr = nil
	}

	return c.newHash(path, sum, copyErr)
}

func (c *Cmp) newHash(path string, sum []byte, e error) ([]byte, error) {

	c.hashTable[path] = hashSum{sum, e}

	if c.Opt.Debug {
		fmt.Printf("newHash[%s]=%v: error=[%v]\n", path, hex.EncodeToString(sum), e)
	}

	return sum, e
}

func (c *Cmp) multipleMode() bool {
	return c.hashType != nil
}

// CompareFile verifies that files with names path1, path2 have same contents.
func (c *Cmp) CompareFile(path1, path2 string) (bool, error) {

	if c.multipleMode() {
		h1, err1 := c.getHash(path1)
		if err1 != nil {
			return false, err1
		}
		h2, err2 := c.getHash(path2)
		if err2 != nil {
			return false, err2
		}
		if !bytes.Equal(h1, h2) {
			return false, nil // hashes mismatch
		}
		// hashes match
		if !c.hashMatchCompare {
			return true, nil // accept hash match without byte-by-byte comparison
		}
		// do byte-by-byte comparison
		if c.Opt.Debug {
			fmt.Printf("CompareFile(%s,%s): hash match, will compare bytes\n", path1, path2)
		}
	}

	r1, openErr1 := os.Open(path1)
	if openErr1 != nil {
		return false, openErr1
	}
	defer r1.Close()
	info1, statErr1 := r1.Stat()
	if statErr1 != nil {
		return false, statErr1
	}

	r2, openErr2 := os.Open(path2)
	if openErr2 != nil {
		return false, openErr2
	}
	defer r2.Close()
	info2, statErr2 := r2.Stat()
	if statErr2 != nil {
		return false, statErr2
	}

	if info1.Size() != info2.Size() {
		return false, nil
	}

	if !c.Opt.ForceFileRead {
		// shortcut: ask the filesystem: are these files the same? (link, pathname, etc)
		if os.SameFile(info1, info2) {
			return true, nil
		}
	}

	// For files, set MaxSize to the initial Stat() size, rather than the
	// defaultMaxSize.  Growing files will return an error during the
	// comparison.
	if c.Opt.MaxSize == 0 {
		c.Opt.MaxSize = info1.Size()
	}

	return c.CompareReader(r1, r2)
}

func (c *Cmp) read(r io.Reader, buf []byte) (int, error) {
	n, err := r.Read(buf)

	if c.Opt.Debug {
		c.readCount++
		c.readSum += int64(n)
		if n < c.readMin {
			c.readMin = n
		}
		if n > c.readMax {
			c.readMax = n
		}
	}

	return n, err
}

// CompareReader verifies that two readers provide same content.
func (c *Cmp) CompareReader(r1, r2 io.Reader) (bool, error) {

	if c.Opt.Debug {
		c.readCount = 0
		c.readMin = 2000000000
		c.readMax = 0
		c.readSum = 0
	}

	equal, err := c.compareReader(r1, r2)

	if c.Opt.Debug {
		fmt.Printf("DEBUG CompareReader(%d,%d): readCount=%d readMin=%d readMax=%d readSum=%d\n",
			len(c.buf), c.Opt.MaxSize, c.readCount, c.readMin, c.readMax, c.readSum)
	}

	return equal, err
}

func readPartial(c *Cmp, r io.Reader, buf []byte, n1, n2 int) (int, error) {
	for n1 < n2 {
		n, err := c.read(r, buf[n1:n2])
		n1 += n
		if err != nil {
			return n1, err
		}
	}
	return n1, nil
}

func (c *Cmp) compareReader(r1, r2 io.Reader) (bool, error) {

	maxSize := c.Opt.MaxSize
	if maxSize < 1 {
		return false, fmt.Errorf("nonpositive max size")
	}

	buf := c.buf

	size := len(buf) / 2
	if size < 1 {
		return false, fmt.Errorf("insufficient buffer size")
	}

	buf1 := buf[:size]
	buf2 := buf[size : 2*size] // must force same size as buf1

	if len(buf1) != len(buf2) {
		return false, fmt.Errorf("buffer size mismatch buf1=%d buf2=%d", len(buf1), len(buf2))
	}

	eof1 := false
	eof2 := false
	var readSize int64

	for !eof1 && !eof2 {
		n1, err1 := c.read(r1, buf1)
		switch err1 {
		case io.EOF:
			eof1 = true
		case nil:
		default:
			return false, err1
		}

		n2, err2 := c.read(r2, buf2)
		switch err2 {
		case io.EOF:
			eof2 = true
		case nil:
		default:
			return false, err2
		}

		switch {
		case n1 < n2:
			n, errPart := readPartial(c, r1, buf1, n1, n2)
			switch errPart {
			case io.EOF:
				eof1 = true
			case nil:
			default:
				return false, errPart
			}
			n1 = n
		case n2 < n1:
			n, errPart := readPartial(c, r2, buf2, n2, n1)
			switch errPart {
			case io.EOF:
				eof2 = true
			case nil:
			default:
				return false, errPart
			}
			n2 = n
		}

		if n1 != n2 {
			return false, nil
		}

		if !bytes.Equal(buf1[:n1], buf2[:n2]) {
			return false, nil
		}

		readSize += int64(n1)
		if readSize > maxSize {
			return true, fmt.Errorf("max read size reached")
		}
	}

	if !eof1 || !eof2 {
		return false, nil
	}

	return true, nil
}
<commit_msg>Setup file MaxSize only when file len is > 0<commit_after>package equalfile

import (
	"bytes"
	"encoding/hex"
	"fmt"
	"hash"
	"io"
	"os"
)

const defaultMaxSize = 10000000000 // Only the first 10^10 bytes of io.Reader are compared.
const defaultBufSize = 20000

type Options struct {
	Debug         bool  // enable debugging to stdout
	ForceFileRead bool  // prevent shortcut at filesystem level (link, pathname, etc)
	MaxSize       int64 // prevent forever reading from an infinite reader
}

type Cmp struct {
	Opt Options

	readCount int
	readMin   int
	readMax   int
	readSum   int64

	hashType         hash.Hash
	hashMatchCompare bool
	hashTable        map[string]hashSum

	buf []byte
}

type hashSum struct {
	result []byte
	err    error
}

// New creates Cmp for multiple comparison mode.
func NewMultiple(buf []byte, options Options, h hash.Hash, compareOnMatch bool) *Cmp {
	c := &Cmp{
		Opt:              options,
		hashType:         h,
		hashMatchCompare: compareOnMatch,
		hashTable:        map[string]hashSum{},
		buf:              buf,
	}
	if c.Opt.MaxSize == 0 {
		c.Opt.MaxSize = defaultMaxSize
	}
	if c.buf == nil || len(c.buf) == 0 {
		c.buf = make([]byte, defaultBufSize)
	}
	return c
}

// New creates Cmp for single comparison mode.
func New(buf []byte, options Options) *Cmp {
	return NewMultiple(buf, options, nil, true)
}

func (c *Cmp) getHash(path string) ([]byte, error) {
	h, found := c.hashTable[path]
	if found {
		return h.result, h.err
	}

	f, openErr := os.Open(path)
	if openErr != nil {
		return nil, openErr
	}
	defer f.Close()

	sum := make([]byte, c.hashType.Size())
	c.hashType.Reset()
	n, copyErr := io.CopyN(c.hashType, f, c.Opt.MaxSize)
	copy(sum, c.hashType.Sum(nil))

	if copyErr == io.EOF && n < c.Opt.MaxSize {
		copyErr = nil
	}

	return c.newHash(path, sum, copyErr)
}

func (c *Cmp) newHash(path string, sum []byte, e error) ([]byte, error) {

	c.hashTable[path] = hashSum{sum, e}

	if c.Opt.Debug {
		fmt.Printf("newHash[%s]=%v: error=[%v]\n", path, hex.EncodeToString(sum), e)
	}

	return sum, e
}

func (c *Cmp) multipleMode() bool {
	return c.hashType != nil
}

// CompareFile verifies that files with names path1, path2 have same contents.
func (c *Cmp) CompareFile(path1, path2 string) (bool, error) {

	if c.multipleMode() {
		h1, err1 := c.getHash(path1)
		if err1 != nil {
			return false, err1
		}
		h2, err2 := c.getHash(path2)
		if err2 != nil {
			return false, err2
		}
		if !bytes.Equal(h1, h2) {
			return false, nil // hashes mismatch
		}
		// hashes match
		if !c.hashMatchCompare {
			return true, nil // accept hash match without byte-by-byte comparison
		}
		// do byte-by-byte comparison
		if c.Opt.Debug {
			fmt.Printf("CompareFile(%s,%s): hash match, will compare bytes\n", path1, path2)
		}
	}

	r1, openErr1 := os.Open(path1)
	if openErr1 != nil {
		return false, openErr1
	}
	defer r1.Close()
	info1, statErr1 := r1.Stat()
	if statErr1 != nil {
		return false, statErr1
	}

	r2, openErr2 := os.Open(path2)
	if openErr2 != nil {
		return false, openErr2
	}
	defer r2.Close()
	info2, statErr2 := r2.Stat()
	if statErr2 != nil {
		return false, statErr2
	}

	if info1.Size() != info2.Size() {
		return false, nil
	}

	if !c.Opt.ForceFileRead {
		// shortcut: ask the filesystem: are these files the same? (link, pathname, etc)
		if os.SameFile(info1, info2) {
			return true, nil
		}
	}

	// For files, set MaxSize to the initial Stat() size, rather than the
	// defaultMaxSize.  Growing files will return an error during the
	// comparison.
	if c.Opt.MaxSize == 0 {
		if info1.Size() > 0 {
			c.Opt.MaxSize = info1.Size()
		} else {
			c.Opt.MaxSize = 1
		}
	}

	return c.CompareReader(r1, r2)
}

func (c *Cmp) read(r io.Reader, buf []byte) (int, error) {
	n, err := r.Read(buf)

	if c.Opt.Debug {
		c.readCount++
		c.readSum += int64(n)
		if n < c.readMin {
			c.readMin = n
		}
		if n > c.readMax {
			c.readMax = n
		}
	}

	return n, err
}

// CompareReader verifies that two readers provide same content.
func (c *Cmp) CompareReader(r1, r2 io.Reader) (bool, error) {

	if c.Opt.Debug {
		c.readCount = 0
		c.readMin = 2000000000
		c.readMax = 0
		c.readSum = 0
	}

	equal, err := c.compareReader(r1, r2)

	if c.Opt.Debug {
		fmt.Printf("DEBUG CompareReader(%d,%d): readCount=%d readMin=%d readMax=%d readSum=%d\n",
			len(c.buf), c.Opt.MaxSize, c.readCount, c.readMin, c.readMax, c.readSum)
	}

	return equal, err
}

func readPartial(c *Cmp, r io.Reader, buf []byte, n1, n2 int) (int, error) {
	for n1 < n2 {
		n, err := c.read(r, buf[n1:n2])
		n1 += n
		if err != nil {
			return n1, err
		}
	}
	return n1, nil
}

func (c *Cmp) compareReader(r1, r2 io.Reader) (bool, error) {

	maxSize := c.Opt.MaxSize
	if maxSize < 1 {
		return false, fmt.Errorf("nonpositive max size")
	}

	buf := c.buf

	size := len(buf) / 2
	if size < 1 {
		return false, fmt.Errorf("insufficient buffer size")
	}

	buf1 := buf[:size]
	buf2 := buf[size : 2*size] // must force same size as buf1

	if len(buf1) != len(buf2) {
		return false, fmt.Errorf("buffer size mismatch buf1=%d buf2=%d", len(buf1), len(buf2))
	}

	eof1 := false
	eof2 := false
	var readSize int64

	for !eof1 && !eof2 {
		n1, err1 := c.read(r1, buf1)
		switch err1 {
		case io.EOF:
			eof1 = true
		case nil:
		default:
			return false, err1
		}

		n2, err2 := c.read(r2, buf2)
		switch err2 {
		case io.EOF:
			eof2 = true
		case nil:
		default:
			return false, err2
		}

		switch {
		case n1 < n2:
			n, errPart := readPartial(c, r1, buf1, n1, n2)
			switch errPart {
			case io.EOF:
				eof1 = true
			case nil:
			default:
				return false, errPart
			}
			n1 = n
		case n2 < n1:
			n, errPart := readPartial(c, r2, buf2, n2, n1)
			switch errPart {
			case io.EOF:
				eof2 = true
			case nil:
			default:
				return false, errPart
			}
			n2 = n
		}

		if n1 != n2 {
			return false, nil
		}

		if !bytes.Equal(buf1[:n1], buf2[:n2]) {
			return false, nil
		}

		readSize += int64(n1)
		if readSize > maxSize {
			return true, fmt.Errorf("max read size reached")
		}
	}

	if !eof1 || !eof2 {
		return false, nil
	}

	return true, nil
}
<|endoftext|>


<commit_before>// Copyright 2014, Joe Tsai. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE.md file.

// Package errs implements helpers functions to deal with errors.
package errs

import "runtime"
import "errors"

// Create a new error.
func New(str string) error {
	return errors.New(str)
}

// Panic if the error is not nil.
func Panic(err error) {
	if err != nil {
		panic(err)
	}
}

// Recovers from any panics and stores errors to the given pointer. If the
// source of the panic was not an error, then the panic continues.
func Recover(err *error) {
	switch ex := recover().(type) {
	case nil:
		// Do nothing
	case runtime.Error:
		panic(ex)
	case error:
		*err = ex
	default:
		panic(ex)
	}
}

// Recovers from any panics and ignores them.
// This is dangerous and should be used sparingly.
func NilRecover() {
	_ = recover()
}

// Convert errors from one type to another.
func Convert(err error, errNew error, errChks ...error) error {
	for _, errChk := range errChks {
		if err == errChk {
			return errNew
		}
	}
	return err
}

// Check if the provided error matches any of the others.
func Match(err error, errChks ...error) bool {
	return err != nil && Convert(err, nil, errChks...) == nil
}

// Ignore certain types of errors.
func Ignore(err error, errChks ...error) error {
	return Convert(err, nil, errChks...)
}

// Return the first non-nil error.
func First(errs ...error) error {
	for _, err := range errs {
		if err != nil {
			return err
		}
	}
	return nil
}

// Assert that an condition is true.
func Assert(cond bool, err error) {
	if !cond {
		panic(err)
	}
}
<commit_msg>Update comment<commit_after>// Copyright 2014, Joe Tsai. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE.md file.

// Package errs implements helpers functions to deal with errors.
package errs

import "runtime"
import "errors"

// Create a new error.
func New(str string) error {
	return errors.New(str)
}

// Panic if the error is not nil.
func Panic(err error) {
	if err != nil {
		panic(err)
	}
}

// Recovers from any panics and stores errors to the given pointer. If the
// source of the panic was not a Runtime error or an error at all, then the
// panic continues.
func Recover(err *error) {
	switch ex := recover().(type) {
	case nil:
		// Do nothing
	case runtime.Error:
		panic(ex)
	case error:
		*err = ex
	default:
		panic(ex)
	}
}

// Recovers from any panics and ignores them.
// This is dangerous and should be used sparingly.
func NilRecover() {
	_ = recover()
}

// Convert errors from one type to another.
func Convert(err error, errNew error, errChks ...error) error {
	for _, errChk := range errChks {
		if err == errChk {
			return errNew
		}
	}
	return err
}

// Check if the provided error matches any of the others.
func Match(err error, errChks ...error) bool {
	return err != nil && Convert(err, nil, errChks...) == nil
}

// Ignore certain types of errors.
func Ignore(err error, errChks ...error) error {
	return Convert(err, nil, errChks...)
}

// Return the first non-nil error.
func First(errs ...error) error {
	for _, err := range errs {
		if err != nil {
			return err
		}
	}
	return nil
}

// Assert that an condition is true.
func Assert(cond bool, err error) {
	if !cond {
		panic(err)
	}
}
<|endoftext|>


<commit_before>package gateway

import (
	"errors"
	"net"
	"time"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/encoding"
	"github.com/NebulousLabs/Sia/modules"
)

var (
	errNodeExists = errors.New("node already added")
	errNoNodes    = errors.New("no nodes in the node list")
	errOurAddress = errors.New("can't add our own address")
)

// addNode adds an address to the set of nodes on the network.
func (g *Gateway) addNode(addr modules.NetAddress) error {
	if addr == g.myAddr {
		return errOurAddress
	} else if _, exists := g.nodes[addr]; exists {
		return errNodeExists
	} else if addr.IsStdValid() != nil {
		return errors.New("address is not valid: " + string(addr))
	} else if net.ParseIP(addr.Host()) == nil {
		return errors.New("address must be an IP address: " + string(addr))
	}
	g.nodes[addr] = struct{}{}
	return nil
}

// managedAddUntrustedNode adds an address to the set of nodes on the network, but
// first verifies that there is a reachable node at the provided address.
func (g *Gateway) managedAddUntrustedNode(addr modules.NetAddress) error {
	// Performing the ping during testing does not work.
	if build.Release == "testing" {
		g.mu.Lock()
		defer g.mu.Unlock()
		return g.addNode(addr)
	}

	// Ping the untrusted node to see whether or not there's acutally a
	// reachable node at the provided address.
	conn, err := g.dial(addr)
	if err != nil {
		return err
	}
	// If connection succeeds, supply an unacceptable version so that we
	// will not be added as a peer.
	//
	// NOTE: this is a somewhat clunky way of specifying that you didn't
	// actually want a connection.
	encoding.WriteObject(conn, "0.0.0")
	conn.Close()

	g.mu.Lock()
	defer g.mu.Unlock()
	err = g.addNode(addr)
	if err != nil {
		return err
	}
	return g.save()
}

// removeNode will remove a node from the gateway.
func (g *Gateway) removeNode(addr modules.NetAddress) error {
	if _, exists := g.nodes[addr]; !exists {
		return errors.New("no record of that node")
	}
	delete(g.nodes, addr)
	return nil
}

// randomNode returns a random node from the gateway. An error can be returned
// if there are no nodes in the node list.
func (g *Gateway) randomNode() (modules.NetAddress, error) {
	if len(g.nodes) == 0 {
		return "", errNoPeers
	}

	// Select a random peer. Note that the algorithm below is roughly linear in
	// the number of nodes known by the gateway, and this number can approach
	// every node on the network. If the network gets large, this algorithm
	// will either need to be refactored, or more likely a cap on the size of
	// g.nodes will need to be added.
	r, err := crypto.RandIntn(len(g.nodes))
	if err != nil {
		return "", err
	}
	for node := range g.nodes {
		if r <= 0 {
			return node, nil
		}
		r--
	}
	return "", errNoPeers
}

// shareNodes is the receiving end of the ShareNodes RPC. It writes up to 10
// randomly selected nodes to the caller.
func (g *Gateway) shareNodes(conn modules.PeerConn) error {
	g.mu.RLock()
	var nodes []modules.NetAddress
	for node := range g.nodes {
		if uint64(len(nodes)) == maxSharedNodes {
			break
		}

		// Don't share local peers with remote peers. That means that if 'node'
		// is loopback, it will only be shared if the remote peer is also
		// loopback. And if 'node' is private, it will only be shared if the
		// remote peer is either the loopback or is also private.
		remoteNA := modules.NetAddress(conn.RemoteAddr().String())
		if node.IsLoopback() && !remoteNA.IsLoopback() {
			continue
		}
		if node.IsLocal() && !remoteNA.IsLocal() {
			continue
		}

		nodes = append(nodes, node)
	}
	g.mu.RUnlock()
	return encoding.WriteObject(conn, nodes)
}

// requestNodes is the calling end of the ShareNodes RPC.
func (g *Gateway) requestNodes(conn modules.PeerConn) error {
	var nodes []modules.NetAddress
	if err := encoding.ReadObject(conn, &nodes, maxSharedNodes*modules.MaxEncodedNetAddressLength); err != nil {
		return err
	}

	g.mu.Lock()
	for _, node := range nodes {
		err := g.addNode(node)
		if err != nil && err != errNodeExists && err != errOurAddress {
			g.log.Printf("WARN: peer '%v' sent the invalid addr '%v'", conn.RPCAddr(), node)
		}
	}
	g.save()
	g.mu.Unlock()
	return nil
}

// permanentNodePurger is a thread that runs throughout the lifetime of the
// gateway, purging unconnectable nodes from the node list in a sustainable
// way.
func (g *Gateway) permanentNodePurger(closeChan chan struct{}) {
	defer close(closeChan)

	for {
		// Start by sleeping for 10 minutes. This means that no nodes will be
		// purged for the first 10 minutes that Sia is running, and that any
		// failures to get nodes will be met by 10 minutes of waiting.
		//
		// At most one node will be contacted every 10 minutes. This minimizes
		// the total amount of keepalive traffic on the network.
		select {
		case <-time.After(nodePurgeDelay):
		case <-g.threads.StopChan():
			// The gateway is shutting down, close out the thread.
			return
		}

		// Get a random node for scanning.
		g.mu.RLock()
		numNodes := len(g.nodes)
		node, err := g.randomNode()
		g.mu.RUnlock()
		if err == errNoNodes {
			// errNoNodes is a common error that will be resovled by the
			// bootstrap process.
			continue
		} else if err != nil {
			// Unusual error, create a logging statement.
			g.log.Println("ERROR: could not pick a random node for uptime check:", err)
			continue
		}
		if numNodes <= pruneNodeListLen {
			// There are not enough nodes in the gateway - pruning more is
			// probably a bad idea, and may affect the user's ability to
			// connect to the network in the future.
			continue
		}

		// Try connecting to the random node. If the node is not reachable,
		// remove them from the node list.
		conn, err := g.dial(node)
		if err != nil {
			// NOTE: an error may be returned if the dial is cancelled
			// partway through, which would cause the node to be pruned
			// even though it may be a good node. Because nodes are
			// plentiful, that's not a huge problem.
			g.mu.Lock()
			g.removeNode(node)
			g.save()
			g.mu.Unlock()
			g.log.Debugf("INFO: removing node %q because dialing it failed: %v", node, err)
			continue
		}

		// If connection succeeds, supply an unacceptable version so that we
		// will not be added as a peer.
		//
		// NOTE: this is a somewhat clunky way of specifying that you didn't
		// actually want a connection.
		encoding.WriteObject(conn, "0.0.0")
		conn.Close()
	}
}

// permanentNodeManager tries to keep the Gateway's node list healthy. As long
// as the Gateway has fewer than healthyNodeListLen nodes, it asks a random
// peer for more nodes. It also continually pings nodes in order to establish
// their connectivity. Unresponsive nodes are aggressively removed.
func (g *Gateway) permanentNodeManager(closeChan chan struct{}) {
	defer close(closeChan)

	for {
		// Wait 5 seconds so that a controlled number of node requests are made
		// to peers.
		select {
		case <-time.After(nodeListDelay):
		case <-g.threads.StopChan():
			// Gateway is shutting down, close the thread.
			return
		}

		g.mu.RLock()
		numNodes := len(g.nodes)
		peer, err := g.randomOutboundPeer()
		g.mu.RUnlock()
		if err == errNoPeers {
			// errNoPeers is a common and expected error, there's no need to
			// log it.
			continue
		} else if err != nil {
			g.log.Println("ERROR: could not fetch a random peer:", err)
			continue
		}

		// Determine whether there are a satisfactory number of nodes in the
		// nodelist. If there are not, use the random peer from earlier to
		// expand the node list.
		if numNodes < healthyNodeListLen {
			err := g.managedRPC(peer, "ShareNodes", g.requestNodes)
			if err != nil {
				g.log.Debugf("WARN: RPC ShareNodes failed on peer %q: %v", peer, err)
				continue
			}
		} else {
			// There are enough nodes in the gateway, no need to check for more
			// every 5 seconds. Wait a while before checking again.
			select {
			case <-time.After(wellConnectedDelay):
			case <-g.threads.StopChan():
				// Gateway is shutting down, close the thread.
				return
			}
		}
	}
}
<commit_msg>add a deadline to shareNodes, also correct randomness<commit_after>package gateway

import (
	"errors"
	"net"
	"time"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/encoding"
	"github.com/NebulousLabs/Sia/modules"
)

var (
	errNodeExists = errors.New("node already added")
	errNoNodes    = errors.New("no nodes in the node list")
	errOurAddress = errors.New("can't add our own address")
)

// addNode adds an address to the set of nodes on the network.
func (g *Gateway) addNode(addr modules.NetAddress) error {
	if addr == g.myAddr {
		return errOurAddress
	} else if _, exists := g.nodes[addr]; exists {
		return errNodeExists
	} else if addr.IsStdValid() != nil {
		return errors.New("address is not valid: " + string(addr))
	} else if net.ParseIP(addr.Host()) == nil {
		return errors.New("address must be an IP address: " + string(addr))
	}
	g.nodes[addr] = struct{}{}
	return nil
}

// managedAddUntrustedNode adds an address to the set of nodes on the network, but
// first verifies that there is a reachable node at the provided address.
func (g *Gateway) managedAddUntrustedNode(addr modules.NetAddress) error {
	// Performing the ping during testing does not work.
	if build.Release == "testing" {
		g.mu.Lock()
		defer g.mu.Unlock()
		return g.addNode(addr)
	}

	// Ping the untrusted node to see whether or not there's acutally a
	// reachable node at the provided address.
	conn, err := g.dial(addr)
	if err != nil {
		return err
	}
	// If connection succeeds, supply an unacceptable version so that we
	// will not be added as a peer.
	//
	// NOTE: this is a somewhat clunky way of specifying that you didn't
	// actually want a connection.
	encoding.WriteObject(conn, "0.0.0")
	conn.Close()

	g.mu.Lock()
	defer g.mu.Unlock()
	err = g.addNode(addr)
	if err != nil {
		return err
	}
	return g.save()
}

// removeNode will remove a node from the gateway.
func (g *Gateway) removeNode(addr modules.NetAddress) error {
	if _, exists := g.nodes[addr]; !exists {
		return errors.New("no record of that node")
	}
	delete(g.nodes, addr)
	return nil
}

// randomNode returns a random node from the gateway. An error can be returned
// if there are no nodes in the node list.
func (g *Gateway) randomNode() (modules.NetAddress, error) {
	if len(g.nodes) == 0 {
		return "", errNoPeers
	}

	// Select a random peer. Note that the algorithm below is roughly linear in
	// the number of nodes known by the gateway, and this number can approach
	// every node on the network. If the network gets large, this algorithm
	// will either need to be refactored, or more likely a cap on the size of
	// g.nodes will need to be added.
	r, err := crypto.RandIntn(len(g.nodes))
	if err != nil {
		return "", err
	}
	for node := range g.nodes {
		if r <= 0 {
			return node, nil
		}
		r--
	}
	return "", errNoPeers
}

// shareNodes is the receiving end of the ShareNodes RPC. It writes up to 10
// randomly selected nodes to the caller.
func (g *Gateway) shareNodes(conn modules.PeerConn) error {
	// Set a deadline on the connection.
	conn.SetDeadline(time.Now().Add(connStdDeadline))

	// Assemble a list of nodes to send to the peer.
	var nodes []modules.NetAddress
	func () {
		g.mu.RLock()
		defer g.mu.RUnlock()

		// Create a random permutation of nodes from the gateway to iterate
		// through.
		gnodes := make([]modules.NetAddress, len(g.nodes))
		for node := range g.nodes {
			gnodes = append(node)
		}
		perm, err := crypto.Perm(len(g.nodes))
		if err != nil {
			g.log.Severe("Unable to get random permutation for sharing nodes")
		}

		// Iterate through the random permutation of nodes and select the
		// desirable ones.
		remoteNA := modules.NetAddress(conn.RemoteAddr().String())
		for _, i := range perm {
			if uint64(len(nodes)) == maxSharedNodes {
				break
			}

			// Don't share local peers with remote peers. That means that if 'node'
			// is loopback, it will only be shared if the remote peer is also
			// loopback. And if 'node' is private, it will only be shared if the
			// remote peer is either the loopback or is also private.
			node := gnodes[i]
			if node.IsLoopback() && !remoteNA.IsLoopback() {
				continue
			}
			if node.IsLocal() && !remoteNA.IsLocal() {
				continue
			}

			nodes = append(nodes, node)
		}
	}()
	return encoding.WriteObject(conn, nodes)
}

// requestNodes is the calling end of the ShareNodes RPC.
func (g *Gateway) requestNodes(conn modules.PeerConn) error {
	var nodes []modules.NetAddress
	if err := encoding.ReadObject(conn, &nodes, maxSharedNodes*modules.MaxEncodedNetAddressLength); err != nil {
		return err
	}

	g.mu.Lock()
	for _, node := range nodes {
		err := g.addNode(node)
		if err != nil && err != errNodeExists && err != errOurAddress {
			g.log.Printf("WARN: peer '%v' sent the invalid addr '%v'", conn.RPCAddr(), node)
		}
	}
	g.save()
	g.mu.Unlock()
	return nil
}

// permanentNodePurger is a thread that runs throughout the lifetime of the
// gateway, purging unconnectable nodes from the node list in a sustainable
// way.
func (g *Gateway) permanentNodePurger(closeChan chan struct{}) {
	defer close(closeChan)

	for {
		// Start by sleeping for 10 minutes. This means that no nodes will be
		// purged for the first 10 minutes that Sia is running, and that any
		// failures to get nodes will be met by 10 minutes of waiting.
		//
		// At most one node will be contacted every 10 minutes. This minimizes
		// the total amount of keepalive traffic on the network.
		select {
		case <-time.After(nodePurgeDelay):
		case <-g.threads.StopChan():
			// The gateway is shutting down, close out the thread.
			return
		}

		// Get a random node for scanning.
		g.mu.RLock()
		numNodes := len(g.nodes)
		node, err := g.randomNode()
		g.mu.RUnlock()
		if err == errNoNodes {
			// errNoNodes is a common error that will be resovled by the
			// bootstrap process.
			continue
		} else if err != nil {
			// Unusual error, create a logging statement.
			g.log.Println("ERROR: could not pick a random node for uptime check:", err)
			continue
		}
		if numNodes <= pruneNodeListLen {
			// There are not enough nodes in the gateway - pruning more is
			// probably a bad idea, and may affect the user's ability to
			// connect to the network in the future.
			continue
		}

		// Try connecting to the random node. If the node is not reachable,
		// remove them from the node list.
		conn, err := g.dial(node)
		if err != nil {
			// NOTE: an error may be returned if the dial is cancelled
			// partway through, which would cause the node to be pruned
			// even though it may be a good node. Because nodes are
			// plentiful, that's not a huge problem.
			g.mu.Lock()
			g.removeNode(node)
			g.save()
			g.mu.Unlock()
			g.log.Debugf("INFO: removing node %q because dialing it failed: %v", node, err)
			continue
		}

		// If connection succeeds, supply an unacceptable version so that we
		// will not be added as a peer.
		//
		// NOTE: this is a somewhat clunky way of specifying that you didn't
		// actually want a connection.
		encoding.WriteObject(conn, "0.0.0")
		conn.Close()
	}
}

// permanentNodeManager tries to keep the Gateway's node list healthy. As long
// as the Gateway has fewer than healthyNodeListLen nodes, it asks a random
// peer for more nodes. It also continually pings nodes in order to establish
// their connectivity. Unresponsive nodes are aggressively removed.
func (g *Gateway) permanentNodeManager(closeChan chan struct{}) {
	defer close(closeChan)

	for {
		// Wait 5 seconds so that a controlled number of node requests are made
		// to peers.
		select {
		case <-time.After(nodeListDelay):
		case <-g.threads.StopChan():
			// Gateway is shutting down, close the thread.
			return
		}

		g.mu.RLock()
		numNodes := len(g.nodes)
		peer, err := g.randomOutboundPeer()
		g.mu.RUnlock()
		if err == errNoPeers {
			// errNoPeers is a common and expected error, there's no need to
			// log it.
			continue
		} else if err != nil {
			g.log.Println("ERROR: could not fetch a random peer:", err)
			continue
		}

		// Determine whether there are a satisfactory number of nodes in the
		// nodelist. If there are not, use the random peer from earlier to
		// expand the node list.
		if numNodes < healthyNodeListLen {
			err := g.managedRPC(peer, "ShareNodes", g.requestNodes)
			if err != nil {
				g.log.Debugf("WARN: RPC ShareNodes failed on peer %q: %v", peer, err)
				continue
			}
		} else {
			// There are enough nodes in the gateway, no need to check for more
			// every 5 seconds. Wait a while before checking again.
			select {
			case <-time.After(wellConnectedDelay):
			case <-g.threads.StopChan():
				// Gateway is shutting down, close the thread.
				return
			}
		}
	}
}
<|endoftext|>