<commit_before>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package params

import "github.com/juju/juju/storage"

// MachineBlockDevices holds a machine tag and the block devices present
// on that machine.
type MachineBlockDevices struct {
	Machine      string                `json:"machine"`
	BlockDevices []storage.BlockDevice `json:"blockdevices,omitempty"`
}

// SetMachineBlockDevices holds the arguments for recording the block
// devices present on a set of machines.
type SetMachineBlockDevices struct {
	MachineBlockDevices []MachineBlockDevices `json:"machineblockdevices"`
}

// BlockDeviceResult holds the result of an API call to retrieve details
// of a block device.
type BlockDeviceResult struct {
	Result storage.BlockDevice `json:"result"`
	Error  *Error              `json:"error,omitempty"`
}

// BlockDeviceResults holds the result of an API call to retrieve details
// of multiple block devices.
type BlockDeviceResults struct {
	Results []BlockDeviceResult `json:"results,omitempty"`
}

// BlockDevicesResult holds the result of an API call to retrieve details
// of all block devices relating to some entity.
type BlockDevicesResult struct {
	Result []storage.BlockDevice `json:"result"`
	Error  *Error                `json:"error,omitempty"`
}

// BlockDevicseResults holds the result of an API call to retrieve details
// of all block devices relating to some entities.
type BlockDevicesResults struct {
	Results []BlockDevicesResult `json:"results,omitempty"`
}

// StorageInstance describes a storage instance.
type StorageInstance struct {
	StorageTag string
	OwnerTag   string
	Kind       StorageKind
}

// StorageKind is the kind of a storage instance.
type StorageKind int

const (
	StorageKindUnknown StorageKind = iota
	StorageKindBlock
	StorageKindFilesystem
)

func (k *StorageKind) String() string {
	switch *k {
	case StorageKindBlock:
		return "block"
	case StorageKindFilesystem:
		return "file system"
	default:
		return "unknown"
	}
}

// StorageInstanceResult holds the result of an API call to retrieve details
// of a storage instance.
type StorageInstanceResult struct {
	Result StorageInstance `json:"result"`
	Error  *Error          `json:"error,omitempty"`
}

// StorageInstanceResults holds the result of an API call to retrieve details
// of multiple storage instances.
type StorageInstanceResults struct {
	Results []StorageInstanceResult `json:"results,omitempty"`
}

// StorageAttachment describes a unit's attached storage instance.
type StorageAttachment struct {
	StorageTag string
	OwnerTag   string
	UnitTag    string

	Kind     StorageKind
	Location string
	Life     Life
}

// StorageAttachmentId identifies a storage attachment by the tags of the
// related unit and storage instance.
type StorageAttachmentId struct {
	StorageTag string `json:"storagetag"`
	UnitTag    string `json:"unittag"`
}

// StorageAttachmentIds holds a set of storage attachment identifiers.
type StorageAttachmentIds struct {
	Ids []StorageAttachmentId `json:"ids"`
}

// StorageAttachmentsResult holds the result of an API call to retrieve details
// of a unit's attached storage instances.
type StorageAttachmentsResult struct {
	Result []StorageAttachment `json:"result"`
	Error  *Error              `json:"error,omitempty"`
}

// StorageAttachmentsResults holds the result of an API call to retrieve details
// of multiple units' attached storage instances.
type StorageAttachmentsResults struct {
	Results []StorageAttachmentsResult `json:"results,omitempty"`
}

// StorageAttachmentResult holds the result of an API call to retrieve details
// of a storage attachment.
type StorageAttachmentResult struct {
	Result StorageAttachment `json:"result"`
	Error  *Error            `json:"error,omitempty"`
}

// StorageAttachmentResults holds the result of an API call to retrieve details
// of multiple storage attachments.
type StorageAttachmentResults struct {
	Results []StorageAttachmentResult `json:"results,omitempty"`
}

// Volume describes a storage volume in the environment.
type Volume struct {
	VolumeTag string `json:"volumetag"`
	VolumeId  string `json:"volumeid"`
	Serial    string `json:"serial"`
	// Size is the size of the volume in MiB.
	Size uint64 `json:"size"`
}

// VolumeAttachmentId identifies a volume attachment by the tags of the
// related machine and volume.
type VolumeAttachmentId struct {
	VolumeTag  string `json:"volumetag"`
	MachineTag string `json:"machinetag"`
}

// VolumeAttachmentIds holds a set of volume attachment identifiers.
type VolumeAttachmentIds struct {
	Ids []VolumeAttachmentId `json:"ids"`
}

// VolumeAttachment describes a volume attachment.
type VolumeAttachment struct {
	VolumeTag  string `json:"volumetag"`
	MachineTag string `json:"machinetag"`
	DeviceName string `json:"devicename,omitempty"`
}

// VolumeParams holds the parameters for creating a storage volume.
type VolumeParams struct {
	VolumeTag  string                 `json:"volumetag"`
	Size       uint64                 `json:"size"`
	Provider   string                 `json:"provider"`
	Attributes map[string]interface{} `json:"attributes,omitempty"`

	// Machine is the tag of the machine that the volume should
	// be initially attached to, if any.
	MachineTag string `json:"machinetag,omitempty"`
}

// VolumePreparationInfo holds the information regarding preparing
// a storage volume for use.
type VolumePreparationInfo struct {
	NeedsFilesystem bool   `json:"needsfilesystem"`
	DevicePath      string `json:"devicepath"`
}

// VolumePreparationInfoResult holds a singular VolumePreparationInfo
// result, or an error.
type VolumePreparationInfoResult struct {
	Result VolumePreparationInfo `json:"result"`
	Error  *Error                `json:"error,omitempty"`
}

// VolumePreparationInfoResult holds a set of VolumePreparationInfoResults.
type VolumePreparationInfoResults struct {
	Results []VolumePreparationInfoResult `json:"results,omitempty"`
}

// VolumeAttachmentsResult holds the volume attachments for a single
// machine, or an error.
type VolumeAttachmentsResult struct {
	Attachments []VolumeAttachment `json:"attachments,omitempty"`
	Error       *Error             `json:"error,omitempty"`
}

// VolumeAttachmensResult holds a set of VolumeAttachmentsResults for
// a set of machines.
type VolumeAttachmentsResults struct {
	Results []VolumeAttachmentsResult `json:"results,omitempty"`
}

// StorageInfo holds information about storage.
type StorageInfo struct {
	// StorageTag holds tag for this storage.
	StorageTag string `json:"storagetag"`
	// OwnerTag holds tag for the owner of this storage, unit or service.
	OwnerTag string `json:"ownertag"`
	// Kind holds what kind of storage this instance is.
	Kind StorageKind `json:"kind"`
	// Attached explicitly states if this instance is attached.
	// Having this information on the struct allows to
	// have this logic in one place rather than deducing it
	// every time this instance is used.
	Attached bool `json:"attached"`
	// UnitTag holds tag for unit for attached instances.
	UnitTag string `json:"unittag,omitempty"`
	// Location holds location for provisioned attached instances.
	Location string `json:"location,omitempty"`
	// Provisioned explicitly states if this instance is provisioned.
	// Having this information on the struct allows to
	// have this logic in one place rather than deducing it
	// every time this instance is used.
	Provisioned bool `json:"provisioned,omitempty"`
}

// StorageShowResult holds information about a storage instance
// or error related to its retrieval.
type StorageShowResult struct {
	Result StorageInfo `json:"result,omitempty"`
	Error  *Error      `json:"error,omitempty"`
}

// StorageShowResults holds a collection of storage instances.
type StorageShowResults struct {
	Results []StorageShowResult `json:"results,omitempty"`
}
<commit_msg>Changed json optionality for some properties.<commit_after>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package params

import "github.com/juju/juju/storage"

// MachineBlockDevices holds a machine tag and the block devices present
// on that machine.
type MachineBlockDevices struct {
	Machine      string                `json:"machine"`
	BlockDevices []storage.BlockDevice `json:"blockdevices,omitempty"`
}

// SetMachineBlockDevices holds the arguments for recording the block
// devices present on a set of machines.
type SetMachineBlockDevices struct {
	MachineBlockDevices []MachineBlockDevices `json:"machineblockdevices"`
}

// BlockDeviceResult holds the result of an API call to retrieve details
// of a block device.
type BlockDeviceResult struct {
	Result storage.BlockDevice `json:"result"`
	Error  *Error              `json:"error,omitempty"`
}

// BlockDeviceResults holds the result of an API call to retrieve details
// of multiple block devices.
type BlockDeviceResults struct {
	Results []BlockDeviceResult `json:"results,omitempty"`
}

// BlockDevicesResult holds the result of an API call to retrieve details
// of all block devices relating to some entity.
type BlockDevicesResult struct {
	Result []storage.BlockDevice `json:"result"`
	Error  *Error                `json:"error,omitempty"`
}

// BlockDevicseResults holds the result of an API call to retrieve details
// of all block devices relating to some entities.
type BlockDevicesResults struct {
	Results []BlockDevicesResult `json:"results,omitempty"`
}

// StorageInstance describes a storage instance.
type StorageInstance struct {
	StorageTag string
	OwnerTag   string
	Kind       StorageKind
}

// StorageKind is the kind of a storage instance.
type StorageKind int

const (
	StorageKindUnknown StorageKind = iota
	StorageKindBlock
	StorageKindFilesystem
)

func (k *StorageKind) String() string {
	switch *k {
	case StorageKindBlock:
		return "block"
	case StorageKindFilesystem:
		return "file system"
	default:
		return "unknown"
	}
}

// StorageInstanceResult holds the result of an API call to retrieve details
// of a storage instance.
type StorageInstanceResult struct {
	Result StorageInstance `json:"result"`
	Error  *Error          `json:"error,omitempty"`
}

// StorageInstanceResults holds the result of an API call to retrieve details
// of multiple storage instances.
type StorageInstanceResults struct {
	Results []StorageInstanceResult `json:"results,omitempty"`
}

// StorageAttachment describes a unit's attached storage instance.
type StorageAttachment struct {
	StorageTag string
	OwnerTag   string
	UnitTag    string

	Kind     StorageKind
	Location string
	Life     Life
}

// StorageAttachmentId identifies a storage attachment by the tags of the
// related unit and storage instance.
type StorageAttachmentId struct {
	StorageTag string `json:"storagetag"`
	UnitTag    string `json:"unittag"`
}

// StorageAttachmentIds holds a set of storage attachment identifiers.
type StorageAttachmentIds struct {
	Ids []StorageAttachmentId `json:"ids"`
}

// StorageAttachmentsResult holds the result of an API call to retrieve details
// of a unit's attached storage instances.
type StorageAttachmentsResult struct {
	Result []StorageAttachment `json:"result"`
	Error  *Error              `json:"error,omitempty"`
}

// StorageAttachmentsResults holds the result of an API call to retrieve details
// of multiple units' attached storage instances.
type StorageAttachmentsResults struct {
	Results []StorageAttachmentsResult `json:"results,omitempty"`
}

// StorageAttachmentResult holds the result of an API call to retrieve details
// of a storage attachment.
type StorageAttachmentResult struct {
	Result StorageAttachment `json:"result"`
	Error  *Error            `json:"error,omitempty"`
}

// StorageAttachmentResults holds the result of an API call to retrieve details
// of multiple storage attachments.
type StorageAttachmentResults struct {
	Results []StorageAttachmentResult `json:"results,omitempty"`
}

// Volume describes a storage volume in the environment.
type Volume struct {
	VolumeTag string `json:"volumetag"`
	VolumeId  string `json:"volumeid"`
	Serial    string `json:"serial"`
	// Size is the size of the volume in MiB.
	Size uint64 `json:"size"`
}

// VolumeAttachmentId identifies a volume attachment by the tags of the
// related machine and volume.
type VolumeAttachmentId struct {
	VolumeTag  string `json:"volumetag"`
	MachineTag string `json:"machinetag"`
}

// VolumeAttachmentIds holds a set of volume attachment identifiers.
type VolumeAttachmentIds struct {
	Ids []VolumeAttachmentId `json:"ids"`
}

// VolumeAttachment describes a volume attachment.
type VolumeAttachment struct {
	VolumeTag  string `json:"volumetag"`
	MachineTag string `json:"machinetag"`
	DeviceName string `json:"devicename,omitempty"`
}

// VolumeParams holds the parameters for creating a storage volume.
type VolumeParams struct {
	VolumeTag  string                 `json:"volumetag"`
	Size       uint64                 `json:"size"`
	Provider   string                 `json:"provider"`
	Attributes map[string]interface{} `json:"attributes,omitempty"`

	// Machine is the tag of the machine that the volume should
	// be initially attached to, if any.
	MachineTag string `json:"machinetag,omitempty"`
}

// VolumePreparationInfo holds the information regarding preparing
// a storage volume for use.
type VolumePreparationInfo struct {
	NeedsFilesystem bool   `json:"needsfilesystem"`
	DevicePath      string `json:"devicepath"`
}

// VolumePreparationInfoResult holds a singular VolumePreparationInfo
// result, or an error.
type VolumePreparationInfoResult struct {
	Result VolumePreparationInfo `json:"result"`
	Error  *Error                `json:"error,omitempty"`
}

// VolumePreparationInfoResult holds a set of VolumePreparationInfoResults.
type VolumePreparationInfoResults struct {
	Results []VolumePreparationInfoResult `json:"results,omitempty"`
}

// VolumeAttachmentsResult holds the volume attachments for a single
// machine, or an error.
type VolumeAttachmentsResult struct {
	Attachments []VolumeAttachment `json:"attachments,omitempty"`
	Error       *Error             `json:"error,omitempty"`
}

// VolumeAttachmensResult holds a set of VolumeAttachmentsResults for
// a set of machines.
type VolumeAttachmentsResults struct {
	Results []VolumeAttachmentsResult `json:"results,omitempty"`
}

// StorageInfo holds information about storage.
type StorageInfo struct {
	// StorageTag holds tag for this storage.
	StorageTag string `json:"storagetag"`
	// OwnerTag holds tag for the owner of this storage, unit or service.
	OwnerTag string `json:"ownertag"`
	// Kind holds what kind of storage this instance is.
	Kind StorageKind `json:"kind"`
	// Attached explicitly states if this instance is attached.
	// Having this information on the struct allows to
	// have this logic in one place rather than deducing it
	// every time this instance is used.
	Attached bool `json:"attached"`
	// UnitTag holds tag for unit for attached instances.
	UnitTag string `json:"unittag,omitempty"`
	// Location holds location for provisioned attached instances.
	Location string `json:"location,omitempty"`
	// Provisioned explicitly states if this instance is provisioned.
	// Having this information on the struct allows to
	// have this logic in one place rather than deducing it
	// every time this instance is used.
	Provisioned bool `json:"provisioned"`
}

// StorageShowResult holds information about a storage instance
// or error related to its retrieval.
type StorageShowResult struct {
	Result StorageInfo `json:"result"`
	Error  *Error      `json:"error,omitempty"`
}

// StorageShowResults holds a collection of storage instances.
type StorageShowResults struct {
	Results []StorageShowResult `json:"results,omitempty"`
}
<|endoftext|>


<commit_before>package main

import (
	"archive/zip"
	"crypto/sha1"
	"encoding/hex"
	"errors"
	"fmt"
	"image"
	"image/gif"
	"image/jpeg"
	"image/png"
	"io"
	"log"
	"os"
	"path/filepath"
	"strconv"
	"time"

	"strings"

	"github.com/beevik/etree"
	zglob "github.com/mattn/go-zglob"
	"github.com/nfnt/resize"
	"golang.org/x/tools/godoc/vfs/zipfs"
)

// Series represents a book series
type Series struct {
	Name  string  `json:"name,omitempty"`
	ID    string  `json:"id,omitempty"`
	Index float64 `json:"index,omitempty"`
}

// Book represents a book
type Book struct {
	ID          string    `json:"id"`
	Title       string    `json:"title"`
	Author      string    `json:"author,omitempty"`
	AuthorID    string    `json:"authorid"`
	Publisher   string    `json:"publisher,omitempty"`
	Description string    `json:"description,omitempty"`
	Series      Series    `json:"series,omitempty"`
	Filepath    string    `json:"filepath"`
	HasCover    bool      `json:"hascover"`
	ModTime     time.Time `json:"modtime,omitempty"`
	FileType    string    `json:"filetype,omitempty"`
}

// NewBookFromFile creates a book object from a file
func NewBookFromFile(path, coverpath string) (*Book, error) {
	book := new(Book)
	book.Title = filepath.Base(path)
	book.Filepath = path
	book.FileType = strings.ToLower(strings.Replace(filepath.Ext(path), ".", "", -1))

	if file, err := os.Stat(path); err == nil {
		book.ModTime = file.ModTime()
	}

	processed := false

	switch ft := book.FileType; ft {
	case "epub":
		zr, err := zip.OpenReader(path)
		if err != nil {
			return nil, err
		}

		zfs := zipfs.New(zr, "epub")

		rsk, err := zfs.Open("/META-INF/container.xml")
		if err != nil {
			return nil, err
		}
		defer rsk.Close()
		container := etree.NewDocument()
		_, err = container.ReadFrom(rsk)
		if err != nil {
			return nil, err
		}
		rootfile := ""
		for _, e := range container.FindElements("//rootfiles/rootfile[@full-path]") {
			rootfile = e.SelectAttrValue("full-path", "")
		}
		if rootfile == "" {
			return nil, errors.New("Cannot parse container")
		}

		rrsk, err := zfs.Open("/" + rootfile)
		if err != nil {
			return nil, err
		}
		defer rrsk.Close()
		opfdir := filepath.Dir(rootfile)
		opf := etree.NewDocument()
		_, err = opf.ReadFrom(rrsk)
		if err != nil {
			return nil, err
		}
		book.Title = filepath.Base(path)
		for _, e := range opf.FindElements("//title") {
			book.Title = e.Text()
			break
		}
		for _, e := range opf.FindElements("//creator") {
			book.Author = e.Text()
			break
		}
		for _, e := range opf.FindElements("//publisher") {
			book.Publisher = e.Text()
			break
		}
		for _, e := range opf.FindElements("//description") {
			book.Description = e.Text()
			break
		}
		for _, e := range opf.FindElements("//meta[@name='calibre:series']") {
			book.Series.Name = e.SelectAttrValue("content", "")
			seriesid := sha1.New()
			io.WriteString(seriesid, book.Series.Name)
			book.Series.ID = hex.EncodeToString(seriesid.Sum(nil))[:10]
			break
		}
		for _, e := range opf.FindElements("//meta[@name='calibre:series_index']") {
			i, err := strconv.ParseFloat(e.SelectAttrValue("content", "0"), 64)
			if err == nil {
				book.Series.Index = i
				break
			}
		}

		id := sha1.New()
		io.WriteString(id, book.Author)
		book.AuthorID = hex.EncodeToString(id.Sum(nil))[:10]
		io.WriteString(id, book.Series.Name)
		io.WriteString(id, book.Title)
		book.ID = hex.EncodeToString(id.Sum(nil))[:10]

		for _, e := range opf.FindElements("//meta[@name='cover']") {
			coverid := e.SelectAttrValue("content", "")
			if coverid != "" {
				for _, f := range opf.FindElements("//[@id='" + coverid + "']") {
					cover := f.SelectAttrValue("href", "")
					if cover != "" {
						cr, err := zfs.Open("/" + opfdir + "/" + cover)
						if err != nil {
							continue
						}
						defer cr.Close()

						ext := filepath.Ext(cover)
						if ext == ".jpeg" {
							ext = ".jpg"
						}
						cpath := filepath.Join(coverpath, book.ID+".jpg")
						thumbpath := filepath.Join(coverpath, book.ID+"_thumb"+".jpg")

						var img image.Image

						switch ext {
						case ".jpg":
							img, err = jpeg.Decode(cr)
							if err != nil {
								continue
							}
						case ".gif":
							img, err = gif.Decode(cr)
							if err != nil {
								continue
							}
						case ".png":
							img, err = png.Decode(cr)
							if err != nil {
								continue
							}
						}

						coverfile, err := os.Create(cpath)
						if err != nil {
							continue
						}
						defer coverfile.Close()
						err = jpeg.Encode(coverfile, img, nil)
						if err != nil {
							continue
						}

						// Better quality: thumb := resize.Resize(200, 0, img, resize.Lanczos2)
						thumb := resize.Resize(200, 0, img, resize.Bicubic)
						thumbfile, err := os.Create(thumbpath)
						if err != nil {
							continue
						}
						defer thumbfile.Close()
						err = jpeg.Encode(thumbfile, thumb, nil)
						if err != nil {
							continue
						}
						book.HasCover = true
						break
					}
				}
				break
			}
		}
		processed = true
	}

	if !processed {
		return nil, fmt.Errorf("Unknown filetype: %s", book.FileType)
	}

	return book, nil
}

// BookList is a slice of books
type BookList []Book

// NewBookListFromDir creates a BookList from the books in a dir. It will still return if there are errors indexing some of the books.
func NewBookListFromDir(path, coverdir string, printlog bool) (*BookList, error) {
	matches, err := zglob.Glob(filepath.Join(path, "/**/*.epub"))
	if err != nil {
		return nil, err
	}

	var books BookList
	for i, filename := range matches {
		if printlog {
			log.Printf("%.f%% Indexing %s\n", float64(i)/float64(len(matches))*100, filename)
		}
		book, err := NewBookFromFile(filename, coverdir)
		if err != nil {
			if printlog {
				log.Printf("Error indexing %s: %s\n", filename, err)
			}
			continue
		}
		books = append(books, *book)
	}
	return &books, nil
}
<commit_msg>Added initial PDF support<commit_after>package main

import (
	"archive/zip"
	"crypto/sha1"
	"encoding/hex"
	"errors"
	"fmt"
	"image"
	"image/gif"
	"image/jpeg"
	"image/png"
	"io"
	"log"
	"os"
	"path/filepath"
	"strconv"
	"time"

	"strings"

	"github.com/beevik/etree"
	zglob "github.com/mattn/go-zglob"
	"github.com/nfnt/resize"
	"golang.org/x/tools/godoc/vfs/zipfs"
)

// Series represents a book series
type Series struct {
	Name  string  `json:"name,omitempty"`
	ID    string  `json:"id,omitempty"`
	Index float64 `json:"index,omitempty"`
}

// Book represents a book
type Book struct {
	ID          string    `json:"id"`
	Title       string    `json:"title"`
	Author      string    `json:"author,omitempty"`
	AuthorID    string    `json:"authorid"`
	Publisher   string    `json:"publisher,omitempty"`
	Description string    `json:"description,omitempty"`
	Series      Series    `json:"series,omitempty"`
	Filepath    string    `json:"filepath"`
	HasCover    bool      `json:"hascover"`
	ModTime     time.Time `json:"modtime,omitempty"`
	FileType    string    `json:"filetype,omitempty"`
}

// NewBookFromFile creates a book object from a file
func NewBookFromFile(path, coverpath string) (*Book, error) {
	book := new(Book)
	book.Title = filepath.Base(path)
	book.Filepath = path
	book.FileType = strings.ToLower(strings.Replace(filepath.Ext(path), ".", "", -1))

	if file, err := os.Stat(path); err == nil {
		book.ModTime = file.ModTime()
	}

	processed := false

	switch ft := book.FileType; ft {
	case "pdf":
		book.Title = filepath.Base(path)

		id := sha1.New()
		io.WriteString(id, book.Author)
		book.AuthorID = hex.EncodeToString(id.Sum(nil))[:10]
		io.WriteString(id, book.Series.Name)
		io.WriteString(id, book.Title)
		book.ID = hex.EncodeToString(id.Sum(nil))[:10]

		processed = true
	case "epub":
		zr, err := zip.OpenReader(path)
		if err != nil {
			return nil, err
		}

		zfs := zipfs.New(zr, "epub")

		rsk, err := zfs.Open("/META-INF/container.xml")
		if err != nil {
			return nil, err
		}
		defer rsk.Close()
		container := etree.NewDocument()
		_, err = container.ReadFrom(rsk)
		if err != nil {
			return nil, err
		}
		rootfile := ""
		for _, e := range container.FindElements("//rootfiles/rootfile[@full-path]") {
			rootfile = e.SelectAttrValue("full-path", "")
		}
		if rootfile == "" {
			return nil, errors.New("Cannot parse container")
		}

		rrsk, err := zfs.Open("/" + rootfile)
		if err != nil {
			return nil, err
		}
		defer rrsk.Close()
		opfdir := filepath.Dir(rootfile)
		opf := etree.NewDocument()
		_, err = opf.ReadFrom(rrsk)
		if err != nil {
			return nil, err
		}
		book.Title = filepath.Base(path)
		for _, e := range opf.FindElements("//title") {
			book.Title = e.Text()
			break
		}
		for _, e := range opf.FindElements("//creator") {
			book.Author = e.Text()
			break
		}
		for _, e := range opf.FindElements("//publisher") {
			book.Publisher = e.Text()
			break
		}
		for _, e := range opf.FindElements("//description") {
			book.Description = e.Text()
			break
		}
		for _, e := range opf.FindElements("//meta[@name='calibre:series']") {
			book.Series.Name = e.SelectAttrValue("content", "")
			seriesid := sha1.New()
			io.WriteString(seriesid, book.Series.Name)
			book.Series.ID = hex.EncodeToString(seriesid.Sum(nil))[:10]
			break
		}
		for _, e := range opf.FindElements("//meta[@name='calibre:series_index']") {
			i, err := strconv.ParseFloat(e.SelectAttrValue("content", "0"), 64)
			if err == nil {
				book.Series.Index = i
				break
			}
		}

		id := sha1.New()
		io.WriteString(id, book.Author)
		book.AuthorID = hex.EncodeToString(id.Sum(nil))[:10]
		io.WriteString(id, book.Series.Name)
		io.WriteString(id, book.Title)
		book.ID = hex.EncodeToString(id.Sum(nil))[:10]

		for _, e := range opf.FindElements("//meta[@name='cover']") {
			coverid := e.SelectAttrValue("content", "")
			if coverid != "" {
				for _, f := range opf.FindElements("//[@id='" + coverid + "']") {
					cover := f.SelectAttrValue("href", "")
					if cover != "" {
						cr, err := zfs.Open("/" + opfdir + "/" + cover)
						if err != nil {
							continue
						}
						defer cr.Close()

						ext := filepath.Ext(cover)
						if ext == ".jpeg" {
							ext = ".jpg"
						}
						cpath := filepath.Join(coverpath, book.ID+".jpg")
						thumbpath := filepath.Join(coverpath, book.ID+"_thumb"+".jpg")

						var img image.Image

						switch ext {
						case ".jpg":
							img, err = jpeg.Decode(cr)
							if err != nil {
								continue
							}
						case ".gif":
							img, err = gif.Decode(cr)
							if err != nil {
								continue
							}
						case ".png":
							img, err = png.Decode(cr)
							if err != nil {
								continue
							}
						}

						coverfile, err := os.Create(cpath)
						if err != nil {
							continue
						}
						defer coverfile.Close()
						err = jpeg.Encode(coverfile, img, nil)
						if err != nil {
							continue
						}

						// Better quality: thumb := resize.Resize(200, 0, img, resize.Lanczos2)
						thumb := resize.Resize(200, 0, img, resize.Bicubic)
						thumbfile, err := os.Create(thumbpath)
						if err != nil {
							continue
						}
						defer thumbfile.Close()
						err = jpeg.Encode(thumbfile, thumb, nil)
						if err != nil {
							continue
						}
						book.HasCover = true
						break
					}
				}
				break
			}
		}
		processed = true
	}

	if !processed {
		return nil, fmt.Errorf("Unknown filetype: %s", book.FileType)
	}

	return book, nil
}

// BookList is a slice of books
type BookList []Book

// NewBookListFromDir creates a BookList from the books in a dir. It will still return if there are errors indexing some of the books.
func NewBookListFromDir(path, coverdir string, printlog bool) (*BookList, error) {
	matches, err := zglob.Glob(filepath.Join(path, "/**/*.epub"))
	if err != nil {
		return nil, err
	}

	pdfmatches, err := zglob.Glob(filepath.Join(path, "/**/*.pdf"))
	if err != nil {
		return nil, err
	}
	matches = append(matches, pdfmatches...)

	var books BookList
	for i, filename := range matches {
		if printlog {
			log.Printf("%.f%% Indexing %s\n", float64(i)/float64(len(matches))*100, filename)
		}
		book, err := NewBookFromFile(filename, coverdir)
		if err != nil {
			if printlog {
				log.Printf("Error indexing %s: %s\n", filename, err)
			}
			continue
		}
		books = append(books, *book)
	}
	return &books, nil
}
<|endoftext|>


<commit_before>package main

import (
	"archive/zip"
	"crypto/sha1"
	"encoding/hex"
	"errors"
	"fmt"
	"image"
	"image/gif"
	"image/jpeg"
	"image/png"
	"io"
	"log"
	"os"
	"path/filepath"
	"strconv"
	"time"

	"strings"

	"github.com/beevik/etree"
	zglob "github.com/mattn/go-zglob"
	"github.com/nfnt/resize"
	"golang.org/x/tools/godoc/vfs/zipfs"
)

// Series represents a book series
type Series struct {
	Name  string  `json:"name,omitempty"`
	ID    string  `json:"id,omitempty"`
	Index float64 `json:"index,omitempty"`
}

// Book represents a book
type Book struct {
	ID          string    `json:"id"`
	Title       string    `json:"title"`
	Author      string    `json:"author,omitempty"`
	AuthorID    string    `json:"authorid"`
	Publisher   string    `json:"publisher,omitempty"`
	Description string    `json:"description,omitempty"`
	Series      Series    `json:"series,omitempty"`
	Filepath    string    `json:"filepath"`
	HasCover    bool      `json:"hascover"`
	ModTime     time.Time `json:"modtime,omitempty"`
	FileType    string    `json:"filetype,omitempty"`
}

// NewBookFromFile creates a book object from a file
func NewBookFromFile(path, coverpath string) (bk *Book, err error) {
	defer func() {
		if r := recover(); r != nil {
			bk = nil
			err = fmt.Errorf("Unknown error parsing book. Skipping. Error: %s", r)
		}
	}()

	book := new(Book)
	book.Title = filepath.Base(path)
	book.Filepath = path
	book.FileType = strings.ToLower(strings.Replace(filepath.Ext(path), ".", "", -1))

	if file, err := os.Stat(path); err == nil {
		book.ModTime = file.ModTime()
	}

	switch ft := book.FileType; ft {
	case "pdf":
		book.Title = filepath.Base(path)

		m, err := GetPDFMeta(path)
		if err == nil {
			book.Title = m.Title
			book.Author = m.Author
		}

		id := sha1.New()
		io.WriteString(id, book.Author)
		book.AuthorID = hex.EncodeToString(id.Sum(nil))[:10]
		io.WriteString(id, book.Series.Name)
		io.WriteString(id, book.Title)
		book.ID = hex.EncodeToString(id.Sum(nil))[:10]
	case "epub":
		zr, err := zip.OpenReader(path)
		if err != nil {
			return nil, err
		}

		zfs := zipfs.New(zr, "epub")

		rsk, err := zfs.Open("/META-INF/container.xml")
		if err != nil {
			return nil, err
		}
		defer rsk.Close()
		container := etree.NewDocument()
		_, err = container.ReadFrom(rsk)
		if err != nil {
			return nil, err
		}
		rootfile := ""
		for _, e := range container.FindElements("//rootfiles/rootfile[@full-path]") {
			rootfile = e.SelectAttrValue("full-path", "")
		}
		if rootfile == "" {
			return nil, errors.New("Cannot parse container")
		}

		rrsk, err := zfs.Open("/" + rootfile)
		if err != nil {
			return nil, err
		}
		defer rrsk.Close()
		opfdir := filepath.Dir(rootfile)
		opf := etree.NewDocument()
		_, err = opf.ReadFrom(rrsk)
		if err != nil {
			return nil, err
		}
		book.Title = filepath.Base(path)
		for _, e := range opf.FindElements("//title") {
			book.Title = e.Text()
			break
		}
		for _, e := range opf.FindElements("//creator") {
			book.Author = e.Text()
			break
		}
		for _, e := range opf.FindElements("//publisher") {
			book.Publisher = e.Text()
			break
		}
		for _, e := range opf.FindElements("//description") {
			book.Description = e.Text()
			break
		}
		for _, e := range opf.FindElements("//meta[@name='calibre:series']") {
			book.Series.Name = e.SelectAttrValue("content", "")
			seriesid := sha1.New()
			io.WriteString(seriesid, book.Series.Name)
			book.Series.ID = hex.EncodeToString(seriesid.Sum(nil))[:10]
			break
		}
		for _, e := range opf.FindElements("//meta[@name='calibre:series_index']") {
			i, err := strconv.ParseFloat(e.SelectAttrValue("content", "0"), 64)
			if err == nil {
				book.Series.Index = i
				break
			}
		}

		id := sha1.New()
		io.WriteString(id, book.Author)
		book.AuthorID = hex.EncodeToString(id.Sum(nil))[:10]
		io.WriteString(id, book.Series.Name)
		io.WriteString(id, book.Title)
		book.ID = hex.EncodeToString(id.Sum(nil))[:10]

		for _, e := range opf.FindElements("//meta[@name='cover']") {
			coverid := e.SelectAttrValue("content", "")
			if coverid != "" {
				for _, f := range opf.FindElements("//[@id='" + coverid + "']") {
					cover := f.SelectAttrValue("href", "")
					if cover != "" {
						cr, err := zfs.Open("/" + opfdir + "/" + cover)
						if err != nil {
							continue
						}
						defer cr.Close()

						ext := filepath.Ext(cover)
						if ext == ".jpeg" {
							ext = ".jpg"
						}
						cpath := filepath.Join(coverpath, book.ID+".jpg")
						thumbpath := filepath.Join(coverpath, book.ID+"_thumb"+".jpg")

						var img image.Image

						switch ext {
						case ".jpg":
							img, err = jpeg.Decode(cr)
							if err != nil {
								continue
							}
						case ".gif":
							img, err = gif.Decode(cr)
							if err != nil {
								continue
							}
						case ".png":
							img, err = png.Decode(cr)
							if err != nil {
								continue
							}
						}

						coverfile, err := os.Create(cpath)
						if err != nil {
							continue
						}
						defer coverfile.Close()
						err = jpeg.Encode(coverfile, img, nil)
						if err != nil {
							continue
						}

						// Better quality: thumb := resize.Resize(200, 0, img, resize.Lanczos2)
						thumb := resize.Resize(200, 0, img, resize.Bicubic)
						thumbfile, err := os.Create(thumbpath)
						if err != nil {
							continue
						}
						defer thumbfile.Close()
						err = jpeg.Encode(thumbfile, thumb, nil)
						if err != nil {
							continue
						}
						book.HasCover = true
						break
					}
				}
				break
			}
		}
	default:
		return nil, fmt.Errorf("Unknown filetype: %s", book.FileType)
	}

	return book, nil
}

// BookList is a slice of books
type BookList []Book

// NewBookListFromDir creates a BookList from the books in a dir. It will still return if there are errors indexing some of the books.
func NewBookListFromDir(path, coverdir string, printlog bool) (*BookList, error) {
	matches, err := zglob.Glob(filepath.Join(path, "/**/*.epub"))
	if err != nil {
		return nil, err
	}

	pdfmatches, err := zglob.Glob(filepath.Join(path, "/**/*.pdf"))
	if err != nil {
		return nil, err
	}
	matches = append(matches, pdfmatches...)

	var books BookList
	for i, filename := range matches {
		if printlog {
			log.Printf("%.f%% Indexing %s\n", float64(i)/float64(len(matches))*100, filename)
		}
		book, err := NewBookFromFile(filename, coverdir)
		if err != nil {
			if printlog {
				log.Printf("Error indexing %s: %s\n", filename, err)
			}
			continue
		}
		books = append(books, *book)
	}
	return &books, nil
}
<commit_msg>Reduce memory usage<commit_after>package main

import (
	"archive/zip"
	"crypto/sha1"
	"encoding/hex"
	"errors"
	"fmt"
	"image"
	"image/gif"
	"image/jpeg"
	"image/png"
	"io"
	"log"
	"os"
	"path/filepath"
	"runtime/debug"
	"strconv"
	"time"

	"strings"

	"github.com/beevik/etree"
	zglob "github.com/mattn/go-zglob"
	"github.com/nfnt/resize"
	"golang.org/x/tools/godoc/vfs/zipfs"
)

// Series represents a book series
type Series struct {
	Name  string  `json:"name,omitempty"`
	ID    string  `json:"id,omitempty"`
	Index float64 `json:"index,omitempty"`
}

// Book represents a book
type Book struct {
	ID          string    `json:"id"`
	Title       string    `json:"title"`
	Author      string    `json:"author,omitempty"`
	AuthorID    string    `json:"authorid"`
	Publisher   string    `json:"publisher,omitempty"`
	Description string    `json:"description,omitempty"`
	Series      Series    `json:"series,omitempty"`
	Filepath    string    `json:"filepath"`
	HasCover    bool      `json:"hascover"`
	ModTime     time.Time `json:"modtime,omitempty"`
	FileType    string    `json:"filetype,omitempty"`
}

// NewBookFromFile creates a book object from a file
func NewBookFromFile(path, coverpath string) (bk *Book, err error) {
	defer func() {
		if r := recover(); r != nil {
			bk = nil
			err = fmt.Errorf("Unknown error parsing book. Skipping. Error: %s", r)
		}
	}()

	book := new(Book)
	book.Title = filepath.Base(path)
	book.Filepath = path
	book.FileType = strings.ToLower(strings.Replace(filepath.Ext(path), ".", "", -1))

	if file, err := os.Stat(path); err == nil {
		book.ModTime = file.ModTime()
	}

	switch ft := book.FileType; ft {
	case "pdf":
		book.Title = filepath.Base(path)

		m, err := GetPDFMeta(path)
		if err == nil {
			book.Title = m.Title
			book.Author = m.Author
		}

		id := sha1.New()
		io.WriteString(id, book.Author)
		book.AuthorID = hex.EncodeToString(id.Sum(nil))[:10]
		io.WriteString(id, book.Series.Name)
		io.WriteString(id, book.Title)
		book.ID = hex.EncodeToString(id.Sum(nil))[:10]
	case "epub":
		zr, err := zip.OpenReader(path)
		if err != nil {
			return nil, err
		}

		zfs := zipfs.New(zr, "epub")

		rsk, err := zfs.Open("/META-INF/container.xml")
		if err != nil {
			return nil, err
		}
		defer rsk.Close()
		container := etree.NewDocument()
		_, err = container.ReadFrom(rsk)
		if err != nil {
			return nil, err
		}
		rootfile := ""
		for _, e := range container.FindElements("//rootfiles/rootfile[@full-path]") {
			rootfile = e.SelectAttrValue("full-path", "")
		}
		if rootfile == "" {
			return nil, errors.New("Cannot parse container")
		}

		rrsk, err := zfs.Open("/" + rootfile)
		if err != nil {
			return nil, err
		}
		defer rrsk.Close()
		opfdir := filepath.Dir(rootfile)
		opf := etree.NewDocument()
		_, err = opf.ReadFrom(rrsk)
		if err != nil {
			return nil, err
		}
		book.Title = filepath.Base(path)
		for _, e := range opf.FindElements("//title") {
			book.Title = e.Text()
			break
		}
		for _, e := range opf.FindElements("//creator") {
			book.Author = e.Text()
			break
		}
		for _, e := range opf.FindElements("//publisher") {
			book.Publisher = e.Text()
			break
		}
		for _, e := range opf.FindElements("//description") {
			book.Description = e.Text()
			break
		}
		for _, e := range opf.FindElements("//meta[@name='calibre:series']") {
			book.Series.Name = e.SelectAttrValue("content", "")
			seriesid := sha1.New()
			io.WriteString(seriesid, book.Series.Name)
			book.Series.ID = hex.EncodeToString(seriesid.Sum(nil))[:10]
			break
		}
		for _, e := range opf.FindElements("//meta[@name='calibre:series_index']") {
			i, err := strconv.ParseFloat(e.SelectAttrValue("content", "0"), 64)
			if err == nil {
				book.Series.Index = i
				break
			}
		}

		id := sha1.New()
		io.WriteString(id, book.Author)
		book.AuthorID = hex.EncodeToString(id.Sum(nil))[:10]
		io.WriteString(id, book.Series.Name)
		io.WriteString(id, book.Title)
		book.ID = hex.EncodeToString(id.Sum(nil))[:10]

		for _, e := range opf.FindElements("//meta[@name='cover']") {
			coverid := e.SelectAttrValue("content", "")
			if coverid != "" {
				for _, f := range opf.FindElements("//[@id='" + coverid + "']") {
					cover := f.SelectAttrValue("href", "")
					if cover != "" {
						cr, err := zfs.Open("/" + opfdir + "/" + cover)
						if err != nil {
							continue
						}
						defer cr.Close()

						ext := filepath.Ext(cover)
						if ext == ".jpeg" {
							ext = ".jpg"
						}
						cpath := filepath.Join(coverpath, book.ID+".jpg")
						thumbpath := filepath.Join(coverpath, book.ID+"_thumb"+".jpg")

						var img image.Image

						switch ext {
						case ".jpg":
							img, err = jpeg.Decode(cr)
							if err != nil {
								continue
							}
						case ".gif":
							img, err = gif.Decode(cr)
							if err != nil {
								continue
							}
						case ".png":
							img, err = png.Decode(cr)
							if err != nil {
								continue
							}
						}

						coverfile, err := os.Create(cpath)
						if err != nil {
							continue
						}
						defer coverfile.Close()
						err = jpeg.Encode(coverfile, img, nil)
						if err != nil {
							continue
						}

						// Better quality: thumb := resize.Resize(200, 0, img, resize.Lanczos2)
						thumb := resize.Resize(200, 0, img, resize.Bicubic)
						thumbfile, err := os.Create(thumbpath)
						if err != nil {
							continue
						}
						defer thumbfile.Close()
						err = jpeg.Encode(thumbfile, thumb, nil)
						if err != nil {
							continue
						}
						book.HasCover = true
						break
					}
				}
				break
			}
		}
	default:
		return nil, fmt.Errorf("Unknown filetype: %s", book.FileType)
	}

	return book, nil
}

// BookList is a slice of books
type BookList []Book

// NewBookListFromDir creates a BookList from the books in a dir. It will still return if there are errors indexing some of the books.
func NewBookListFromDir(path, coverdir string, printlog bool) (*BookList, error) {
	matches, err := zglob.Glob(filepath.Join(path, "/**/*.epub"))
	if err != nil {
		return nil, err
	}

	pdfmatches, err := zglob.Glob(filepath.Join(path, "/**/*.pdf"))
	if err != nil {
		return nil, err
	}
	matches = append(matches, pdfmatches...)

	var books BookList
	for i, filename := range matches {
		if printlog {
			log.Printf("%.f%% Indexing %s\n", float64(i)/float64(len(matches))*100, filename)
		}
		book, err := NewBookFromFile(filename, coverdir)
		if err != nil {
			if printlog {
				log.Printf("Error indexing %s: %s\n", filename, err)
			}
			continue
		}
		books = append(books, *book)
	}
	debug.FreeOSMemory()
	return &books, nil
}
<|endoftext|>


<commit_before>package gofakeit

import "math/rand"

func Bool() bool {
	if rand.Intn(2) == 1 {
		return true
	}

	return false
}
<commit_msg>bool - added description comment.<commit_after>package gofakeit

import "math/rand"

// Generate Random Boolean value
func Bool() bool {
	if rand.Intn(2) == 1 {
		return true
	}

	return false
}
<|endoftext|>


<commit_before>package mainline

import (
	"math/rand"
	"net"
	"sync"
	"time"

	"go.uber.org/zap"
)

type IndexingService struct {
	// Private
	protocol      *Protocol
	started       bool
	interval      time.Duration
	eventHandlers IndexingServiceEventHandlers

	nodeID []byte
	// []byte type would be a much better fit for the keys but unfortunately (and quite
	// understandably) slices cannot be used as keys (since they are not hashable), and using arrays
	// (or even the conversion between each other) is a pain; hence map[string]net.UDPAddr
	//                                                                  ^~~~~~
	routingTable      map[string]*net.UDPAddr
	routingTableMutex *sync.Mutex
	maxNeighbors      uint

	counter          uint16
	getPeersRequests map[[2]byte][20]byte // GetPeersQuery.`t` -> infohash
}

type IndexingServiceEventHandlers struct {
	OnResult func(IndexingResult)
}

type IndexingResult struct {
	infoHash  [20]byte
	peerAddrs []net.TCPAddr
}

func (ir IndexingResult) InfoHash() [20]byte {
	return ir.infoHash
}

func (ir IndexingResult) PeerAddrs() []net.TCPAddr {
	return ir.peerAddrs
}

func NewIndexingService(laddr string, interval time.Duration, maxNeighbors uint, eventHandlers IndexingServiceEventHandlers) *IndexingService {
	service := new(IndexingService)
	service.interval = interval
	service.protocol = NewProtocol(
		laddr,
		ProtocolEventHandlers{
			OnFindNodeResponse:         service.onFindNodeResponse,
			OnGetPeersResponse:         service.onGetPeersResponse,
			OnSampleInfohashesResponse: service.onSampleInfohashesResponse,
		},
	)
	service.nodeID = make([]byte, 20)
	service.routingTable = make(map[string]*net.UDPAddr)
	service.routingTableMutex = new(sync.Mutex)
	service.maxNeighbors = maxNeighbors
	service.eventHandlers = eventHandlers

	service.getPeersRequests = make(map[[2]byte][20]byte)

	return service
}

func (is *IndexingService) Start() {
	if is.started {
		zap.L().Panic("Attempting to Start() a mainline/IndexingService that has been already started! (Programmer error.)")
	}
	is.started = true

	is.protocol.Start()
	go is.index()

	zap.L().Info("Indexing Service started!")
}

func (is *IndexingService) Terminate() {
	is.protocol.Terminate()
}

func (is *IndexingService) index() {
	for range time.Tick(is.interval) {
		is.routingTableMutex.Lock()
		if len(is.routingTable) == 0 {
			is.bootstrap()
		} else {
			zap.L().Info("Latest status:", zap.Int("n", len(is.routingTable)),
				zap.Uint("maxNeighbors", is.maxNeighbors))
			//TODO
			is.findNeighbors()
			is.routingTable = make(map[string]*net.UDPAddr)
		}
		is.routingTableMutex.Unlock()
	}
}

func (is *IndexingService) bootstrap() {
	bootstrappingNodes := []string{
		"router.bittorrent.com:6881",
		"dht.transmissionbt.com:6881",
		"dht.libtorrent.org:25401",
	}

	zap.L().Info("Bootstrapping as routing table is empty...")
	for _, node := range bootstrappingNodes {
		target := make([]byte, 20)
		_, err := rand.Read(target)
		if err != nil {
			zap.L().Panic("Could NOT generate random bytes during bootstrapping!")
		}

		addr, err := net.ResolveUDPAddr("udp", node)
		if err != nil {
			zap.L().Error("Could NOT resolve (UDP) address of the bootstrapping node!",
				zap.String("node", node))
			continue
		}

		is.protocol.SendMessage(NewFindNodeQuery(is.nodeID, target), addr)
	}
}

func (is *IndexingService) findNeighbors() {
	target := make([]byte, 20)
	for _, addr := range is.routingTable {
		_, err := rand.Read(target)
		if err != nil {
			zap.L().Panic("Could NOT generate random bytes during bootstrapping!")
		}

		is.protocol.SendMessage(
			NewSampleInfohashesQuery(is.nodeID, []byte("aa"), target),
			addr,
		)
	}
}

func (is *IndexingService) onFindNodeResponse(response *Message, addr *net.UDPAddr) {
	is.routingTableMutex.Lock()
	defer is.routingTableMutex.Unlock()

	for _, node := range response.R.Nodes {
		if uint(len(is.routingTable)) >= is.maxNeighbors {
			break
		}
		if node.Addr.Port == 0 { // Ignore nodes who "use" port 0.
			continue
		}

		is.routingTable[string(node.ID)] = &node.Addr

		target := make([]byte, 20)
		_, err := rand.Read(target)
		if err != nil {
			zap.L().Panic("Could NOT generate random bytes!")
		}
		is.protocol.SendMessage(
			NewSampleInfohashesQuery(is.nodeID, []byte("aa"), target),
			&node.Addr,
		)
	}
}

func (is *IndexingService) onGetPeersResponse(msg *Message, addr *net.UDPAddr) {
	var t [2]byte
	copy(t[:], msg.T)

	infoHash := is.getPeersRequests[t]
	// We got a response, so free the key!
	delete(is.getPeersRequests, t)

	// BEP 51 specifies that
	//     The new sample_infohashes remote procedure call requests that a remote node return a string of multiple
	//     concatenated infohashes (20 bytes each) FOR WHICH IT HOLDS GET_PEERS VALUES.
	//                                                                          ^^^^^^
	// So theoretically we should never hit the case where `values` is empty, but c'est la vie.
	if len(msg.R.Values) == 0 {
		return
	}

	peerAddrs := make([]net.TCPAddr, 0)
	for _, peer := range msg.R.Values {
		if peer.Port == 0 {
			continue
		}

		peerAddrs = append(peerAddrs, net.TCPAddr{
			IP:   peer.IP,
			Port: peer.Port,
		})
	}

	is.eventHandlers.OnResult(IndexingResult{
		infoHash:  infoHash,
		peerAddrs: peerAddrs,
	})
}

func (is *IndexingService) onSampleInfohashesResponse(msg *Message, addr *net.UDPAddr) {
	// request samples
	for i := 0; i < len(msg.R.Samples)/20; i++ {
		var infoHash [20]byte
		copy(infoHash[:], msg.R.Samples[i:(i+1)*20])

		msg := NewGetPeersQuery(is.nodeID, infoHash[:])
		t := uint16BE(is.counter)
		msg.T = t[:]

		is.protocol.SendMessage(msg, addr)

		is.getPeersRequests[t] = infoHash
		is.counter++
	}

	// iterate
	for _, node := range msg.R.Nodes {
		if uint(len(is.routingTable)) >= is.maxNeighbors {
			break
		}
		if node.Addr.Port == 0 { // Ignore nodes who "use" port 0.
			continue
		}
		is.routingTable[string(node.ID)] = &node.Addr

		// TODO
		/*
			target := make([]byte, 20)
			_, err := rand.Read(target)
			if err != nil {
				zap.L().Panic("Could NOT generate random bytes!")
			}
			is.protocol.SendMessage(
				NewSampleInfohashesQuery(is.nodeID, []byte("aa"), target),
				&node.Addr,
			)
		*/
	}
}

func uint16BE(v uint16) (b [2]byte) {
	b[0] = byte(v >> 8)
	b[1] = byte(v)
	return
}
<commit_msg>fixed concurrency on routingTable map<commit_after>package mainline

import (
	"math/rand"
	"net"
	"sync"
	"time"

	"go.uber.org/zap"
)

type IndexingService struct {
	// Private
	protocol      *Protocol
	started       bool
	interval      time.Duration
	eventHandlers IndexingServiceEventHandlers

	nodeID []byte
	// []byte type would be a much better fit for the keys but unfortunately (and quite
	// understandably) slices cannot be used as keys (since they are not hashable), and using arrays
	// (or even the conversion between each other) is a pain; hence map[string]net.UDPAddr
	//                                                                  ^~~~~~
	routingTable      map[string]*net.UDPAddr
	routingTableMutex sync.RWMutex
	maxNeighbors      uint

	counter          uint16
	getPeersRequests map[[2]byte][20]byte // GetPeersQuery.`t` -> infohash
}

type IndexingServiceEventHandlers struct {
	OnResult func(IndexingResult)
}

type IndexingResult struct {
	infoHash  [20]byte
	peerAddrs []net.TCPAddr
}

func (ir IndexingResult) InfoHash() [20]byte {
	return ir.infoHash
}

func (ir IndexingResult) PeerAddrs() []net.TCPAddr {
	return ir.peerAddrs
}

func NewIndexingService(laddr string, interval time.Duration, maxNeighbors uint, eventHandlers IndexingServiceEventHandlers) *IndexingService {
	service := new(IndexingService)
	service.interval = interval
	service.protocol = NewProtocol(
		laddr,
		ProtocolEventHandlers{
			OnFindNodeResponse:         service.onFindNodeResponse,
			OnGetPeersResponse:         service.onGetPeersResponse,
			OnSampleInfohashesResponse: service.onSampleInfohashesResponse,
		},
	)
	service.nodeID = make([]byte, 20)
	service.routingTable = make(map[string]*net.UDPAddr)
	service.maxNeighbors = maxNeighbors
	service.eventHandlers = eventHandlers

	service.getPeersRequests = make(map[[2]byte][20]byte)

	return service
}

func (is *IndexingService) Start() {
	if is.started {
		zap.L().Panic("Attempting to Start() a mainline/IndexingService that has been already started! (Programmer error.)")
	}
	is.started = true

	is.protocol.Start()
	go is.index()

	zap.L().Info("Indexing Service started!")
}

func (is *IndexingService) Terminate() {
	is.protocol.Terminate()
}

func (is *IndexingService) index() {
	for range time.Tick(is.interval) {
		is.routingTableMutex.RLock()
		routingTableLen := len(is.routingTable)
		is.routingTableMutex.RUnlock()
		if routingTableLen == 0 {
			is.bootstrap()
		} else {
			zap.L().Info("Latest status:", zap.Int("n", routingTableLen),
				zap.Uint("maxNeighbors", is.maxNeighbors))
			//TODO
			is.findNeighbors()
			is.routingTableMutex.Lock()
			is.routingTable = make(map[string]*net.UDPAddr)
			is.routingTableMutex.Unlock()
		}
	}
}

func (is *IndexingService) bootstrap() {
	bootstrappingNodes := []string{
		"router.bittorrent.com:6881",
		"dht.transmissionbt.com:6881",
		"dht.libtorrent.org:25401",
	}

	zap.L().Info("Bootstrapping as routing table is empty...")
	for _, node := range bootstrappingNodes {
		target := make([]byte, 20)
		_, err := rand.Read(target)
		if err != nil {
			zap.L().Panic("Could NOT generate random bytes during bootstrapping!")
		}

		addr, err := net.ResolveUDPAddr("udp", node)
		if err != nil {
			zap.L().Error("Could NOT resolve (UDP) address of the bootstrapping node!",
				zap.String("node", node))
			continue
		}

		is.protocol.SendMessage(NewFindNodeQuery(is.nodeID, target), addr)
	}
}

func (is *IndexingService) findNeighbors() {
	target := make([]byte, 20)

	/*
		We could just RLock and defer RUnlock here, but that would mean that each response that we get could not Lock
		the table because we are sending. So we would basically make read and write NOT concurrent.
		A better approach would be to get all addresses to send in a slice and then work on that, releasing the main map.
	*/
	is.routingTableMutex.RLock()
	addressesToSend := make([]*net.UDPAddr, 0, len(is.routingTable))
	for _, addr := range is.routingTable {
		addressesToSend = append(addressesToSend, addr)
	}
	is.routingTableMutex.RUnlock()

	for _, addr := range addressesToSend {
		_, err := rand.Read(target)
		if err != nil {
			zap.L().Panic("Could NOT generate random bytes during bootstrapping!")
		}

		is.protocol.SendMessage(
			NewSampleInfohashesQuery(is.nodeID, []byte("aa"), target),
			addr,
		)
	}
}

func (is *IndexingService) onFindNodeResponse(response *Message, addr *net.UDPAddr) {
	is.routingTableMutex.Lock()
	defer is.routingTableMutex.Unlock()

	for _, node := range response.R.Nodes {
		if uint(len(is.routingTable)) >= is.maxNeighbors {
			break
		}
		if node.Addr.Port == 0 { // Ignore nodes who "use" port 0.
			continue
		}

		is.routingTable[string(node.ID)] = &node.Addr

		target := make([]byte, 20)
		_, err := rand.Read(target)
		if err != nil {
			zap.L().Panic("Could NOT generate random bytes!")
		}
		is.protocol.SendMessage(
			NewSampleInfohashesQuery(is.nodeID, []byte("aa"), target),
			&node.Addr,
		)
	}
}

func (is *IndexingService) onGetPeersResponse(msg *Message, addr *net.UDPAddr) {
	var t [2]byte
	copy(t[:], msg.T)

	infoHash := is.getPeersRequests[t]
	// We got a response, so free the key!
	delete(is.getPeersRequests, t)

	// BEP 51 specifies that
	//     The new sample_infohashes remote procedure call requests that a remote node return a string of multiple
	//     concatenated infohashes (20 bytes each) FOR WHICH IT HOLDS GET_PEERS VALUES.
	//                                                                          ^^^^^^
	// So theoretically we should never hit the case where `values` is empty, but c'est la vie.
	if len(msg.R.Values) == 0 {
		return
	}

	peerAddrs := make([]net.TCPAddr, 0)
	for _, peer := range msg.R.Values {
		if peer.Port == 0 {
			continue
		}

		peerAddrs = append(peerAddrs, net.TCPAddr{
			IP:   peer.IP,
			Port: peer.Port,
		})
	}

	is.eventHandlers.OnResult(IndexingResult{
		infoHash:  infoHash,
		peerAddrs: peerAddrs,
	})
}

func (is *IndexingService) onSampleInfohashesResponse(msg *Message, addr *net.UDPAddr) {
	// request samples
	for i := 0; i < len(msg.R.Samples)/20; i++ {
		var infoHash [20]byte
		copy(infoHash[:], msg.R.Samples[i:(i+1)*20])

		msg := NewGetPeersQuery(is.nodeID, infoHash[:])
		t := uint16BE(is.counter)
		msg.T = t[:]

		is.protocol.SendMessage(msg, addr)

		is.getPeersRequests[t] = infoHash
		is.counter++
	}

	// iterate
	is.routingTableMutex.Lock()
	defer is.routingTableMutex.Unlock()
	for _, node := range msg.R.Nodes {
		if uint(len(is.routingTable)) >= is.maxNeighbors {
			break
		}
		if node.Addr.Port == 0 { // Ignore nodes who "use" port 0.
			continue
		}
		is.routingTable[string(node.ID)] = &node.Addr

		// TODO
		/*
			target := make([]byte, 20)
			_, err := rand.Read(target)
			if err != nil {
				zap.L().Panic("Could NOT generate random bytes!")
			}
			is.protocol.SendMessage(
				NewSampleInfohashesQuery(is.nodeID, []byte("aa"), target),
				&node.Addr,
			)
		*/
	}
}

func uint16BE(v uint16) (b [2]byte) {
	b[0] = byte(v >> 8)
	b[1] = byte(v)
	return
}
<|endoftext|>


<commit_before>package q

import (
	"fmt"

	"github.com/garyburd/redigo/redis"
)

// A RedisStore stores all records data into redis
type RedisStore struct {
	name string
	pool *redis.Pool
}

// RedisDataStore configures the queue with a redis data store
func RedisDataStore(name string, pool *redis.Pool) func(q *Queue) error {
	return DataStore(&RedisStore{name, pool})
}

func (r *RedisStore) queue() string {
	return fmt.Sprintf("q:%s:queue", r.name)
}

func (r *RedisStore) workingQueue() string {
	return fmt.Sprintf("q:%s:queue:working", r.name)
}

// Store add the provided data to the in-memory array
func (r *RedisStore) Store(d []byte) error {
	conn := r.pool.Get()
	defer conn.Close()

	_, err := conn.Do("LPUSH", r.queue(), d)
	return err
}

// Retrieve pops the latest data from the in-memory array
func (r *RedisStore) Retrieve() ([]byte, error) {
	conn := r.pool.Get()
	defer conn.Close()

	d, err := redis.Bytes(conn.Do("RPOPLPUSH", r.queue(), r.workingQueue()))
	if err == redis.ErrNil {
		err = nil
	}
	return d, err
}

// Finish marks a task as finished
func (r *RedisStore) Finish(d []byte) error {
	conn := r.pool.Get()
	defer conn.Close()

	_, err := conn.Do("LREM", r.workingQueue(), 0, d)
	return err
}

// Length returns the number of elements in the in-memory array
func (r *RedisStore) Length() (int, error) {
	conn := r.pool.Get()
	defer conn.Close()

	return redis.Int(conn.Do("LLEN", r.queue()))
}

// WorkingLength returns the number of elements currently being processed
func (r *RedisStore) WorkingLength() (int, error) {
	conn := r.pool.Get()
	defer conn.Close()

	return redis.Int(conn.Do("LLEN", r.workingQueue()))
}
<commit_msg>setup an expiring lock key for working jobs<commit_after>package q

import (
	"crypto/sha256"
	"fmt"

	"github.com/garyburd/redigo/redis"
)

const (
	lockDuration = 60 // seconds
)

// A RedisStore stores all records data into redis
type RedisStore struct {
	name string
	pool *redis.Pool
}

// RedisDataStore configures the queue with a redis data store
func RedisDataStore(name string, pool *redis.Pool) func(q *Queue) error {
	return DataStore(&RedisStore{name, pool})
}

func (r *RedisStore) queue() string {
	return fmt.Sprintf("q:%s:queue", r.name)
}

func (r *RedisStore) workingQueue() string {
	return fmt.Sprintf("q:%s:queue:working", r.name)
}

func (r *RedisStore) lockKey(d []byte) string {
	h := sha256.New()
	h.Write(d)

	return fmt.Sprintf("q:%s:lock:%s", r.name, h.Sum(nil))
}

// Store add the provided data to the in-memory array
func (r *RedisStore) Store(d []byte) error {
	conn := r.pool.Get()
	defer conn.Close()

	_, err := conn.Do("LPUSH", r.queue(), d)
	return err
}

// Retrieve pops the latest data from the in-memory array
func (r *RedisStore) Retrieve() ([]byte, error) {
	conn := r.pool.Get()
	defer conn.Close()

	d, err := redis.Bytes(conn.Do("RPOPLPUSH", r.queue(), r.workingQueue()))
	if err == redis.ErrNil {
		err = nil
	}

	_, err = conn.Do("SETEX", r.lockKey(d), lockDuration, d)
	if err != nil {
		return nil, err
	}
	return d, err
}

// Finish marks a task as finished
func (r *RedisStore) Finish(d []byte) error {
	conn := r.pool.Get()
	defer conn.Close()

	conn.Send("MULTI")
	conn.Send("LREM", r.workingQueue(), 0, d)
	conn.Send("DEL", r.lockKey(d))
	_, err := conn.Do("EXEC")

	return err
}

// Length returns the number of elements in the in-memory array
func (r *RedisStore) Length() (int, error) {
	conn := r.pool.Get()
	defer conn.Close()

	return redis.Int(conn.Do("LLEN", r.queue()))
}

// WorkingLength returns the number of elements currently being processed
func (r *RedisStore) WorkingLength() (int, error) {
	conn := r.pool.Get()
	defer conn.Close()

	return redis.Int(conn.Do("LLEN", r.workingQueue()))
}
<|endoftext|>


<commit_before>// Copyright © 2015 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

// package pointer provides helper method to quickly define pointer from basic go types
package pointer

import (
	"fmt"
	"reflect"
	"strings"
	"time"
)

// String creates a pointer to a string from a string value
func String(v string) *string {
	p := new(string)
	*p = v
	return p
}

// Int creates a pointer to an int from an int value
func Int(v int) *int {
	p := new(int)
	*p = v
	return p
}

// Uint creates a pointer to an unsigned int from an unsigned int value
func Uint(v uint) *uint {
	p := new(uint)
	*p = v
	return p
}

// Float64 creates a pointer to a float64 from a float64 value
func Float64(v float64) *float64 {
	p := new(float64)
	*p = v
	return p
}

// Bool creates a pointer to a boolean from a boolean value
func Bool(v bool) *bool {
	p := new(bool)
	*p = v
	return p
}

// Time creates a pointer to a time.Time from a time.Time value
func Time(v time.Time) *time.Time {
	p := new(time.Time)
	*p = v
	return p
}

// DumpStruct prints the content of a struct of pointers
func DumpPStruct(s interface{}) {
	v := reflect.ValueOf(s)

	if v.Kind() != reflect.Struct {
		fmt.Printf("Unable to dump: Not a struct.")
		return
	}

	for k := 0; k < v.NumField(); k += 1 {
		name := v.Type().Field(k).Name
		if name[0] == strings.ToLower(name)[0] { // Unexported field
			continue
		}
		i := v.Field(k).Interface()
		fmt.Printf("%v: ", v.Type().Field(k).Name)

		switch t := i.(type) {
		case *bool:
			if t == nil {
				fmt.Printf("nil\n")
			} else {
				fmt.Printf("%+v\n", *t)
			}
		case *int:
			if t == nil {
				fmt.Printf("nil\n")
			} else {
				fmt.Printf("%+v\n", *t)
			}
		case *uint:
			if t == nil {
				fmt.Printf("nil\n")
			} else {
				fmt.Printf("%+v\n", *t)
			}
		case *string:
			if t == nil {
				fmt.Printf("nil\n")
			} else {
				fmt.Printf("%+v\n", *t)
			}
		case *float64:
			if t == nil {
				fmt.Printf("nil\n")
			} else {
				fmt.Printf("%+v\n", *t)
			}
		case *time.Time:
			if t == nil {
				fmt.Printf("nil\n")
			} else {
				fmt.Printf("%+v\n", *t)
			}
		default:
			fmt.Printf("unknown\n")
		}
	}
}
<commit_msg>[broker] Make pointer.DumpPStruct() more flexible.<commit_after>// Copyright © 2015 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

// package pointer provides helper method to quickly define pointer from basic go types
package pointer

import (
	"fmt"
	"reflect"
	"strings"
	"time"
)

// String creates a pointer to a string from a string value
func String(v string) *string {
	p := new(string)
	*p = v
	return p
}

// Int creates a pointer to an int from an int value
func Int(v int) *int {
	p := new(int)
	*p = v
	return p
}

// Uint creates a pointer to an unsigned int from an unsigned int value
func Uint(v uint) *uint {
	p := new(uint)
	*p = v
	return p
}

// Float64 creates a pointer to a float64 from a float64 value
func Float64(v float64) *float64 {
	p := new(float64)
	*p = v
	return p
}

// Bool creates a pointer to a boolean from a boolean value
func Bool(v bool) *bool {
	p := new(bool)
	*p = v
	return p
}

// Time creates a pointer to a time.Time from a time.Time value
func Time(v time.Time) *time.Time {
	p := new(time.Time)
	*p = v
	return p
}

// DumpStruct prints the content of a struct of pointers
func DumpPStruct(s interface{}, multiline bool) string {
	v := reflect.ValueOf(s)

	if v.Kind() != reflect.Struct {
		return "Not a struct"
	}

	nl := ", "
	str := "{ "
	if multiline {
		nl = "\n\t"
		str += nl
	}

	for k := 0; k < v.NumField(); k += 1 {
		name := v.Type().Field(k).Name
		if name[0] == strings.ToLower(name)[0] { // Unexported field
			continue
		}
		i := v.Field(k).Interface()
		key := fmt.Sprintf("%v", v.Type().Field(k).Name)

		switch t := i.(type) {
		case *bool:
			if t != nil {
				str += fmt.Sprintf("%s: %+v%s", key, *t, nl)
			}
		case *int:
			if t != nil {
				str += fmt.Sprintf("%s: %+v%s", key, *t, nl)
			}
		case *uint:
			if t != nil {
				str += fmt.Sprintf("%s: %+v%s", key, *t, nl)
			}
		case *string:
			if t != nil {
				str += fmt.Sprintf("%s: %+v%s", key, *t, nl)
			}
		case *float64:
			if t != nil {
				str += fmt.Sprintf("%s: %+v%s", key, *t, nl)
			}
		case *time.Time:
			if t != nil {
				str += fmt.Sprintf("%s: %+v%s", key, *t, nl)
			}
		default:
			str += fmt.Sprintf("%s: unknown%s", key, nl)
		}
	}

	str = str[:len(str)-2]
	if multiline {
		str += "\n}"
	} else {
		str += " }"
	}
	return str
}
<|endoftext|>


<commit_before>package db

import (
	"time"

	"github.com/couchbaselabs/go-couchbase"

	"github.com/couchbaselabs/sync_gateway/base"
)

// Unmarshaled JSON structure for "changes" view results
type channelsViewResult struct {
	TotalRows int `json:"total_rows"`
	Rows      []channelsViewRow
	Errors    []couchbase.ViewError
}

// One "changes" row in a channelsViewResult
type channelsViewRow struct {
	ID    string
	Key   []interface{} // Actually [channelName, sequence]
	Value struct {
		Rev   string
		Flags uint8
	}
}

// Queries the 'channels' view to get a range of sequences of a single channel as LogEntries.
func (dbc *DatabaseContext) getChangesInChannelFromView(
	channelName string, endSeq uint64, options ChangesOptions) (LogEntries, error) {
	start := time.Now()
	// Query the view:
	optMap := changesViewOptions(channelName, endSeq, options)
	base.LogTo("Cache", "  Querying 'channels' view for %q (start=#%d, end=#%d, limit=%d)", channelName, options.Since.Seq+1, endSeq, options.Limit)
	vres := channelsViewResult{}
	err := dbc.Bucket.ViewCustom("sync_gateway", "channels", optMap, &vres)
	if err != nil {
		base.Log("Error from 'channels' view: %v", err)
		return nil, err
	} else if len(vres.Rows) == 0 {
		base.LogTo("Cache", "    Got no rows from view for %q", channelName)
		return nil, nil
	}

	// Convert the output to LogEntries:
	entries := make(LogEntries, 0, len(vres.Rows))
	for _, row := range vres.Rows {
		entry := &LogEntry{
			Sequence:     uint64(row.Key[1].(float64)),
			DocID:        row.ID,
			RevID:        row.Value.Rev,
			Flags:        row.Value.Flags,
			TimeReceived: time.Now(),
		}
		// base.LogTo("Cache", "  Got view sequence #%d (%q / %q)", entry.Sequence, entry.DocID, entry.RevID)
		entries = append(entries, entry)
	}

	base.LogTo("Cache", "    Got %d rows from view for %q: #%d ... #%d",
		len(entries), channelName, entries[0].Sequence, entries[len(entries)-1].Sequence)
	if elapsed := time.Since(start); elapsed > 200*time.Millisecond {
		base.Log("changes_view: Query took %v to return %d rows, options = %#v",
			elapsed, len(entries), optMap)
	}
	changeCacheExpvars.Add("view_queries", 1)
	return entries, nil
}

func changesViewOptions(channelName string, endSeq uint64, options ChangesOptions) Body {
	endKey := []interface{}{channelName, endSeq}
	if endSeq == 0 {
		endKey[1] = map[string]interface{}{} // infinity
	}
	optMap := Body{
		"stale":    false,
		"startkey": []interface{}{channelName, options.Since.Seq + 1},
		"endkey":   endKey,
	}
	if options.Limit > 0 {
		optMap["limit"] = options.Limit
	}
	return optMap
}
<commit_msg>Ensure all required revisions are queried from view<commit_after>package db

import (
	"time"

	"github.com/couchbaselabs/go-couchbase"

	"github.com/couchbaselabs/sync_gateway/base"
)

// Unmarshaled JSON structure for "changes" view results
type channelsViewResult struct {
	TotalRows int `json:"total_rows"`
	Rows      []channelsViewRow
	Errors    []couchbase.ViewError
}

// One "changes" row in a channelsViewResult
type channelsViewRow struct {
	ID    string
	Key   []interface{} // Actually [channelName, sequence]
	Value struct {
		Rev   string
		Flags uint8
	}
}

// Queries the 'channels' view to get a range of sequences of a single channel as LogEntries.
func (dbc *DatabaseContext) getChangesInChannelFromView(
	channelName string, endSeq uint64, options ChangesOptions) (LogEntries, error) {
	start := time.Now()
	// Query the view:
	options.Limit = 0
	optMap := changesViewOptions(channelName, endSeq, options)
	base.LogTo("Cache", "  Querying 'channels' view for %q (start=#%d, end=#%d, limit=%d)", channelName, options.Since.Seq+1, endSeq, options.Limit)
	vres := channelsViewResult{}
	err := dbc.Bucket.ViewCustom("sync_gateway", "channels", optMap, &vres)
	if err != nil {
		base.Log("Error from 'channels' view: %v", err)
		return nil, err
	} else if len(vres.Rows) == 0 {
		base.LogTo("Cache", "    Got no rows from view for %q", channelName)
		return nil, nil
	}

	// Convert the output to LogEntries:
	entries := make(LogEntries, 0, len(vres.Rows))
	for _, row := range vres.Rows {
		entry := &LogEntry{
			Sequence:     uint64(row.Key[1].(float64)),
			DocID:        row.ID,
			RevID:        row.Value.Rev,
			Flags:        row.Value.Flags,
			TimeReceived: time.Now(),
		}
		// base.LogTo("Cache", "  Got view sequence #%d (%q / %q)", entry.Sequence, entry.DocID, entry.RevID)
		entries = append(entries, entry)
	}

	base.LogTo("Cache", "    Got %d rows from view for %q: #%d ... #%d",
		len(entries), channelName, entries[0].Sequence, entries[len(entries)-1].Sequence)
	if elapsed := time.Since(start); elapsed > 200*time.Millisecond {
		base.Log("changes_view: Query took %v to return %d rows, options = %#v",
			elapsed, len(entries), optMap)
	}
	changeCacheExpvars.Add("view_queries", 1)
	return entries, nil
}

func changesViewOptions(channelName string, endSeq uint64, options ChangesOptions) Body {
	endKey := []interface{}{channelName, endSeq}
	if endSeq == 0 {
		endKey[1] = map[string]interface{}{} // infinity
	}
	optMap := Body{
		"stale":    false,
		"startkey": []interface{}{channelName, options.Since.Seq + 1},
		"endkey":   endKey,
	}
	if options.Limit > 0 {
		optMap["limit"] = options.Limit
	}
	return optMap
}
<|endoftext|>


<commit_before>package logging

import (
	"bytes"
	"encoding/json"
	"strings"
	"testing"

	"github.com/getlantern/go-loggly"
	"github.com/getlantern/golog"
	"github.com/getlantern/testify/assert"
)

func TestLoggly(t *testing.T) {
	var buf bytes.Buffer
	var result map[string]interface{}
	loggly := loggly.New("token not required")
	loggly.Writer = &buf
	lw := logglyErrorWriter{client: loggly}
	golog.SetOutputs(lw, nil)
	log := golog.LoggerFor("test")

	log.Error("")
	if assert.NoError(t, json.Unmarshal(buf.Bytes(), &result), "Unmarshal error") {
		assert.Equal(t, "test", result["locationInfo"])
		assert.Equal(t, "", result["message"], "empty message should be logged as is")
	}

	buf.Reset()
	log.Error("short message")
	if assert.NoError(t, json.Unmarshal(buf.Bytes(), &result), "Unmarshal error") {
		assert.Equal(t, "test", result["locationInfo"])
		assert.Equal(t, "short message", result["message"], "short message should be logged as is")
	}

	buf.Reset()
	log.Error("message with: reason")
	if assert.NoError(t, json.Unmarshal(buf.Bytes(), &result), "Unmarshal error") {
		assert.Equal(t, "test", result["locationInfo"])
		assert.Equal(t, "message with: reason", result["message"], "message should be last 2 chunks")
	}

	buf.Reset()
	log.Error("deep reason: message with: reason")
	if assert.NoError(t, json.Unmarshal(buf.Bytes(), &result), "Unmarshal error") {
		assert.Equal(t, "test", result["locationInfo"])
		assert.Equal(t, "message with: reason", result["message"], "message should be last 2 chunks")
	}

	buf.Reset()
	log.Error("deep reason: an url https://a.com in message: reason")
	if assert.NoError(t, json.Unmarshal(buf.Bytes(), &result), "Unmarshal error") {
		assert.Equal(t, "an url https://a.com in message: reason", result["message"], "should not truncate url")
	}

	buf.Reset()
	log.Error("deep reason: an url 127.0.0.1:8787 in message: reason")
	if assert.NoError(t, json.Unmarshal(buf.Bytes(), &result), "Unmarshal error") {
		assert.Equal(t, "test", result["locationInfo"])
		assert.Equal(t, "an url 127.0.0.1:8787 in message: reason", result["message"], "should not truncate url")
	}

	buf.Reset()
	longMsg := "message with: really l" + strings.Repeat("o", 100) + "ng reason"
	log.Error(longMsg)
	if assert.NoError(t, json.Unmarshal(buf.Bytes(), &result), "Unmarshal error") {
		assert.Equal(t, "test", result["locationInfo"])
		assert.Equal(t, longMsg, result["message"], "should not truncate long messages as it's unlikely to happen")
	}
}
<commit_msg>fix logging test<commit_after>package logging

import (
	"bytes"
	"encoding/json"
	"regexp"
	"strings"
	"testing"

	"github.com/getlantern/go-loggly"
	"github.com/getlantern/golog"
	"github.com/getlantern/testify/assert"
)

func TestLoggly(t *testing.T) {
	var buf bytes.Buffer
	var result map[string]interface{}
	loggly := loggly.New("token not required")
	loggly.Writer = &buf
	lw := logglyErrorWriter{client: loggly}
	golog.SetOutputs(lw, nil)
	log := golog.LoggerFor("test")

	log.Error("")
	if assert.NoError(t, json.Unmarshal(buf.Bytes(), &result), "Unmarshal error") {
		assert.Equal(t, "test", result["locationInfo"])
		assert.Regexp(t, regexp.MustCompile("logging_test.go:([0-9]+)"), result["message"])
	}

	buf.Reset()
	log.Error("short message")
	if assert.NoError(t, json.Unmarshal(buf.Bytes(), &result), "Unmarshal error") {
		assert.Equal(t, "test", result["locationInfo"])
		assert.Regexp(t, regexp.MustCompile("logging_test.go:([0-9]+) short message"), result["message"])
	}

	buf.Reset()
	log.Error("message with: reason")
	if assert.NoError(t, json.Unmarshal(buf.Bytes(), &result), "Unmarshal error") {
		assert.Equal(t, "test", result["locationInfo"])
		assert.Regexp(t, "logging_test.go:([0-9]+) message with: reason", result["message"])
	}

	buf.Reset()
	log.Error("deep reason: message with: reason")
	if assert.NoError(t, json.Unmarshal(buf.Bytes(), &result), "Unmarshal error") {
		assert.Equal(t, "test", result["locationInfo"])
		assert.Equal(t, "message with: reason", result["message"], "message should be last 2 chunks")
	}

	buf.Reset()
	log.Error("deep reason: an url https://a.com in message: reason")
	if assert.NoError(t, json.Unmarshal(buf.Bytes(), &result), "Unmarshal error") {
		assert.Equal(t, "an url https://a.com in message: reason", result["message"], "should not truncate url")
	}

	buf.Reset()
	log.Error("deep reason: an url 127.0.0.1:8787 in message: reason")
	if assert.NoError(t, json.Unmarshal(buf.Bytes(), &result), "Unmarshal error") {
		assert.Equal(t, "test", result["locationInfo"])
		assert.Equal(t, "an url 127.0.0.1:8787 in message: reason", result["message"], "should not truncate url")
	}

	buf.Reset()
	longMsg := "message with: really l" + strings.Repeat("o", 100) + "ng reason"
	log.Error(longMsg)
	if assert.NoError(t, json.Unmarshal(buf.Bytes(), &result), "Unmarshal error") {
		assert.Equal(t, "test", result["locationInfo"])
		assert.Regexp(t, regexp.MustCompile("logging_test.go:([0-9]+) "+longMsg), result["message"])
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 Vector Creations Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package config

import (
	"github.com/matrix-org/dendrite/mediaapi/types"
	"github.com/matrix-org/gomatrixserverlib"
)

// MediaAPI contains the config information necessary to spin up a mediaapi process.
type MediaAPI struct {
	// The name of the server. This is usually the domain name, e.g 'matrix.org', 'localhost'.
	ServerName gomatrixserverlib.ServerName `yaml:"server_name"`
	// The absolute base path to where media files will be stored.
	AbsBasePath types.Path `yaml:"abs_base_path"`
	// The maximum file size in bytes that is allowed to be stored on this server.
	// Note that remote files larger than this can still be proxied to a client, they will just not be cached.
	// Note: if MaxFileSizeBytes is set to 0, the size is unlimited.
	MaxFileSizeBytes types.FileSizeBytes `yaml:"max_file_size_bytes"`
	// The postgres connection config for connecting to the database e.g a postgres:// URI
	DataSource string `yaml:"database"`
}
<commit_msg>mediaapi/config: Remove obsolete proxying comment and add default comment<commit_after>// Copyright 2017 Vector Creations Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package config

import (
	"github.com/matrix-org/dendrite/mediaapi/types"
	"github.com/matrix-org/gomatrixserverlib"
)

// MediaAPI contains the config information necessary to spin up a mediaapi process.
type MediaAPI struct {
	// The name of the server. This is usually the domain name, e.g 'matrix.org', 'localhost'.
	ServerName gomatrixserverlib.ServerName `yaml:"server_name"`
	// The absolute base path to where media files will be stored.
	AbsBasePath types.Path `yaml:"abs_base_path"`
	// The maximum file size in bytes that is allowed to be stored on this server.
	// Note: if MaxFileSizeBytes is set to 0, the size is unlimited.
	// Note: if max_file_size_bytes is not set, it will default to 10485760 (10MB)
	MaxFileSizeBytes types.FileSizeBytes `yaml:"max_file_size_bytes"`
	// The postgres connection config for connecting to the database e.g a postgres:// URI
	DataSource string `yaml:"database"`
}
<|endoftext|>


<commit_before>/**
 * Provides a redis storage adapter for storing and retrieving Watches.
 */

package msWatchStorage

import (
	// Utilities
	"encoding/json"
	"fmt"
	"strconv"

	// Redis.
	"github.com/mediocregopher/radix.v2/redis"

	// Internal dependencies.
	common "github.com/krystalcode/go-mantis-shrimp/watches/common"
	wrapper "github.com/krystalcode/go-mantis-shrimp/watches/wrapper"
)

/**
 * Redis storage provider.
 */

// Redis implements the Storage interface, allowing to use Redis as a Storage
// engine.
type Redis struct {
	dsn    string
	client *redis.Client
}

// Get implements Storage.Get(). It retrieves from Storage and returns the Watch
// for the given ID.
func (storage Redis) Get(_id int) common.Watch {
	if storage.client == nil {
		panic("The Redis client has not been initialized yet.")
	}

	key := redisKey(_id)

	r := storage.client.Cmd("GET", key)
	if r.Err != nil {
		panic(r.Err)
	}

	jsonWatch, err := r.Bytes()
	// If an error happens here, it should be because there is no value for this
	// key. It could be the case that the data is corrupted or the wrong data is
	// stored, we should see how to handle this later.
	// @I Handle edge cases when deserializing json in Redis
	if err != nil {
		return nil
	}

	// Create and initialize a Watch object based on the given JSON object.
	watch, err := wrapper.Create(jsonWatch)
	if err != nil {
		panic(err)
	}

	return watch
}

// Set implements Storage.Set(). It stores the given Watch object to the Redis
// Storage.
func (storage Redis) Set(watch common.Watch) int {
	// @I Consider using hashmaps instead of json values
	// @I Investigate risk of a Watch overriding another due to race conditions when
	//    creating them

	if storage.client == nil {
		panic("The Redis client has not been initialized yet.")
	}

	// We'll be storing a WatchWrapper which contains the Watch type as well.
	wrapper, err := wrapper.Wrapper(watch)
	if err != nil {
		panic(err)
	}
	jsonWatch, err := json.Marshal(wrapper)
	if err != nil {
		panic(err)
	}

	// Generate an ID, store the Watch, and update the Watches index set.
	_id := storage.generateID()
	key := redisKey(_id)
	err = storage.client.Cmd("SET", key, jsonWatch).Err
	if err != nil {
		panic(err)
	}
	err = storage.client.Cmd("ZADD", "watches", _id, key).Err
	if err != nil {
		panic(err)
	}

	return _id
}

// generateID generates an ID for a new Watch by incrementing the last known
// Watch ID.
func (storage Redis) generateID() int {
	// Get the last ID that exists on the Watches index set, so that we can generate
	// the next one.
	r, err := storage.client.Cmd("ZREVRANGE", "watches", 0, 0, "WITHSCORES").List()
	if err != nil {
		panic(err)
	}

	// If there are no watches yet, start with ID 1.
	if len(r) == 0 {
		return 1
	}

	_id, err := strconv.Atoi(r[1])
	if err != nil {
		panic(err)
	}

	return _id + 1
}

// NewRedisStorage implements the StorageFactory function type. It initiates a
// connection to the Redis database defined in the given configuration, and it
// returns the Storage engine object.
var NewRedisStorage = func(config map[string]string) (Storage, error) {
	dsn, ok := config["STORAGE_REDIS_DSN"]
	if !ok {
		err := fmt.Errorf(
			"the \"%s\" configuration option is required for the Redis storage",
			"STORAGE_REDIS_DSN",
		)
		return nil, err
	}

	client, err := redis.Dial("tcp", dsn)
	if err != nil {
		err := fmt.Errorf("failed to connect to Redis: %s", err.Error())
		return nil, err
	}

	storage := Redis{
		dsn:    dsn,
		client: client,
	}

	return storage, nil
}

/**
 * For internal use.
 */

// Generate a Redis key for the given Watch ID.
func redisKey(_id int) string {
	return "watch:" + strconv.Itoa(_id)
}
<commit_msg>ms_watches_storage Added Issue for improving error handling<commit_after>/**
 * Provides a redis storage adapter for storing and retrieving Watches.
 */

package msWatchStorage

import (
	// Utilities
	"encoding/json"
	"fmt"
	"strconv"

	// Redis.
	"github.com/mediocregopher/radix.v2/redis"

	// Internal dependencies.
	common "github.com/krystalcode/go-mantis-shrimp/watches/common"
	wrapper "github.com/krystalcode/go-mantis-shrimp/watches/wrapper"
)

/**
 * Redis storage provider.
 */

// Redis implements the Storage interface, allowing to use Redis as a Storage
// engine.
type Redis struct {
	dsn    string
	client *redis.Client
}

// Get implements Storage.Get(). It retrieves from Storage and returns the Watch
// for the given ID.
func (storage Redis) Get(_id int) common.Watch {
	// @I Delegate error handling to the caller in Storage API functions

	if storage.client == nil {
		panic("The Redis client has not been initialized yet.")
	}

	key := redisKey(_id)

	r := storage.client.Cmd("GET", key)
	if r.Err != nil {
		panic(r.Err)
	}

	jsonWatch, err := r.Bytes()
	// If an error happens here, it should be because there is no value for this
	// key. It could be the case that the data is corrupted or the wrong data is
	// stored, we should see how to handle this later.
	// @I Handle edge cases when deserializing json in Redis
	if err != nil {
		return nil
	}

	// Create and initialize a Watch object based on the given JSON object.
	watch, err := wrapper.Create(jsonWatch)
	if err != nil {
		panic(err)
	}

	return watch
}

// Set implements Storage.Set(). It stores the given Watch object to the Redis
// Storage.
func (storage Redis) Set(watch common.Watch) int {
	// @I Consider using hashmaps instead of json values
	// @I Investigate risk of a Watch overriding another due to race conditions when
	//    creating them

	if storage.client == nil {
		panic("The Redis client has not been initialized yet.")
	}

	// We'll be storing a WatchWrapper which contains the Watch type as well.
	wrapper, err := wrapper.Wrapper(watch)
	if err != nil {
		panic(err)
	}
	jsonWatch, err := json.Marshal(wrapper)
	if err != nil {
		panic(err)
	}

	// Generate an ID, store the Watch, and update the Watches index set.
	_id := storage.generateID()
	key := redisKey(_id)
	err = storage.client.Cmd("SET", key, jsonWatch).Err
	if err != nil {
		panic(err)
	}
	err = storage.client.Cmd("ZADD", "watches", _id, key).Err
	if err != nil {
		panic(err)
	}

	return _id
}

// generateID generates an ID for a new Watch by incrementing the last known
// Watch ID.
func (storage Redis) generateID() int {
	// Get the last ID that exists on the Watches index set, so that we can generate
	// the next one.
	r, err := storage.client.Cmd("ZREVRANGE", "watches", 0, 0, "WITHSCORES").List()
	if err != nil {
		panic(err)
	}

	// If there are no watches yet, start with ID 1.
	if len(r) == 0 {
		return 1
	}

	_id, err := strconv.Atoi(r[1])
	if err != nil {
		panic(err)
	}

	return _id + 1
}

// NewRedisStorage implements the StorageFactory function type. It initiates a
// connection to the Redis database defined in the given configuration, and it
// returns the Storage engine object.
var NewRedisStorage = func(config map[string]string) (Storage, error) {
	dsn, ok := config["STORAGE_REDIS_DSN"]
	if !ok {
		err := fmt.Errorf(
			"the \"%s\" configuration option is required for the Redis storage",
			"STORAGE_REDIS_DSN",
		)
		return nil, err
	}

	client, err := redis.Dial("tcp", dsn)
	if err != nil {
		err := fmt.Errorf("failed to connect to Redis: %s", err.Error())
		return nil, err
	}

	storage := Redis{
		dsn:    dsn,
		client: client,
	}

	return storage, nil
}

/**
 * For internal use.
 */

// Generate a Redis key for the given Watch ID.
func redisKey(_id int) string {
	return "watch:" + strconv.Itoa(_id)
}
<|endoftext|>


<commit_before>package dragonfly

import (
	//"fmt"
	"os"
)

type Job struct {
	Steps []Step
	Temp  *os.File
}

func (job *Job) Apply() (*os.File, error) {
	fileChan := make(chan *os.File)
	errChan := make(chan error)
	defer close(fileChan)
	defer close(errChan)

	var (
		temp *os.File
		err  error
	)

	go job.Steps[0].Fetch(fileChan, errChan)

	select {
	case err = <-errChan:
		return nil, err
	case temp = <-fileChan:
		//defer temp.Close()
		//defer os.Remove(temp.Name())
	}

	go job.Steps[1].Process(temp, fileChan, errChan)
	select {
	case err = <-errChan:
		return nil, err
	case temp = <-fileChan:
		//defer temp.Close()
		//defer os.Remove(temp.Name())
	}

	return temp, err
}
<commit_msg>buffered channels yield a happier family<commit_after>package dragonfly

import (
	//"fmt"
	"os"
)

type Job struct {
	Steps []Step
	Temp  *os.File
}

func (job *Job) Apply() (*os.File, error) {
	fileChan := make(chan *os.File, 1)
	errChan := make(chan error, 1)
	defer close(fileChan)
	defer close(errChan)

	var (
		temp *os.File
		err  error
	)

	go job.Steps[0].Fetch(fileChan, errChan)

	select {
	case err = <-errChan:
		return nil, err
	case temp = <-fileChan:
		//defer temp.Close()
		//defer os.Remove(temp.Name())
	}

	go job.Steps[1].Process(temp, fileChan, errChan)
	select {
	case err = <-errChan:
		return nil, err
	case temp = <-fileChan:
		//defer temp.Close()
		//defer os.Remove(temp.Name())
	}

	return temp, err
}
<|endoftext|>


<commit_before>package dragonfly

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
)

type Step interface {
	Apply(in chan *os.File) (out chan *os.File, errChan chan error)
	//Args    []string
	//Command string
}

type Job struct {
	Steps []Step
	Temp  *os.File
}

func (job *Job) Apply() (temp *os.File, err error) {
	blarrr := make(chan *os.File)
	close(blarrr)
	seed1, errChan := job.Steps[0].Apply(blarrr)
	seed2, errChan2 := job.Steps[1].Apply(seed1)
	//blarrr <- temp

	select {
	//case err = <-errChan:

	case err = <-errChan:
		fmt.Println("errr")
		fmt.Println(err)

		if err == nil {
			temp = <-seed2
		}

	case err = <-errChan2:
		fmt.Println("errr2")
		fmt.Println(err)
		if err == nil {
			temp = <-seed2
		}
	case temp = <-seed2:
		fmt.Println("seed")
		fmt.Println("tmp")

	}

	return

}

type stepApplication func(temp *os.File) (*os.File, error)

func applyStepPipeline(in chan *os.File, step stepApplication) (out chan *os.File, errChan chan error) {
	out = make(chan *os.File)
	errChan = make(chan error)

	go func() {
		prev := <-in
		defer close(out)
		defer close(errChan)

		content, err := step(prev)
		if err != nil {
			errChan <- err
			return
		}

		out <- content
	}()

	return out, errChan
}

type FetchFileStep struct {
	Args    []string
	Command string
}

type ResizeStep struct {
	Args    []string
	Command string
}

func (step ResizeStep) Apply(in chan *os.File) (out chan *os.File, errChan chan error) {
	return applyStepPipeline(in, func(temp *os.File) (newTemp *os.File, err error) {
		format := step.Args[1]
		return step.resize(temp, format)
	})
}

func (step ResizeStep) resize(image *os.File, format string) (*os.File, error) {
	binary, err := exec.LookPath("convert")
	if err != nil {
		return nil, err
	}

	tempPrefix := "godragonfly" + format
	resized, err := ioutil.TempFile(os.TempDir(), tempPrefix)
	if err != nil {
		return nil, err
	}

	if image == nil {
		return nil, err
	}

	args := []string{
		image.Name(),
		"-resize", format,
		resized.Name(),
	}

	cmd := exec.Command(binary, args...)
	cmd.Run()

	return resized, err
}

func (step FetchFileStep) Apply(in chan *os.File) (out chan *os.File, errChan chan error) {
	return applyStepPipeline(in, func(_ *os.File) (temp *os.File, err error) {
		filename := step.Args[0]
		return fechFile(filename)
		//return nil, errors.New("please don't stop the music")
	})
}

func fechFile(filename string) (*os.File, error) {
	content, err := ioutil.ReadFile(filename)

	if err != nil {
		return nil, err
	}

	temp, err := ioutil.TempFile(os.TempDir(), "godragonfly")

	if err != nil {
		return nil, err
	}

	_, err = temp.Write(content)
	if err != nil {
		return nil, err
	}

	return temp, err
}
<commit_msg>clear up variable names<commit_after>package dragonfly

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
)

type Step interface {
	Apply(in chan *os.File) (out chan *os.File, errChan chan error)
	//Args    []string
	//Command string
}

type Job struct {
	Steps []Step
	Temp  *os.File
}

func (job *Job) Apply() (temp *os.File, err error) {

	head := make(chan *os.File)
	tail, errChan := job.Steps[0].Apply(head)
	tail, errChan2 := job.Steps[1].Apply(tail)

	close(head)

	select {

	case err = <-errChan:
		fmt.Println("errr")
		fmt.Println(err)

		if err == nil {
			temp = <-tail
		}

	case err = <-errChan2:
		fmt.Println("errr2")
		fmt.Println(err)
		if err == nil {
			temp = <-tail
		}
	case temp = <-tail:
		fmt.Println("seed")
		fmt.Println("tmp")

	}

	return

}

type stepApplication func(temp *os.File) (*os.File, error)

func applyStepPipeline(in chan *os.File, step stepApplication) (out chan *os.File, errChan chan error) {
	out = make(chan *os.File)
	errChan = make(chan error)

	go func() {
		prev := <-in
		defer close(out)
		defer close(errChan)

		content, err := step(prev)
		if err != nil {
			errChan <- err
			return
		}

		out <- content
	}()

	return out, errChan
}

type FetchFileStep struct {
	Args    []string
	Command string
}

type ResizeStep struct {
	Args    []string
	Command string
}

func (step ResizeStep) Apply(in chan *os.File) (out chan *os.File, errChan chan error) {
	return applyStepPipeline(in, func(temp *os.File) (newTemp *os.File, err error) {
		format := step.Args[1]
		return step.resize(temp, format)
	})
}

func (step ResizeStep) resize(image *os.File, format string) (*os.File, error) {
	binary, err := exec.LookPath("convert")
	if err != nil {
		return nil, err
	}

	tempPrefix := "godragonfly" + format
	resized, err := ioutil.TempFile(os.TempDir(), tempPrefix)
	if err != nil {
		return nil, err
	}

	if image == nil {
		return nil, err
	}

	args := []string{
		image.Name(),
		"-resize", format,
		resized.Name(),
	}

	cmd := exec.Command(binary, args...)
	cmd.Run()

	return resized, err
}

func (step FetchFileStep) Apply(in chan *os.File) (out chan *os.File, errChan chan error) {
	return applyStepPipeline(in, func(_ *os.File) (temp *os.File, err error) {
		filename := step.Args[0]
		return fechFile(filename)
		//return nil, errors.New("please don't stop the music")
	})
}

func fechFile(filename string) (*os.File, error) {
	content, err := ioutil.ReadFile(filename)

	if err != nil {
		return nil, err
	}

	temp, err := ioutil.TempFile(os.TempDir(), "godragonfly")

	if err != nil {
		return nil, err
	}

	_, err = temp.Write(content)
	if err != nil {
		return nil, err
	}

	return temp, err
}
<|endoftext|>