<commit_before>package crawler

import (
	"encoding/json"
	"fmt"
	"github.com/dokterbob/ipfs-search/indexer"
	"github.com/dokterbob/ipfs-search/queue"
	"gopkg.in/ipfs/go-ipfs-api.v1"
	"log"
	"net"
	"net/http"
	"net/url"
	// "path"
	"strings"
	"syscall"
	"time"
)

const (
	// Reconnect time in seconds
	RECONNECT_WAIT = 2
	TIKA_TIMEOUT   = 120
)

type CrawlerArgs struct {
	Hash       string
	Name       string
	Size       uint64
	ParentHash string
	ParentName string // This is legacy, should be removed
}

type Crawler struct {
	sh *shell.Shell
	id *indexer.Indexer
	fq *queue.TaskQueue
	hq *queue.TaskQueue
}

func NewCrawler(sh *shell.Shell, id *indexer.Indexer, fq *queue.TaskQueue, hq *queue.TaskQueue) *Crawler {
	return &Crawler{
		sh: sh,
		id: id,
		fq: fq,
		hq: hq,
	}
}

func hashUrl(hash string) string {
	return fmt.Sprintf("/ipfs/%s", hash)
}

// Update references with name, parent_hash and parent_name. Returns true when updated
func update_references(references []indexer.Reference, name string, parent_hash string) ([]indexer.Reference, bool) {
	if parent_hash == "" {
		// No parent hash, don't bother adding reference
		return references, false
	}

	for i := range references {
		if references[i].ParentHash == parent_hash {
			log.Printf("Reference '%s' for %s exists, not updating", name, parent_hash)
			return references, false
		}
	}

	log.Printf("Adding reference '%s' for %s", name, parent_hash)

	references = append(references, indexer.Reference{
		Name:       name,
		ParentHash: parent_hash,
	})

	return references, true
}

// Handle IPFS errors graceously, returns try again bool and original error
func (c Crawler) handleError(err error, hash string) (bool, error) {
	if _, ok := err.(*shell.Error); ok && strings.Contains(err.Error(), "proto") {
		// We're not recovering from protocol errors, so panic

		// Attempt to index panic to prevent re-indexing
		metadata := map[string]interface{}{
			"error": err.Error(),
		}

		c.id.IndexItem("invalid", hash, metadata)

		panic(err)
	}

	if uerr, ok := err.(*url.Error); ok {
		// URL errors

		log.Printf("URL error %v", uerr)

		if uerr.Timeout() {
			// Fail on timeouts
			return false, err
		}

		if uerr.Temporary() {
			// Retry on other temp errors
			return true, nil
		}

		// Somehow, the errors below are not temp errors !?
		switch t := uerr.Err.(type) {
		case *net.OpError:
			if t.Op == "dial" {
				log.Printf("Unknown host %v", t)
				return true, nil

			} else if t.Op == "read" {
				log.Printf("Connection refused %v", t)
				return true, nil
			}

		case syscall.Errno:
			if t == syscall.ECONNREFUSED {
				log.Printf("Connection refused %v", t)
				return true, nil
			}
		}
	}

	return false, err
}

func (c Crawler) index_references(hash string, name string, parent_hash string) ([]indexer.Reference, bool, error) {
	var references []indexer.Reference

	references, item_type, err := c.id.GetReferences(hash)
	if err != nil {
		return nil, false, err
	}

	if references != nil {
		log.Printf("Already indexed '%s'.", hash)

		references, references_updated := update_references(references, name, parent_hash)

		if references_updated {
			log.Printf("Updating references for '%s'.", hash)

			properties := map[string]interface{}{
				"references": references,
			}

			err := c.id.IndexItem(item_type, hash, properties)
			if err != nil {
				return nil, false, err
			}
		} else {
			log.Printf("Not updating references for '%s'", hash)
		}

		return references, true, nil
	} else {
		// Initialize references
		references = []indexer.Reference{
			{
				Name:       name,
				ParentHash: parent_hash,
			}}
	}

	return references, false, nil
}

// Given a particular hash (file or directory), start crawling
func (c Crawler) CrawlHash(hash string, name string, parent_hash string, parent_name string) error {
	references, already_indexed, err := c.index_references(hash, name, parent_hash)

	if err != nil {
		return err
	}

	if already_indexed {
		return nil
	}

	log.Printf("Crawling hash '%s' (%s)", hash, name)

	url := hashUrl(hash)

	var list *shell.UnixLsObject

	try_again := true
	for try_again {
		list, err = c.sh.FileList(url)

		try_again, err = c.handleError(err, hash)

		if try_again {
			log.Printf("Retrying in %d seconds", RECONNECT_WAIT)
			time.Sleep(RECONNECT_WAIT * time.Duration(time.Second))
		}
	}

	if err != nil {
		return err
	}

	switch list.Type {
	case "File":
		// Add to file crawl queue
		// Note: we're expecting no references here, see comment below
		args := CrawlerArgs{
			Hash: hash,
			Name: name,
			Size: list.Size,
		}

		err = c.fq.AddTask(args)
		if err != nil {
			// failed to send the task
			return err
		}
	case "Directory":
		// Index name and size for directory and directory items
		properties := map[string]interface{}{
			"links":      list.Links,
			"size":       list.Size,
			"references": references,
		}

		err := c.id.IndexItem("directory", hash, properties)
		if err != nil {
			return err
		}

		for _, link := range list.Links {
			args := CrawlerArgs{
				Hash:       link.Hash,
				Name:       link.Name,
				Size:       link.Size,
				ParentHash: hash,
			}

			switch link.Type {
			case "File":
				// Add file to crawl queue
				err = c.fq.AddTask(args)
				if err != nil {
					// failed to send the task
					return err
				}

			case "Directory":
				// Add directory to crawl queue
				c.hq.AddTask(args)
				if err != nil {
					// failed to send the task
					return err
				}
			default:
				log.Printf("Type '%s' skipped for '%s'", list.Type, hash)
			}
		}
	default:
		log.Printf("Type '%s' skipped for '%s'", list.Type, hash)
	}

	log.Printf("Finished hash %s", hash)

	return nil
}

func getMetadata(path string, metadata *map[string]interface{}) error {
	const ipfs_tika_url = "http://localhost:8081"

	client := http.Client{
		Timeout: TIKA_TIMEOUT * time.Duration(time.Second),
	}

	resp, err := client.Get(ipfs_tika_url + path)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		return fmt.Errorf("Undesired status '%s' from ipfs-tika.", resp.Status)
	}

	// Parse resulting JSON
	if err := json.NewDecoder(resp.Body).Decode(&metadata); err != nil {
		return err
	}

	return err
}

// Crawl a single object, known to be a file
func (c Crawler) CrawlFile(hash string, name string, parent_hash string, parent_name string, size uint64) error {
	references, already_indexed, err := c.index_references(hash, name, parent_hash)

	if err != nil {
		return err
	}

	if already_indexed {
		return nil
	}

	log.Printf("Crawling file %s (%s)\n", hash, name)

	metadata := make(map[string]interface{})

	if size > 0 {
		if size > 10*1024*1024 {
			// Fail hard for really large files, for now
			return fmt.Errorf("%s (%s) too large, not indexing (for now).", hash, name)
		}

		var path string
		if name != "" && parent_hash != "" {
			path = fmt.Sprintf("/ipfs/%s/%s", parent_hash, name)
		} else {
			path = fmt.Sprintf("/ipfs/%s", hash)
		}

		try_again := true
		for try_again {
			err = getMetadata(path, &metadata)

			try_again, err = c.handleError(err, hash)

			if try_again {
				log.Printf("Retrying in %d seconds", RECONNECT_WAIT)
				time.Sleep(RECONNECT_WAIT * time.Duration(time.Second))
			}
		}

		if err != nil {
			return err
		}

		// Check for IPFS links in content
		/*
			for raw_url := range metadata.urls {
				url, err := URL.Parse(raw_url)

				if err != nil {
					return err
				}

				if strings.HasPrefix(url.Path, "/ipfs/") {
					// Found IPFS link!
					args := CrawlerArgs{
						Hash:       link.Hash,
						Name:       link.Name,
						Size:       link.Size,
						ParentHash: hash,
					}

				}
			}
		*/
	}

	metadata["size"] = size
	metadata["references"] = references

	err = c.id.IndexItem("file", hash, metadata)
	if err != nil {
		return err
	}

	log.Printf("Finished file %s", hash)

	return nil
}
<commit_msg>50 MB maximum size for metadata indexing.<commit_after>package crawler

import (
	"encoding/json"
	"fmt"
	"github.com/dokterbob/ipfs-search/indexer"
	"github.com/dokterbob/ipfs-search/queue"
	"gopkg.in/ipfs/go-ipfs-api.v1"
	"log"
	"net"
	"net/http"
	"net/url"
	// "path"
	"strings"
	"syscall"
	"time"
)

const (
	// Reconnect time in seconds
	RECONNECT_WAIT    = 2
	TIKA_TIMEOUT      = 120
	METADATA_MAX_SIZE = 50 * 1024 * 1024
)

type CrawlerArgs struct {
	Hash       string
	Name       string
	Size       uint64
	ParentHash string
	ParentName string // This is legacy, should be removed
}

type Crawler struct {
	sh *shell.Shell
	id *indexer.Indexer
	fq *queue.TaskQueue
	hq *queue.TaskQueue
}

func NewCrawler(sh *shell.Shell, id *indexer.Indexer, fq *queue.TaskQueue, hq *queue.TaskQueue) *Crawler {
	return &Crawler{
		sh: sh,
		id: id,
		fq: fq,
		hq: hq,
	}
}

func hashUrl(hash string) string {
	return fmt.Sprintf("/ipfs/%s", hash)
}

// Update references with name, parent_hash and parent_name. Returns true when updated
func update_references(references []indexer.Reference, name string, parent_hash string) ([]indexer.Reference, bool) {
	if parent_hash == "" {
		// No parent hash, don't bother adding reference
		return references, false
	}

	for i := range references {
		if references[i].ParentHash == parent_hash {
			log.Printf("Reference '%s' for %s exists, not updating", name, parent_hash)
			return references, false
		}
	}

	log.Printf("Adding reference '%s' for %s", name, parent_hash)

	references = append(references, indexer.Reference{
		Name:       name,
		ParentHash: parent_hash,
	})

	return references, true
}

// Handle IPFS errors graceously, returns try again bool and original error
func (c Crawler) handleError(err error, hash string) (bool, error) {
	if _, ok := err.(*shell.Error); ok && strings.Contains(err.Error(), "proto") {
		// We're not recovering from protocol errors, so panic

		// Attempt to index panic to prevent re-indexing
		metadata := map[string]interface{}{
			"error": err.Error(),
		}

		c.id.IndexItem("invalid", hash, metadata)

		panic(err)
	}

	if uerr, ok := err.(*url.Error); ok {
		// URL errors

		log.Printf("URL error %v", uerr)

		if uerr.Timeout() {
			// Fail on timeouts
			return false, err
		}

		if uerr.Temporary() {
			// Retry on other temp errors
			return true, nil
		}

		// Somehow, the errors below are not temp errors !?
		switch t := uerr.Err.(type) {
		case *net.OpError:
			if t.Op == "dial" {
				log.Printf("Unknown host %v", t)
				return true, nil

			} else if t.Op == "read" {
				log.Printf("Connection refused %v", t)
				return true, nil
			}

		case syscall.Errno:
			if t == syscall.ECONNREFUSED {
				log.Printf("Connection refused %v", t)
				return true, nil
			}
		}
	}

	return false, err
}

func (c Crawler) index_references(hash string, name string, parent_hash string) ([]indexer.Reference, bool, error) {
	var references []indexer.Reference

	references, item_type, err := c.id.GetReferences(hash)
	if err != nil {
		return nil, false, err
	}

	if references != nil {
		log.Printf("Already indexed '%s'.", hash)

		references, references_updated := update_references(references, name, parent_hash)

		if references_updated {
			log.Printf("Updating references for '%s'.", hash)

			properties := map[string]interface{}{
				"references": references,
			}

			err := c.id.IndexItem(item_type, hash, properties)
			if err != nil {
				return nil, false, err
			}
		} else {
			log.Printf("Not updating references for '%s'", hash)
		}

		return references, true, nil
	} else {
		// Initialize references
		references = []indexer.Reference{
			{
				Name:       name,
				ParentHash: parent_hash,
			}}
	}

	return references, false, nil
}

// Given a particular hash (file or directory), start crawling
func (c Crawler) CrawlHash(hash string, name string, parent_hash string, parent_name string) error {
	references, already_indexed, err := c.index_references(hash, name, parent_hash)

	if err != nil {
		return err
	}

	if already_indexed {
		return nil
	}

	log.Printf("Crawling hash '%s' (%s)", hash, name)

	url := hashUrl(hash)

	var list *shell.UnixLsObject

	try_again := true
	for try_again {
		list, err = c.sh.FileList(url)

		try_again, err = c.handleError(err, hash)

		if try_again {
			log.Printf("Retrying in %d seconds", RECONNECT_WAIT)
			time.Sleep(RECONNECT_WAIT * time.Duration(time.Second))
		}
	}

	if err != nil {
		return err
	}

	switch list.Type {
	case "File":
		// Add to file crawl queue
		// Note: we're expecting no references here, see comment below
		args := CrawlerArgs{
			Hash: hash,
			Name: name,
			Size: list.Size,
		}

		err = c.fq.AddTask(args)
		if err != nil {
			// failed to send the task
			return err
		}
	case "Directory":
		// Index name and size for directory and directory items
		properties := map[string]interface{}{
			"links":      list.Links,
			"size":       list.Size,
			"references": references,
		}

		err := c.id.IndexItem("directory", hash, properties)
		if err != nil {
			return err
		}

		for _, link := range list.Links {
			args := CrawlerArgs{
				Hash:       link.Hash,
				Name:       link.Name,
				Size:       link.Size,
				ParentHash: hash,
			}

			switch link.Type {
			case "File":
				// Add file to crawl queue
				err = c.fq.AddTask(args)
				if err != nil {
					// failed to send the task
					return err
				}

			case "Directory":
				// Add directory to crawl queue
				c.hq.AddTask(args)
				if err != nil {
					// failed to send the task
					return err
				}
			default:
				log.Printf("Type '%s' skipped for '%s'", list.Type, hash)
			}
		}
	default:
		log.Printf("Type '%s' skipped for '%s'", list.Type, hash)
	}

	log.Printf("Finished hash %s", hash)

	return nil
}

func getMetadata(path string, metadata *map[string]interface{}) error {
	const ipfs_tika_url = "http://localhost:8081"

	client := http.Client{
		Timeout: TIKA_TIMEOUT * time.Duration(time.Second),
	}

	resp, err := client.Get(ipfs_tika_url + path)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		return fmt.Errorf("Undesired status '%s' from ipfs-tika.", resp.Status)
	}

	// Parse resulting JSON
	if err := json.NewDecoder(resp.Body).Decode(&metadata); err != nil {
		return err
	}

	return err
}

// Crawl a single object, known to be a file
func (c Crawler) CrawlFile(hash string, name string, parent_hash string, parent_name string, size uint64) error {
	references, already_indexed, err := c.index_references(hash, name, parent_hash)

	if err != nil {
		return err
	}

	if already_indexed {
		return nil
	}

	log.Printf("Crawling file %s (%s)\n", hash, name)

	metadata := make(map[string]interface{})

	if size > 0 {
		if size > METADATA_MAX_SIZE {
			// Fail hard for really large files, for now
			return fmt.Errorf("%s (%s) too large, not indexing (for now).", hash, name)
		}

		var path string
		if name != "" && parent_hash != "" {
			path = fmt.Sprintf("/ipfs/%s/%s", parent_hash, name)
		} else {
			path = fmt.Sprintf("/ipfs/%s", hash)
		}

		try_again := true
		for try_again {
			err = getMetadata(path, &metadata)

			try_again, err = c.handleError(err, hash)

			if try_again {
				log.Printf("Retrying in %d seconds", RECONNECT_WAIT)
				time.Sleep(RECONNECT_WAIT * time.Duration(time.Second))
			}
		}

		if err != nil {
			return err
		}

		// Check for IPFS links in content
		/*
			for raw_url := range metadata.urls {
				url, err := URL.Parse(raw_url)

				if err != nil {
					return err
				}

				if strings.HasPrefix(url.Path, "/ipfs/") {
					// Found IPFS link!
					args := CrawlerArgs{
						Hash:       link.Hash,
						Name:       link.Name,
						Size:       link.Size,
						ParentHash: hash,
					}

				}
			}
		*/
	}

	metadata["size"] = size
	metadata["references"] = references

	err = c.id.IndexItem("file", hash, metadata)
	if err != nil {
		return err
	}

	log.Printf("Finished file %s", hash)

	return nil
}
<|endoftext|>


<commit_before>// +build windows

package box

import (
	"syscall"
	"unsafe"
)

var kernel32 = syscall.NewLazyDLL("kernel32")

var hConout, _ = syscall.GetStdHandle(syscall.STD_OUTPUT_HANDLE)

type coord_t struct {
	X int16
	Y int16
}

type small_rect_t struct {
	Left   int16
	Top    int16
	Right  int16
	Bottom int16
}

type console_screen_buffer_info_t struct {
	Size              coord_t
	CursorPosition    coord_t
	Attributes        uint16
	Window            small_rect_t
	MaximumWindowSize coord_t
}

var getConsoleScreenBufferInfo = kernel32.NewProc("GetConsoleScreenBufferInfo")

func GetScreenBufferInfo() *console_screen_buffer_info_t {
	var csbi console_screen_buffer_info_t
	getConsoleScreenBufferInfo.Call(
		uintptr(hConout),
		uintptr(unsafe.Pointer(&csbi)))
	return &csbi
}

func (this *console_screen_buffer_info_t) ViewSize() (int, int) {
	return int(this.Window.Right-this.Window.Left) + 1,
		int(this.Window.Bottom-this.Window.Top) + 1
}

func (this *console_screen_buffer_info_t) CursorPos() (int, int) {
	return int(this.CursorPosition.X), int(this.CursorPosition.Y)
}

func GetLocate() (int, int) {
	return GetScreenBufferInfo().CursorPos()
}
<commit_msg>Fix #2 : When stdout is redirected, no output is on screen.<commit_after>// +build windows

package box

import (
	"syscall"
	"unsafe"
)

var kernel32 = syscall.NewLazyDLL("kernel32")

var hConout, _ = syscall.GetStdHandle(syscall.STD_ERROR_HANDLE)

type coord_t struct {
	X int16
	Y int16
}

type small_rect_t struct {
	Left   int16
	Top    int16
	Right  int16
	Bottom int16
}

type console_screen_buffer_info_t struct {
	Size              coord_t
	CursorPosition    coord_t
	Attributes        uint16
	Window            small_rect_t
	MaximumWindowSize coord_t
}

var getConsoleScreenBufferInfo = kernel32.NewProc("GetConsoleScreenBufferInfo")

func GetScreenBufferInfo() *console_screen_buffer_info_t {
	var csbi console_screen_buffer_info_t
	getConsoleScreenBufferInfo.Call(
		uintptr(hConout),
		uintptr(unsafe.Pointer(&csbi)))
	return &csbi
}

func (this *console_screen_buffer_info_t) ViewSize() (int, int) {
	return int(this.Window.Right-this.Window.Left) + 1,
		int(this.Window.Bottom-this.Window.Top) + 1
}

func (this *console_screen_buffer_info_t) CursorPos() (int, int) {
	return int(this.CursorPosition.X), int(this.CursorPosition.Y)
}

func GetLocate() (int, int) {
	return GetScreenBufferInfo().CursorPos()
}
<|endoftext|>


<commit_before>package git

/*
#include <git2.h>

extern void _go_git_populate_checkout_cb(git_checkout_options *opts);
*/
import "C"
import (
	"os"
	"runtime"
	"unsafe"
)

type CheckoutNotifyType uint
type CheckoutStrategy uint

const (
	CheckoutNotifyNone      CheckoutNotifyType = C.GIT_CHECKOUT_NOTIFY_NONE
	CheckoutNotifyConflict  CheckoutNotifyType = C.GIT_CHECKOUT_NOTIFY_CONFLICT
	CheckoutNotifyDirty     CheckoutNotifyType = C.GIT_CHECKOUT_NOTIFY_DIRTY
	CheckoutNotifyUpdated   CheckoutNotifyType = C.GIT_CHECKOUT_NOTIFY_UPDATED
	CheckoutNotifyUntracked CheckoutNotifyType = C.GIT_CHECKOUT_NOTIFY_UNTRACKED
	CheckoutNotifyIgnored   CheckoutNotifyType = C.GIT_CHECKOUT_NOTIFY_IGNORED
	CheckoutNotifyAll       CheckoutNotifyType = C.GIT_CHECKOUT_NOTIFY_ALL

	CheckoutNone                      CheckoutStrategy = C.GIT_CHECKOUT_NONE                         // Dry run, no actual updates
	CheckoutSafe                      CheckoutStrategy = C.GIT_CHECKOUT_SAFE                         // Allow safe updates that cannot overwrite uncommitted data
	CheckoutForce                     CheckoutStrategy = C.GIT_CHECKOUT_FORCE                        // Allow all updates to force working directory to look like index
	CheckoutRecreateMissing           CheckoutStrategy = C.GIT_CHECKOUT_RECREATE_MISSING             // Allow checkout to recreate missing files
	CheckoutAllowConflicts            CheckoutStrategy = C.GIT_CHECKOUT_ALLOW_CONFLICTS              // Allow checkout to make safe updates even if conflicts are found
	CheckoutRemoveUntracked           CheckoutStrategy = C.GIT_CHECKOUT_REMOVE_UNTRACKED             // Remove untracked files not in index (that are not ignored)
	CheckoutRemoveIgnored             CheckoutStrategy = C.GIT_CHECKOUT_REMOVE_IGNORED               // Remove ignored files not in index
	CheckoutUpdateOnly                CheckoutStrategy = C.GIT_CHECKOUT_UPDATE_ONLY                  // Only update existing files, don't create new ones
	CheckoutDontUpdateIndex           CheckoutStrategy = C.GIT_CHECKOUT_DONT_UPDATE_INDEX            // Normally checkout updates index entries as it goes; this stops that
	CheckoutNoRefresh                 CheckoutStrategy = C.GIT_CHECKOUT_NO_REFRESH                   // Don't refresh index/config/etc before doing checkout
	CheckoutSkipUnmerged              CheckoutStrategy = C.GIT_CHECKOUT_SKIP_UNMERGED                // Allow checkout to skip unmerged files
	CheckoutUserOurs                  CheckoutStrategy = C.GIT_CHECKOUT_USE_OURS                     // For unmerged files, checkout stage 2 from index
	CheckoutUseTheirs                 CheckoutStrategy = C.GIT_CHECKOUT_USE_THEIRS                   // For unmerged files, checkout stage 3 from index
	CheckoutDisablePathspecMatch      CheckoutStrategy = C.GIT_CHECKOUT_DISABLE_PATHSPEC_MATCH       // Treat pathspec as simple list of exact match file paths
	CheckoutSkipLockedDirectories     CheckoutStrategy = C.GIT_CHECKOUT_SKIP_LOCKED_DIRECTORIES      // Ignore directories in use, they will be left empty
	CheckoutDontOverwriteIgnored      CheckoutStrategy = C.GIT_CHECKOUT_DONT_OVERWRITE_IGNORED       // Don't overwrite ignored files that exist in the checkout target
	CheckoutConflictStyleMerge        CheckoutStrategy = C.GIT_CHECKOUT_CONFLICT_STYLE_MERGE         // Write normal merge files for conflicts
	CheckoutConflictStyleDiff3        CheckoutStrategy = C.GIT_CHECKOUT_CONFLICT_STYLE_DIFF3         // Include common ancestor data in diff3 format files for conflicts
	CheckoutDontRemoveExisting        CheckoutStrategy = C.GIT_CHECKOUT_DONT_REMOVE_EXISTING         // Don't overwrite existing files or folders
	CheckoutDontWriteIndex            CheckoutStrategy = C.GIT_CHECKOUT_DONT_WRITE_INDEX             // Normally checkout writes the index upon completion; this prevents that
	CheckoutUpdateSubmodules          CheckoutStrategy = C.GIT_CHECKOUT_UPDATE_SUBMODULES            // Recursively checkout submodules with same options (NOT IMPLEMENTED)
	CheckoutUpdateSubmodulesIfChanged CheckoutStrategy = C.GIT_CHECKOUT_UPDATE_SUBMODULES_IF_CHANGED // Recursively checkout submodules if HEAD moved in super repo (NOT IMPLEMENTED)
)

type CheckoutNotifyCallback func(why CheckoutNotifyType, path string, baseline, target, workdir DiffFile) ErrorCode
type CheckoutProgressCallback func(path string, completed, total uint) ErrorCode

type CheckoutOpts struct {
	Strategy         CheckoutStrategy   // Default will be a dry run
	DisableFilters   bool               // Don't apply filters like CRLF conversion
	DirMode          os.FileMode        // Default is 0755
	FileMode         os.FileMode        // Default is 0644 or 0755 as dictated by blob
	FileOpenFlags    int                // Default is O_CREAT | O_TRUNC | O_WRONLY
	NotifyFlags      CheckoutNotifyType // Default will be none
	NotifyCallback   CheckoutNotifyCallback
	ProgressCallback CheckoutProgressCallback
	TargetDirectory  string // Alternative checkout path to workdir
	Paths            []string
	Baseline         *Tree
}

func checkoutOptionsFromC(c *C.git_checkout_options) CheckoutOpts {
	opts := CheckoutOpts{}
	opts.Strategy = CheckoutStrategy(c.checkout_strategy)
	opts.DisableFilters = c.disable_filters != 0
	opts.DirMode = os.FileMode(c.dir_mode)
	opts.FileMode = os.FileMode(c.file_mode)
	opts.FileOpenFlags = int(c.file_open_flags)
	opts.NotifyFlags = CheckoutNotifyType(c.notify_flags)
	if c.notify_payload != nil {
		opts.NotifyCallback = pointerHandles.Get(c.notify_payload).(CheckoutOpts).NotifyCallback
	}
	if c.progress_payload != nil {
		opts.ProgressCallback = pointerHandles.Get(c.progress_payload).(CheckoutOpts).ProgressCallback
	}
	if c.target_directory != nil {
		opts.TargetDirectory = C.GoString(c.target_directory)
	}
	return opts
}

func (opts *CheckoutOpts) toC() *C.git_checkout_options {
	if opts == nil {
		return nil
	}
	c := C.git_checkout_options{}
	populateCheckoutOpts(&c, opts)
	return &c
}

//export checkoutNotifyCallback
func checkoutNotifyCallback(why C.git_checkout_notify_t, cpath *C.char, cbaseline, ctarget, cworkdir, data unsafe.Pointer) int {
	if data == nil {
		return 0
	}
	path := C.GoString(cpath)
	var baseline, target, workdir DiffFile
	if cbaseline != nil {
		baseline = diffFileFromC((*C.git_diff_file)(cbaseline))
	}
	if ctarget != nil {
		target = diffFileFromC((*C.git_diff_file)(ctarget))
	}
	if cworkdir != nil {
		workdir = diffFileFromC((*C.git_diff_file)(cworkdir))
	}
	opts := pointerHandles.Get(data).(CheckoutOpts)
	if opts.NotifyCallback == nil {
		return 0
	}
	return int(opts.NotifyCallback(CheckoutNotifyType(why), path, baseline, target, workdir))
}

//export checkoutProgressCallback
func checkoutProgressCallback(path *C.char, completed_steps, total_steps C.size_t, data unsafe.Pointer) int {
	if data == nil {
		return 0
	}
	opts := pointerHandles.Get(data).(CheckoutOpts)
	if opts.ProgressCallback == nil {
		return 0
	}
	return int(opts.ProgressCallback(C.GoString(path), uint(completed_steps), uint(total_steps)))
}

// Convert the CheckoutOpts struct to the corresponding
// C-struct. Returns a pointer to ptr, or nil if opts is nil, in order
// to help with what to pass.
func populateCheckoutOpts(ptr *C.git_checkout_options, opts *CheckoutOpts) *C.git_checkout_options {
	if opts == nil {
		return nil
	}

	C.git_checkout_init_options(ptr, 1)
	ptr.checkout_strategy = C.uint(opts.Strategy)
	ptr.disable_filters = cbool(opts.DisableFilters)
	ptr.dir_mode = C.uint(opts.DirMode.Perm())
	ptr.file_mode = C.uint(opts.FileMode.Perm())
	ptr.notify_flags = C.uint(opts.NotifyFlags)
	if opts.NotifyCallback != nil || opts.ProgressCallback != nil {
		C._go_git_populate_checkout_cb(ptr)
	}
	if opts.NotifyCallback != nil {
		ptr.notify_payload = pointerHandles.Track(*opts)
	}
	if opts.ProgressCallback != nil {
		ptr.progress_payload = pointerHandles.Track(*opts)
	}
	if opts.TargetDirectory != "" {
		ptr.target_directory = C.CString(opts.TargetDirectory)
	}
	if len(opts.Paths) > 0 {
		ptr.paths.strings = makeCStringsFromStrings(opts.Paths)
		ptr.paths.count = C.size_t(len(opts.Paths))
	}

	if opts.Baseline != nil {
		ptr.baseline = opts.Baseline.cast_ptr
	}

	return ptr
}

func freeCheckoutOpts(ptr *C.git_checkout_options) {
	if ptr == nil {
		return
	}
	C.free(unsafe.Pointer(ptr.target_directory))
	if ptr.paths.count > 0 {
		freeStrarray(&ptr.paths)
	}
	if ptr.notify_payload != nil {
		pointerHandles.Untrack(ptr.notify_payload)
	}
	if ptr.progress_payload != nil {
		pointerHandles.Untrack(ptr.progress_payload)
	}
}

// Updates files in the index and the working tree to match the content of
// the commit pointed at by HEAD. opts may be nil.
func (v *Repository) CheckoutHead(opts *CheckoutOpts) error {
	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	cOpts := opts.toC()
	defer freeCheckoutOpts(cOpts)

	ret := C.git_checkout_head(v.ptr, cOpts)
	if ret < 0 {
		return MakeGitError(ret)
	}

	return nil
}

// Updates files in the working tree to match the content of the given
// index. If index is nil, the repository's index will be used. opts
// may be nil.
func (v *Repository) CheckoutIndex(index *Index, opts *CheckoutOpts) error {
	var iptr *C.git_index = nil
	if index != nil {
		iptr = index.ptr
	}

	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	cOpts := opts.toC()
	defer freeCheckoutOpts(cOpts)

	ret := C.git_checkout_index(v.ptr, iptr, cOpts)
	if ret < 0 {
		return MakeGitError(ret)
	}

	return nil
}

func (v *Repository) CheckoutTree(tree *Tree, opts *CheckoutOpts) error {
	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	cOpts := opts.toC()
	defer freeCheckoutOpts(cOpts)

	ret := C.git_checkout_tree(v.ptr, tree.ptr, cOpts)
	if ret < 0 {
		return MakeGitError(ret)
	}

	return nil
}
<commit_msg>Checkout Payload Pointer Fix<commit_after>package git

/*
#include <git2.h>

extern void _go_git_populate_checkout_cb(git_checkout_options *opts);
*/
import "C"
import (
	"os"
	"runtime"
	"unsafe"
)

type CheckoutNotifyType uint
type CheckoutStrategy uint

const (
	CheckoutNotifyNone      CheckoutNotifyType = C.GIT_CHECKOUT_NOTIFY_NONE
	CheckoutNotifyConflict  CheckoutNotifyType = C.GIT_CHECKOUT_NOTIFY_CONFLICT
	CheckoutNotifyDirty     CheckoutNotifyType = C.GIT_CHECKOUT_NOTIFY_DIRTY
	CheckoutNotifyUpdated   CheckoutNotifyType = C.GIT_CHECKOUT_NOTIFY_UPDATED
	CheckoutNotifyUntracked CheckoutNotifyType = C.GIT_CHECKOUT_NOTIFY_UNTRACKED
	CheckoutNotifyIgnored   CheckoutNotifyType = C.GIT_CHECKOUT_NOTIFY_IGNORED
	CheckoutNotifyAll       CheckoutNotifyType = C.GIT_CHECKOUT_NOTIFY_ALL

	CheckoutNone                      CheckoutStrategy = C.GIT_CHECKOUT_NONE                         // Dry run, no actual updates
	CheckoutSafe                      CheckoutStrategy = C.GIT_CHECKOUT_SAFE                         // Allow safe updates that cannot overwrite uncommitted data
	CheckoutForce                     CheckoutStrategy = C.GIT_CHECKOUT_FORCE                        // Allow all updates to force working directory to look like index
	CheckoutRecreateMissing           CheckoutStrategy = C.GIT_CHECKOUT_RECREATE_MISSING             // Allow checkout to recreate missing files
	CheckoutAllowConflicts            CheckoutStrategy = C.GIT_CHECKOUT_ALLOW_CONFLICTS              // Allow checkout to make safe updates even if conflicts are found
	CheckoutRemoveUntracked           CheckoutStrategy = C.GIT_CHECKOUT_REMOVE_UNTRACKED             // Remove untracked files not in index (that are not ignored)
	CheckoutRemoveIgnored             CheckoutStrategy = C.GIT_CHECKOUT_REMOVE_IGNORED               // Remove ignored files not in index
	CheckoutUpdateOnly                CheckoutStrategy = C.GIT_CHECKOUT_UPDATE_ONLY                  // Only update existing files, don't create new ones
	CheckoutDontUpdateIndex           CheckoutStrategy = C.GIT_CHECKOUT_DONT_UPDATE_INDEX            // Normally checkout updates index entries as it goes; this stops that
	CheckoutNoRefresh                 CheckoutStrategy = C.GIT_CHECKOUT_NO_REFRESH                   // Don't refresh index/config/etc before doing checkout
	CheckoutSkipUnmerged              CheckoutStrategy = C.GIT_CHECKOUT_SKIP_UNMERGED                // Allow checkout to skip unmerged files
	CheckoutUserOurs                  CheckoutStrategy = C.GIT_CHECKOUT_USE_OURS                     // For unmerged files, checkout stage 2 from index
	CheckoutUseTheirs                 CheckoutStrategy = C.GIT_CHECKOUT_USE_THEIRS                   // For unmerged files, checkout stage 3 from index
	CheckoutDisablePathspecMatch      CheckoutStrategy = C.GIT_CHECKOUT_DISABLE_PATHSPEC_MATCH       // Treat pathspec as simple list of exact match file paths
	CheckoutSkipLockedDirectories     CheckoutStrategy = C.GIT_CHECKOUT_SKIP_LOCKED_DIRECTORIES      // Ignore directories in use, they will be left empty
	CheckoutDontOverwriteIgnored      CheckoutStrategy = C.GIT_CHECKOUT_DONT_OVERWRITE_IGNORED       // Don't overwrite ignored files that exist in the checkout target
	CheckoutConflictStyleMerge        CheckoutStrategy = C.GIT_CHECKOUT_CONFLICT_STYLE_MERGE         // Write normal merge files for conflicts
	CheckoutConflictStyleDiff3        CheckoutStrategy = C.GIT_CHECKOUT_CONFLICT_STYLE_DIFF3         // Include common ancestor data in diff3 format files for conflicts
	CheckoutDontRemoveExisting        CheckoutStrategy = C.GIT_CHECKOUT_DONT_REMOVE_EXISTING         // Don't overwrite existing files or folders
	CheckoutDontWriteIndex            CheckoutStrategy = C.GIT_CHECKOUT_DONT_WRITE_INDEX             // Normally checkout writes the index upon completion; this prevents that
	CheckoutUpdateSubmodules          CheckoutStrategy = C.GIT_CHECKOUT_UPDATE_SUBMODULES            // Recursively checkout submodules with same options (NOT IMPLEMENTED)
	CheckoutUpdateSubmodulesIfChanged CheckoutStrategy = C.GIT_CHECKOUT_UPDATE_SUBMODULES_IF_CHANGED // Recursively checkout submodules if HEAD moved in super repo (NOT IMPLEMENTED)
)

type CheckoutNotifyCallback func(why CheckoutNotifyType, path string, baseline, target, workdir DiffFile) ErrorCode
type CheckoutProgressCallback func(path string, completed, total uint) ErrorCode

type CheckoutOpts struct {
	Strategy         CheckoutStrategy   // Default will be a dry run
	DisableFilters   bool               // Don't apply filters like CRLF conversion
	DirMode          os.FileMode        // Default is 0755
	FileMode         os.FileMode        // Default is 0644 or 0755 as dictated by blob
	FileOpenFlags    int                // Default is O_CREAT | O_TRUNC | O_WRONLY
	NotifyFlags      CheckoutNotifyType // Default will be none
	NotifyCallback   CheckoutNotifyCallback
	ProgressCallback CheckoutProgressCallback
	TargetDirectory  string // Alternative checkout path to workdir
	Paths            []string
	Baseline         *Tree
}

func checkoutOptionsFromC(c *C.git_checkout_options) CheckoutOpts {
	opts := CheckoutOpts{}
	opts.Strategy = CheckoutStrategy(c.checkout_strategy)
	opts.DisableFilters = c.disable_filters != 0
	opts.DirMode = os.FileMode(c.dir_mode)
	opts.FileMode = os.FileMode(c.file_mode)
	opts.FileOpenFlags = int(c.file_open_flags)
	opts.NotifyFlags = CheckoutNotifyType(c.notify_flags)
	if c.notify_payload != nil {
		opts.NotifyCallback = pointerHandles.Get(c.notify_payload).(*CheckoutOpts).NotifyCallback
	}
	if c.progress_payload != nil {
		opts.ProgressCallback = pointerHandles.Get(c.progress_payload).(*CheckoutOpts).ProgressCallback
	}
	if c.target_directory != nil {
		opts.TargetDirectory = C.GoString(c.target_directory)
	}
	return opts
}

func (opts *CheckoutOpts) toC() *C.git_checkout_options {
	if opts == nil {
		return nil
	}
	c := C.git_checkout_options{}
	populateCheckoutOpts(&c, opts)
	return &c
}

//export checkoutNotifyCallback
func checkoutNotifyCallback(why C.git_checkout_notify_t, cpath *C.char, cbaseline, ctarget, cworkdir, data unsafe.Pointer) int {
	if data == nil {
		return 0
	}
	path := C.GoString(cpath)
	var baseline, target, workdir DiffFile
	if cbaseline != nil {
		baseline = diffFileFromC((*C.git_diff_file)(cbaseline))
	}
	if ctarget != nil {
		target = diffFileFromC((*C.git_diff_file)(ctarget))
	}
	if cworkdir != nil {
		workdir = diffFileFromC((*C.git_diff_file)(cworkdir))
	}
	opts := pointerHandles.Get(data).(*CheckoutOpts)
	if opts.NotifyCallback == nil {
		return 0
	}
	return int(opts.NotifyCallback(CheckoutNotifyType(why), path, baseline, target, workdir))
}

//export checkoutProgressCallback
func checkoutProgressCallback(path *C.char, completed_steps, total_steps C.size_t, data unsafe.Pointer) int {
	if data == nil {
		return 0
	}
	opts := pointerHandles.Get(data).(*CheckoutOpts)
	if opts.ProgressCallback == nil {
		return 0
	}
	return int(opts.ProgressCallback(C.GoString(path), uint(completed_steps), uint(total_steps)))
}

// Convert the CheckoutOpts struct to the corresponding
// C-struct. Returns a pointer to ptr, or nil if opts is nil, in order
// to help with what to pass.
func populateCheckoutOpts(ptr *C.git_checkout_options, opts *CheckoutOpts) *C.git_checkout_options {
	if opts == nil {
		return nil
	}

	C.git_checkout_init_options(ptr, 1)
	ptr.checkout_strategy = C.uint(opts.Strategy)
	ptr.disable_filters = cbool(opts.DisableFilters)
	ptr.dir_mode = C.uint(opts.DirMode.Perm())
	ptr.file_mode = C.uint(opts.FileMode.Perm())
	ptr.notify_flags = C.uint(opts.NotifyFlags)
	if opts.NotifyCallback != nil || opts.ProgressCallback != nil {
		C._go_git_populate_checkout_cb(ptr)
	}
	payload := pointerHandles.Track(opts)
	if opts.NotifyCallback != nil {
		ptr.notify_payload = payload
	}
	if opts.ProgressCallback != nil {
		ptr.progress_payload = payload
	}
	if opts.TargetDirectory != "" {
		ptr.target_directory = C.CString(opts.TargetDirectory)
	}
	if len(opts.Paths) > 0 {
		ptr.paths.strings = makeCStringsFromStrings(opts.Paths)
		ptr.paths.count = C.size_t(len(opts.Paths))
	}

	if opts.Baseline != nil {
		ptr.baseline = opts.Baseline.cast_ptr
	}

	return ptr
}

func freeCheckoutOpts(ptr *C.git_checkout_options) {
	if ptr == nil {
		return
	}
	C.free(unsafe.Pointer(ptr.target_directory))
	if ptr.paths.count > 0 {
		freeStrarray(&ptr.paths)
	}
	if ptr.notify_payload != nil {
		pointerHandles.Untrack(ptr.notify_payload)
	}
}

// Updates files in the index and the working tree to match the content of
// the commit pointed at by HEAD. opts may be nil.
func (v *Repository) CheckoutHead(opts *CheckoutOpts) error {
	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	cOpts := opts.toC()
	defer freeCheckoutOpts(cOpts)

	ret := C.git_checkout_head(v.ptr, cOpts)
	if ret < 0 {
		return MakeGitError(ret)
	}

	return nil
}

// Updates files in the working tree to match the content of the given
// index. If index is nil, the repository's index will be used. opts
// may be nil.
func (v *Repository) CheckoutIndex(index *Index, opts *CheckoutOpts) error {
	var iptr *C.git_index = nil
	if index != nil {
		iptr = index.ptr
	}

	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	cOpts := opts.toC()
	defer freeCheckoutOpts(cOpts)

	ret := C.git_checkout_index(v.ptr, iptr, cOpts)
	if ret < 0 {
		return MakeGitError(ret)
	}

	return nil
}

func (v *Repository) CheckoutTree(tree *Tree, opts *CheckoutOpts) error {
	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	cOpts := opts.toC()
	defer freeCheckoutOpts(cOpts)

	ret := C.git_checkout_tree(v.ptr, tree.ptr, cOpts)
	if ret < 0 {
		return MakeGitError(ret)
	}

	return nil
}
<|endoftext|>


<commit_before>package id3

import (
	"bytes"
	"encoding/binary"
	"io"
	"io/ioutil"
)

type Peeker interface {
	Peek(n int) ([]byte, error)
}

// Check reports whether r looks like it starts with an ID3 tag.
func Check(r Peeker) (bool, error) {
	b, err := r.Peek(3)
	if err != nil {
		return false, err
	}
	return bytes.Equal(b, Magic[:]), nil
}

type Decoder struct {
	r io.Reader
	h TagHeader
}

func NewDecoder(r io.Reader) *Decoder {
	return &Decoder{r: r}
}

// ParseHeader parses only the ID3 header.
func (d *Decoder) ParseHeader() (TagHeader, error) {
	header, err := d.readHeader()
	if err != nil {
		return TagHeader{}, err
	}

	d.h = header
	d.r = io.LimitReader(d.r, int64(header.size))

	return header, nil
}

func (d *Decoder) remaining() int64 {
	return d.r.(*io.LimitedReader).N
}

// Parse parses a tag.
//
// Parse will always return a valid tag. In the case of an error, the
// tag will be empty.
//
// If Parse successfully parsed a tag, the reader will be positioned
// immediately after the tag, which usually is directly before audio
// data. If there wasn't a valid tag, however, the position of the
// reader is undefined. If you're not sure if your reader starts with
// a tag at all, consider using Check first.
func (d *Decoder) Parse() (*Tag, error) {
	// TODO return how many bytes we read into the reader; so people
	// know where the audio begins
	tag := NewTag()
	header, err := d.ParseHeader()
	if err != nil {
		return tag, err
	}
	tag.Header = header

	// FIXME consider moving this to ParseHeader
	if header.Flags.ExtendedHeader() {
		panic("not implemented: cannot parse extended header")
	}

	if header.Flags.Unsynchronisation() {
		panic("not implemented: cannot parse unsynchronised tag")
	}

	for {
		frame, err := d.ParseFrame()
		if err != nil {
			if err == io.EOF {
				break
			}

			return tag, err
		}
		tag.Frames[frame.ID()] = append(tag.Frames[frame.ID()], frame)
	}

	if header.Version < 0x0400 {
		tag.upgrade()
	}

	return tag, nil
}

func readBinary(r io.Reader, args ...interface{}) (err error) {
	for _, arg := range args {
		err = binary.Read(r, binary.BigEndian, arg)
		if err != nil {
			break
		}
	}

	return
}

// readHeader reads an ID3v2 header. It expects the reader to be
// seeked to the beginning of the header.
func (d *Decoder) readHeader() (header TagHeader, err error) {
	var bytes struct {
		Magic   [3]byte
		Version [2]byte
		Flags   byte
		Size    [4]byte
	}

	err = binary.Read(d.r, binary.BigEndian, &bytes)
	if err != nil {
		return header, err
	}
	if bytes.Magic != Magic {
		return TagHeader{}, notATagHeader{bytes.Magic}
	}
	version := Version(int16(bytes.Version[0])<<8 | int16(bytes.Version[1]))
	if bytes.Version[0] > 4 || bytes.Version[0] < 3 {
		return TagHeader{}, UnsupportedVersion{version}
	}

	header.Version = version
	header.Flags = HeaderFlags(bytes.Flags)
	header.size = desynchsafeInt(bytes.Size)

	return header, nil
}

// ParseFrame reads the next ID3 frame. When it reaches padding, it
// will discard it and return io.EOF. This should set the reader
// immediately before the audio data.
func (d *Decoder) ParseFrame() (Frame, error) {
	if d.remaining() == 0 {
		return nil, io.EOF
	}

	var (
		headerBytes struct {
			ID    [4]byte
			Size  [4]byte
			Flags [2]byte
		}
		header FrameHeader
	)

	err := binary.Read(d.r, binary.BigEndian, &headerBytes)
	if err != nil {
		return nil, err
	}

	// We're in the padding, discard remaining bytes and return io.EOF
	if headerBytes.ID == [4]byte{0, 0, 0, 0} {
		_, err := io.Copy(ioutil.Discard, d.r)
		if err != nil {
			return nil, err
		}
		return nil, io.EOF
	}

	for _, byte := range headerBytes.ID {
		// Allow 0-9
		if byte >= 48 && byte <= 57 {
			continue
		}

		// Allow A-Z
		if byte >= 65 && byte <= 90 {
			continue
		}

		return nil, NotAFrameHeader{headerBytes}
	}

	header.id = FrameType(headerBytes.ID[:])
	header.flags = FrameFlags(int16(headerBytes.Flags[0])<<8 | int16(headerBytes.Flags[1]))
	frameSize := desynchsafeInt(headerBytes.Size)

	if header.flags.Compressed() {
		panic("not implemented: cannot read compressed frame")
		// TODO: Read decompressed size (4 bytes)
	}

	if header.flags.Encrypted() {
		panic("not implemented: cannot read encrypted frame")
		// TODO: Read encryption method (1 byte)
	}

	if header.flags.Grouped() {
		panic("not implemented: cannot read grouped frame")
		// TODO: Read group identifier (1 byte)
	}

	if header.id[0] == 'T' && header.id != "TXXX" {
		var encoding Encoding
		frame := TextInformationFrame{FrameHeader: header}
		information := make([]byte, frameSize-1)
		err := readBinary(d.r, &encoding, &information)
		if err != nil {
			return nil, err
		}

		frame.Text = string(encoding.toUTF8(information))

		return frame, nil
	}

	if header.id[0] == 'W' && header.id != "WXXX" {
		frame := URLLinkFrame{FrameHeader: header}
		url := make([]byte, frameSize)
		_, err = d.r.Read(url)
		if err != nil {
			return nil, err
		}
		frame.URL = string(iso88591.toUTF8(url))

		return frame, nil
	}

	switch header.id {
	case "TXXX":
		return d.readTXXXFrame(header, frameSize)
	case "WXXX":
		return d.readWXXXFrame(header, frameSize)
	case "UFID":
		return d.readUFIDFrame(header, frameSize)
	case "COMM":
		return d.readCOMMFrame(header, frameSize)
	case "PRIV":
		return d.readPRIVFrame(header, frameSize)
	case "APIC":
		return d.readAPICFrame(header, frameSize)
	case "MCDI":
		return d.readMCDIFrame(header, frameSize)
	case "USLT":
		return d.readUFIDFrame(header, frameSize)
	default:
		data := make([]byte, frameSize)
		n, err := d.r.Read(data)

		return UnsupportedFrame{
			FrameHeader: header,
			Data:        data[:n],
		}, err
	}
}

func (d *Decoder) readTXXXFrame(header FrameHeader, frameSize int) (Frame, error) {
	var encoding Encoding
	frame := UserTextInformationFrame{FrameHeader: header}
	rest := make([]byte, frameSize-1)

	err := readBinary(d.r, &encoding, &rest)
	if err != nil {
		return nil, err
	}
	parts := splitNullN(rest, encoding, 2)

	frame.Description = string(encoding.toUTF8(parts[0]))
	frame.Text = string(encoding.toUTF8(parts[1]))

	return frame, nil
}

func (d *Decoder) readWXXXFrame(header FrameHeader, frameSize int) (Frame, error) {
	var encoding Encoding
	frame := UserDefinedURLLinkFrame{FrameHeader: header}
	rest := make([]byte, frameSize-1)

	err := readBinary(d.r, &encoding, &rest)
	if err != nil {
		return nil, err
	}

	parts := splitNullN(rest, encoding, 2)
	frame.Description = string(encoding.toUTF8(parts[0]))
	frame.URL = string(iso88591ToUTF8(parts[1]))

	return frame, nil
}

func (d *Decoder) readUFIDFrame(header FrameHeader, frameSize int) (Frame, error) {
	frame := UniqueFileIdentifierFrame{FrameHeader: header}
	rest := make([]byte, frameSize)

	err := binary.Read(d.r, binary.BigEndian, rest)
	if err != nil {
		return nil, err
	}

	parts := bytes.SplitN(rest, []byte{0}, 2)
	frame.Owner = string(iso88591.toUTF8(parts[0]))
	frame.Identifier = parts[1]

	return frame, nil
}

func (d *Decoder) readCOMMFrame(header FrameHeader, frameSize int) (Frame, error) {
	frame := CommentFrame{FrameHeader: header}
	var (
		encoding Encoding
		language [3]byte
		rest     []byte
	)
	rest = make([]byte, frameSize-4)

	err := readBinary(d.r, &encoding, &language, &rest)
	if err != nil {
		return nil, err
	}

	parts := splitNullN(rest, encoding, 2)

	frame.Language = string(language[:])

	frame.Description = string(encoding.toUTF8(parts[0]))
	frame.Text = string(encoding.toUTF8(parts[1]))

	return frame, nil
}

func (d *Decoder) readPRIVFrame(header FrameHeader, frameSize int) (Frame, error) {
	frame := PrivateFrame{FrameHeader: header}
	data := make([]byte, frameSize)
	_, err := d.r.Read(data)
	if err != nil {
		return frame, err
	}

	parts := bytes.SplitN(data, nul, 2)
	frame.Owner = parts[0]
	frame.Data = parts[1]

	return frame, nil
}

func (d *Decoder) readAPICFrame(header FrameHeader, frameSize int) (Frame, error) {
	frame := PictureFrame{FrameHeader: header}
	var (
		encoding Encoding
		rest     []byte
	)
	rest = make([]byte, frameSize-1)
	err := readBinary(d.r, &encoding, &rest)
	if err != nil {
		return frame, err
	}

	parts1 := bytes.SplitN(rest, nul, 2)
	parts2 := splitNullN(parts1[1][1:], encoding, 2)

	frame.MIMEType = string(iso88591.toUTF8(parts1[0]))
	frame.PictureType = PictureType(parts1[1][0])
	frame.Description = string(encoding.toUTF8(parts2[0]))
	frame.Data = parts2[1]

	return frame, nil
}

func (d *Decoder) readMCDIFrame(header FrameHeader, frameSize int) (Frame, error) {
	frame := MusicCDIdentifierFrame{FrameHeader: header}
	frame.TOC = make([]byte, frameSize)
	_, err := d.r.Read(frame.TOC)
	return frame, err
}

func (d *Decoder) readUSLTFrame(header FrameHeader, frameSize int) (Frame, error) {
	frame := UnsynchronisedLyricsFrame{FrameHeader: header}
	var (
		encoding Encoding
		language [3]byte
		rest     []byte
	)
	rest = make([]byte, frameSize-4)

	err := readBinary(d.r, &encoding, &language, rest)
	if err != nil {
		return frame, err
	}

	parts := splitNullN(rest, encoding, 2)
	frame.Language = string(language[:])

	frame.Description = string(encoding.toUTF8(parts[0]))
	frame.Lyrics = string(encoding.toUTF8(parts[1]))

	return frame, nil
}
<commit_msg>document Parse and ParseFrame better<commit_after>package id3

import (
	"bytes"
	"encoding/binary"
	"io"
	"io/ioutil"
)

type Peeker interface {
	Peek(n int) ([]byte, error)
}

// Check reports whether r looks like it starts with an ID3 tag.
func Check(r Peeker) (bool, error) {
	b, err := r.Peek(3)
	if err != nil {
		return false, err
	}
	return bytes.Equal(b, Magic[:]), nil
}

type Decoder struct {
	r io.Reader
	h TagHeader
}

func NewDecoder(r io.Reader) *Decoder {
	return &Decoder{r: r}
}

// ParseHeader parses only the ID3 header.
func (d *Decoder) ParseHeader() (TagHeader, error) {
	header, err := d.readHeader()
	if err != nil {
		return TagHeader{}, err
	}

	d.h = header
	d.r = io.LimitReader(d.r, int64(header.size))

	return header, nil
}

func (d *Decoder) remaining() int64 {
	return d.r.(*io.LimitedReader).N
}

// Parse parses a tag.
//
// Parse will always return a valid tag. In the case of an error, the
// tag will be empty.
//
// If Parse successfully parsed a tag, the reader will be positioned
// immediately after the tag, which usually is directly before audio
// data. If there wasn't a valid tag, however, the position of the
// reader is undefined. If you're not sure if your reader starts with
// a tag at all, consider using Check first.
//
// Parse cannot be called if either ParseHeader or ParseFrame have
// been called for the current tag.
func (d *Decoder) Parse() (*Tag, error) {
	tag := NewTag()
	header, err := d.ParseHeader()
	if err != nil {
		return tag, err
	}
	tag.Header = header

	// FIXME consider moving this to ParseHeader
	if header.Flags.ExtendedHeader() {
		panic("not implemented: cannot parse extended header")
	}

	if header.Flags.Unsynchronisation() {
		panic("not implemented: cannot parse unsynchronised tag")
	}

	for {
		frame, err := d.ParseFrame()
		if err != nil {
			if err == io.EOF {
				break
			}

			return tag, err
		}
		tag.Frames[frame.ID()] = append(tag.Frames[frame.ID()], frame)
	}

	if header.Version < 0x0400 {
		tag.upgrade()
	}

	return tag, nil
}

func readBinary(r io.Reader, args ...interface{}) (err error) {
	for _, arg := range args {
		err = binary.Read(r, binary.BigEndian, arg)
		if err != nil {
			break
		}
	}

	return
}

// readHeader reads an ID3v2 header. It expects the reader to be
// seeked to the beginning of the header.
func (d *Decoder) readHeader() (header TagHeader, err error) {
	var bytes struct {
		Magic   [3]byte
		Version [2]byte
		Flags   byte
		Size    [4]byte
	}

	err = binary.Read(d.r, binary.BigEndian, &bytes)
	if err != nil {
		return header, err
	}
	if bytes.Magic != Magic {
		return TagHeader{}, notATagHeader{bytes.Magic}
	}
	version := Version(int16(bytes.Version[0])<<8 | int16(bytes.Version[1]))
	if bytes.Version[0] > 4 || bytes.Version[0] < 3 {
		return TagHeader{}, UnsupportedVersion{version}
	}

	header.Version = version
	header.Flags = HeaderFlags(bytes.Flags)
	header.size = desynchsafeInt(bytes.Size)

	return header, nil
}

// ParseFrame reads the next ID3 frame. When it reaches padding, it
// will read and discard all of it and return io.EOF. This should set
// the reader immediately before the audio data.
//
// ParseHeader must be called before calling ParseFrame.
func (d *Decoder) ParseFrame() (Frame, error) {
	if d.remaining() == 0 {
		return nil, io.EOF
	}

	var (
		headerBytes struct {
			ID    [4]byte
			Size  [4]byte
			Flags [2]byte
		}
		header FrameHeader
	)

	err := binary.Read(d.r, binary.BigEndian, &headerBytes)
	if err != nil {
		return nil, err
	}

	// We're in the padding, discard remaining bytes and return io.EOF
	if headerBytes.ID == [4]byte{0, 0, 0, 0} {
		_, err := io.Copy(ioutil.Discard, d.r)
		if err != nil {
			return nil, err
		}
		return nil, io.EOF
	}

	for _, byte := range headerBytes.ID {
		// Allow 0-9
		if byte >= 48 && byte <= 57 {
			continue
		}

		// Allow A-Z
		if byte >= 65 && byte <= 90 {
			continue
		}

		return nil, NotAFrameHeader{headerBytes}
	}

	header.id = FrameType(headerBytes.ID[:])
	header.flags = FrameFlags(int16(headerBytes.Flags[0])<<8 | int16(headerBytes.Flags[1]))
	frameSize := desynchsafeInt(headerBytes.Size)

	if header.flags.Compressed() {
		panic("not implemented: cannot read compressed frame")
		// TODO: Read decompressed size (4 bytes)
	}

	if header.flags.Encrypted() {
		panic("not implemented: cannot read encrypted frame")
		// TODO: Read encryption method (1 byte)
	}

	if header.flags.Grouped() {
		panic("not implemented: cannot read grouped frame")
		// TODO: Read group identifier (1 byte)
	}

	if header.id[0] == 'T' && header.id != "TXXX" {
		var encoding Encoding
		frame := TextInformationFrame{FrameHeader: header}
		information := make([]byte, frameSize-1)
		err := readBinary(d.r, &encoding, &information)
		if err != nil {
			return nil, err
		}

		frame.Text = string(encoding.toUTF8(information))

		return frame, nil
	}

	if header.id[0] == 'W' && header.id != "WXXX" {
		frame := URLLinkFrame{FrameHeader: header}
		url := make([]byte, frameSize)
		_, err = d.r.Read(url)
		if err != nil {
			return nil, err
		}
		frame.URL = string(iso88591.toUTF8(url))

		return frame, nil
	}

	switch header.id {
	case "TXXX":
		return d.readTXXXFrame(header, frameSize)
	case "WXXX":
		return d.readWXXXFrame(header, frameSize)
	case "UFID":
		return d.readUFIDFrame(header, frameSize)
	case "COMM":
		return d.readCOMMFrame(header, frameSize)
	case "PRIV":
		return d.readPRIVFrame(header, frameSize)
	case "APIC":
		return d.readAPICFrame(header, frameSize)
	case "MCDI":
		return d.readMCDIFrame(header, frameSize)
	case "USLT":
		return d.readUFIDFrame(header, frameSize)
	default:
		data := make([]byte, frameSize)
		n, err := d.r.Read(data)

		return UnsupportedFrame{
			FrameHeader: header,
			Data:        data[:n],
		}, err
	}
}

func (d *Decoder) readTXXXFrame(header FrameHeader, frameSize int) (Frame, error) {
	var encoding Encoding
	frame := UserTextInformationFrame{FrameHeader: header}
	rest := make([]byte, frameSize-1)

	err := readBinary(d.r, &encoding, &rest)
	if err != nil {
		return nil, err
	}
	parts := splitNullN(rest, encoding, 2)

	frame.Description = string(encoding.toUTF8(parts[0]))
	frame.Text = string(encoding.toUTF8(parts[1]))

	return frame, nil
}

func (d *Decoder) readWXXXFrame(header FrameHeader, frameSize int) (Frame, error) {
	var encoding Encoding
	frame := UserDefinedURLLinkFrame{FrameHeader: header}
	rest := make([]byte, frameSize-1)

	err := readBinary(d.r, &encoding, &rest)
	if err != nil {
		return nil, err
	}

	parts := splitNullN(rest, encoding, 2)
	frame.Description = string(encoding.toUTF8(parts[0]))
	frame.URL = string(iso88591ToUTF8(parts[1]))

	return frame, nil
}

func (d *Decoder) readUFIDFrame(header FrameHeader, frameSize int) (Frame, error) {
	frame := UniqueFileIdentifierFrame{FrameHeader: header}
	rest := make([]byte, frameSize)

	err := binary.Read(d.r, binary.BigEndian, rest)
	if err != nil {
		return nil, err
	}

	parts := bytes.SplitN(rest, []byte{0}, 2)
	frame.Owner = string(iso88591.toUTF8(parts[0]))
	frame.Identifier = parts[1]

	return frame, nil
}

func (d *Decoder) readCOMMFrame(header FrameHeader, frameSize int) (Frame, error) {
	frame := CommentFrame{FrameHeader: header}
	var (
		encoding Encoding
		language [3]byte
		rest     []byte
	)
	rest = make([]byte, frameSize-4)

	err := readBinary(d.r, &encoding, &language, &rest)
	if err != nil {
		return nil, err
	}

	parts := splitNullN(rest, encoding, 2)

	frame.Language = string(language[:])

	frame.Description = string(encoding.toUTF8(parts[0]))
	frame.Text = string(encoding.toUTF8(parts[1]))

	return frame, nil
}

func (d *Decoder) readPRIVFrame(header FrameHeader, frameSize int) (Frame, error) {
	frame := PrivateFrame{FrameHeader: header}
	data := make([]byte, frameSize)
	_, err := d.r.Read(data)
	if err != nil {
		return frame, err
	}

	parts := bytes.SplitN(data, nul, 2)
	frame.Owner = parts[0]
	frame.Data = parts[1]

	return frame, nil
}

func (d *Decoder) readAPICFrame(header FrameHeader, frameSize int) (Frame, error) {
	frame := PictureFrame{FrameHeader: header}
	var (
		encoding Encoding
		rest     []byte
	)
	rest = make([]byte, frameSize-1)
	err := readBinary(d.r, &encoding, &rest)
	if err != nil {
		return frame, err
	}

	parts1 := bytes.SplitN(rest, nul, 2)
	parts2 := splitNullN(parts1[1][1:], encoding, 2)

	frame.MIMEType = string(iso88591.toUTF8(parts1[0]))
	frame.PictureType = PictureType(parts1[1][0])
	frame.Description = string(encoding.toUTF8(parts2[0]))
	frame.Data = parts2[1]

	return frame, nil
}

func (d *Decoder) readMCDIFrame(header FrameHeader, frameSize int) (Frame, error) {
	frame := MusicCDIdentifierFrame{FrameHeader: header}
	frame.TOC = make([]byte, frameSize)
	_, err := d.r.Read(frame.TOC)
	return frame, err
}

func (d *Decoder) readUSLTFrame(header FrameHeader, frameSize int) (Frame, error) {
	frame := UnsynchronisedLyricsFrame{FrameHeader: header}
	var (
		encoding Encoding
		language [3]byte
		rest     []byte
	)
	rest = make([]byte, frameSize-4)

	err := readBinary(d.r, &encoding, &language, rest)
	if err != nil {
		return frame, err
	}

	parts := splitNullN(rest, encoding, 2)
	frame.Language = string(language[:])

	frame.Description = string(encoding.toUTF8(parts[0]))
	frame.Lyrics = string(encoding.toUTF8(parts[1]))

	return frame, nil
}
<|endoftext|>


<commit_before>package pg

import (
	"encoding/hex"
	"reflect"
	"strconv"
	"time"
)

var (
	timeType = reflect.TypeOf((*time.Time)(nil)).Elem()
)

func Decode(dst interface{}, f []byte) error {
	// NULL.
	if f == nil {
		return nil
	}

	switch v := dst.(type) {
	case *bool:
		if len(f) == 1 && f[0] == 't' {
			*v = true
		} else {
			*v = false
		}
		return nil
	case *int8:
		n, err := strconv.ParseInt(string(f), 10, 8)
		if err != nil {
			return err
		}
		*v = int8(n)
		return nil
	case *int16:
		n, err := strconv.ParseInt(string(f), 10, 16)
		if err != nil {
			return err
		}
		*v = int16(n)
		return nil
	case *int32:
		n, err := strconv.ParseInt(string(f), 10, 32)
		if err != nil {
			return err
		}
		*v = int32(n)
		return nil
	case *int64:
		n, err := strconv.ParseInt(string(f), 10, 64)
		if err != nil {
			return err
		}
		*v = n
		return nil
	case *int:
		n, err := strconv.ParseInt(string(f), 10, 64)
		if err != nil {
			return err
		}
		*v = int(n)
		return nil
	case *uint8:
		n, err := strconv.ParseUint(string(f), 10, 8)
		if err != nil {
			return err
		}
		*v = uint8(n)
		return nil
	case *uint16:
		n, err := strconv.ParseUint(string(f), 10, 16)
		if err != nil {
			return err
		}
		*v = uint16(n)
		return nil
	case *uint32:
		n, err := strconv.ParseUint(string(f), 10, 32)
		if err != nil {
			return err
		}
		*v = uint32(n)
		return nil
	case *uint64:
		n, err := strconv.ParseUint(string(f), 10, 64)
		if err != nil {
			return err
		}
		*v = n
		return nil
	case *uint:
		n, err := strconv.ParseInt(string(f), 10, 64)
		if err != nil {
			return err
		}
		*v = uint(n)
		return nil
	case *float32:
		n, err := strconv.ParseFloat(string(f), 32)
		if err != nil {
			return err
		}
		*v = float32(n)
		return nil
	case *float64:
		n, err := strconv.ParseFloat(string(f), 64)
		if err != nil {
			return err
		}
		*v = n
		return nil
	case *string:
		*v = string(f)
		return nil
	case *[]byte:
		b, err := decodeBytes(f)
		if err != nil {
			return err
		}
		*v = b
		return nil
	case *time.Time:
		tm, err := decodeTime(f)
		if err != nil {
			return err
		}
		*v = tm
		return nil
	case *[]string:
		s, err := decodeStringSlice(f)
		if err != nil {
			return err
		}
		*v = s
		return nil
	case *[]int:
		s, err := decodeIntSlice(f)
		if err != nil {
			return err
		}
		*v = s
		return nil
	case *[]int64:
		s, err := decodeInt64Slice(f)
		if err != nil {
			return err
		}
		*v = s
		return nil
	case *map[string]string:
		m, err := decodeStringStringMap(f)
		if err != nil {
			return err
		}
		*v = m
		return nil
	}

	v := reflect.ValueOf(dst)
	if !v.IsValid() {
		return errorf("pg: Decode(%q)", v)
	}
	return decodeValue(v.Elem(), f)
}

func decodeValue(dst reflect.Value, f []byte) error {
	// NULL.
	if f == nil {
		return nil
	}

	kind := dst.Kind()
	if kind == reflect.Ptr {
		return decodeValue(dst.Elem(), f)
	}
	switch kind {
	case reflect.Bool:
		if len(f) == 1 && f[0] == 't' {
			dst.SetBool(true)
		} else {
			dst.SetBool(false)
		}
		return nil
	case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int:
		n, err := strconv.ParseInt(string(f), 10, 64)
		if err != nil {
			return err
		}
		dst.SetInt(n)
		return nil
	case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uint:
		n, err := strconv.ParseInt(string(f), 10, 64)
		if err != nil {
			return err
		}
		dst.SetUint(uint64(n))
		return nil
	case reflect.Float32, reflect.Float64:
		n, err := strconv.ParseFloat(string(f), 64)
		if err != nil {
			return err
		}
		dst.SetFloat(n)
		return nil
	case reflect.String:
		dst.SetString(string(f))
		return nil
	case reflect.Slice:
		return decodeSliceValue(dst, f)
	case reflect.Map:
		return decodeMapValue(dst, f)
	case reflect.Struct:
		if dst.Type() == timeType {
			tm, err := decodeTime(f)
			if err != nil {
				return err
			}
			dst.Set(reflect.ValueOf(tm))
			return nil
		}
	case reflect.Interface:
		return decodeValue(dst.Elem(), f)
	}
	return errorf("pg: unsupported dst: %T", dst.Interface())
}

func decodeSliceValue(dst reflect.Value, f []byte) error {
	elemType := dst.Type().Elem()
	switch elemType.Kind() {
	case reflect.Uint8:
		b, err := decodeBytes(f)
		if err != nil {
			return err
		}
		dst.SetBytes(b)
		return nil
	case reflect.String:
		s, err := decodeStringSlice(f)
		if err != nil {
			return err
		}
		dst.Set(reflect.ValueOf(s))
		return nil
	case reflect.Int:
		s, err := decodeIntSlice(f)
		if err != nil {
			return err
		}
		dst.Set(reflect.ValueOf(s))
		return nil
	case reflect.Int64:
		s, err := decodeInt64Slice(f)
		if err != nil {
			return err
		}
		dst.Set(reflect.ValueOf(s))
		return nil
	}
	return errorf("pg: unsupported dst: %T", dst.Interface())
}

func decodeMapValue(dst reflect.Value, f []byte) error {
	typ := dst.Type()
	if typ.Key().Kind() == reflect.String && typ.Elem().Kind() == reflect.String {
		m, err := decodeStringStringMap(f)
		if err != nil {
			return err
		}
		dst.Set(reflect.ValueOf(m))
		return nil
	}
	return errorf("pg: unsupported dst: %T", dst.Interface())
}

func decodeBytes(f []byte) ([]byte, error) {
	f = f[2:] // Trim off "\\x".
	b := make([]byte, hex.DecodedLen(len(f)))
	_, err := hex.Decode(b, f)
	if err != nil {
		return nil, err
	}
	return b, nil
}

func decodeTime(f []byte) (time.Time, error) {
	switch l := len(f); {
	case l <= len(dateFormat):
		return time.Parse(dateFormat, string(f))
	case l <= len(timeFormat):
		return time.Parse(timeFormat, string(f))
	default:
		if c := f[len(f)-5]; c == '+' || c == '-' {
			return time.Parse(timestampWithTzFormat, string(f))
		}
		if c := f[len(f)-3]; c == '+' || c == '-' {
			return time.Parse(timestampWithTzFormat2, string(f))
		}
		return time.Parse(timestampFormat, string(f))
	}
}

func decodeIntSlice(f []byte) ([]int, error) {
	p := newArrayParser(f[1 : len(f)-1])
	s := make([]int, 0)
	for p.Valid() {
		elem, err := p.NextElem()
		if err != nil {
			return nil, err
		}
		if elem == nil {
			return nil, errorf("pg: unexpected NULL: %q", f)
		}
		n, err := strconv.Atoi(string(elem))
		if err != nil {
			return nil, err
		}
		s = append(s, n)
	}
	return s, nil
}

func decodeInt64Slice(f []byte) ([]int64, error) {
	p := newArrayParser(f[1 : len(f)-1])
	s := make([]int64, 0)
	for p.Valid() {
		elem, err := p.NextElem()
		if err != nil {
			return nil, err
		}
		if elem == nil {
			return nil, errorf("pg: unexpected NULL: %q", f)
		}
		n, err := strconv.ParseInt(string(elem), 10, 64)
		if err != nil {
			return nil, err
		}
		s = append(s, n)
	}
	return s, nil
}

func decodeStringSlice(f []byte) ([]string, error) {
	p := newArrayParser(f[1 : len(f)-1])
	s := make([]string, 0)
	for p.Valid() {
		elem, err := p.NextElem()
		if err != nil {
			return nil, err
		}
		if elem == nil {
			return nil, errorf("pg: unexpected NULL: %q", f)
		}
		s = append(s, string(elem))
	}
	return s, nil
}

func decodeStringStringMap(f []byte) (map[string]string, error) {
	p := newHstoreParser(f)
	m := make(map[string]string)
	for p.Valid() {
		key, err := p.NextKey()
		if err != nil {
			return nil, err
		}
		if key == nil {
			return nil, errorf("pg: unexpected NULL: %q", f)
		}
		value, err := p.NextValue()
		if err != nil {
			return nil, err
		}
		if value == nil {
			return nil, errorf("pg: unexpected NULL: %q", f)
		}
		m[string(key)] = string(value)
	}
	return m, nil
}
<commit_msg>decodeValue() can now decode arbitrary TextMarshalers<commit_after>package pg

import (
	"encoding"
	"encoding/hex"
	"reflect"
	"strconv"
	"time"
)

var (
	timeType = reflect.TypeOf((*time.Time)(nil)).Elem()
)

func Decode(dst interface{}, f []byte) error {
	// NULL.
	if f == nil {
		return nil
	}

	switch v := dst.(type) {
	case *bool:
		if len(f) == 1 && f[0] == 't' {
			*v = true
		} else {
			*v = false
		}
		return nil
	case *int8:
		n, err := strconv.ParseInt(string(f), 10, 8)
		if err != nil {
			return err
		}
		*v = int8(n)
		return nil
	case *int16:
		n, err := strconv.ParseInt(string(f), 10, 16)
		if err != nil {
			return err
		}
		*v = int16(n)
		return nil
	case *int32:
		n, err := strconv.ParseInt(string(f), 10, 32)
		if err != nil {
			return err
		}
		*v = int32(n)
		return nil
	case *int64:
		n, err := strconv.ParseInt(string(f), 10, 64)
		if err != nil {
			return err
		}
		*v = n
		return nil
	case *int:
		n, err := strconv.ParseInt(string(f), 10, 64)
		if err != nil {
			return err
		}
		*v = int(n)
		return nil
	case *uint8:
		n, err := strconv.ParseUint(string(f), 10, 8)
		if err != nil {
			return err
		}
		*v = uint8(n)
		return nil
	case *uint16:
		n, err := strconv.ParseUint(string(f), 10, 16)
		if err != nil {
			return err
		}
		*v = uint16(n)
		return nil
	case *uint32:
		n, err := strconv.ParseUint(string(f), 10, 32)
		if err != nil {
			return err
		}
		*v = uint32(n)
		return nil
	case *uint64:
		n, err := strconv.ParseUint(string(f), 10, 64)
		if err != nil {
			return err
		}
		*v = n
		return nil
	case *uint:
		n, err := strconv.ParseInt(string(f), 10, 64)
		if err != nil {
			return err
		}
		*v = uint(n)
		return nil
	case *float32:
		n, err := strconv.ParseFloat(string(f), 32)
		if err != nil {
			return err
		}
		*v = float32(n)
		return nil
	case *float64:
		n, err := strconv.ParseFloat(string(f), 64)
		if err != nil {
			return err
		}
		*v = n
		return nil
	case *string:
		*v = string(f)
		return nil
	case *[]byte:
		b, err := decodeBytes(f)
		if err != nil {
			return err
		}
		*v = b
		return nil
	case *time.Time:
		tm, err := decodeTime(f)
		if err != nil {
			return err
		}
		*v = tm
		return nil
	case *[]string:
		s, err := decodeStringSlice(f)
		if err != nil {
			return err
		}
		*v = s
		return nil
	case *[]int:
		s, err := decodeIntSlice(f)
		if err != nil {
			return err
		}
		*v = s
		return nil
	case *[]int64:
		s, err := decodeInt64Slice(f)
		if err != nil {
			return err
		}
		*v = s
		return nil
	case *map[string]string:
		m, err := decodeStringStringMap(f)
		if err != nil {
			return err
		}
		*v = m
		return nil
	}

	v := reflect.ValueOf(dst)
	if !v.IsValid() {
		return errorf("pg: Decode(%q)", v)
	}
	return decodeValue(v.Elem(), f)
}

func decodeValue(dst reflect.Value, f []byte) error {
	// NULL.
	if f == nil {
		return nil
	}

	kind := dst.Kind()
	if kind == reflect.Ptr {
		if dst.IsNil() {
			// must point to an empty value before proceeding
			dst.Set(reflect.New(dst.Type().Elem()))
		}

		return decodeValue(dst.Elem(), f)
	}
	switch kind {
	case reflect.Bool:
		if len(f) == 1 && f[0] == 't' {
			dst.SetBool(true)
		} else {
			dst.SetBool(false)
		}
		return nil
	case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int:
		n, err := strconv.ParseInt(string(f), 10, 64)
		if err != nil {
			return err
		}
		dst.SetInt(n)
		return nil
	case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uint:
		n, err := strconv.ParseInt(string(f), 10, 64)
		if err != nil {
			return err
		}
		dst.SetUint(uint64(n))
		return nil
	case reflect.Float32, reflect.Float64:
		n, err := strconv.ParseFloat(string(f), 64)
		if err != nil {
			return err
		}
		dst.SetFloat(n)
		return nil
	case reflect.String:
		dst.SetString(string(f))
		return nil
	case reflect.Slice:
		return decodeSliceValue(dst, f)
	case reflect.Map:
		return decodeMapValue(dst, f)
	case reflect.Struct:
		if dst.Type() == timeType {
			tm, err := decodeTime(f)
			if err != nil {
				return err
			}
			dst.Set(reflect.ValueOf(tm))
			return nil
		}

		iface := dst.Addr().Interface()
		unmarshaler, ok := iface.(encoding.TextUnmarshaler)
		if ok {
			return unmarshaler.UnmarshalText(f)
		}
	case reflect.Interface:
		return decodeValue(dst.Elem(), f)
	}
	return errorf("pg: unsupported dst: %T", dst.Interface())
}

func decodeSliceValue(dst reflect.Value, f []byte) error {
	elemType := dst.Type().Elem()
	switch elemType.Kind() {
	case reflect.Uint8:
		b, err := decodeBytes(f)
		if err != nil {
			return err
		}
		dst.SetBytes(b)
		return nil
	case reflect.String:
		s, err := decodeStringSlice(f)
		if err != nil {
			return err
		}
		dst.Set(reflect.ValueOf(s))
		return nil
	case reflect.Int:
		s, err := decodeIntSlice(f)
		if err != nil {
			return err
		}
		dst.Set(reflect.ValueOf(s))
		return nil
	case reflect.Int64:
		s, err := decodeInt64Slice(f)
		if err != nil {
			return err
		}
		dst.Set(reflect.ValueOf(s))
		return nil
	}
	return errorf("pg: unsupported dst: %T", dst.Interface())
}

func decodeMapValue(dst reflect.Value, f []byte) error {
	typ := dst.Type()
	if typ.Key().Kind() == reflect.String && typ.Elem().Kind() == reflect.String {
		m, err := decodeStringStringMap(f)
		if err != nil {
			return err
		}
		dst.Set(reflect.ValueOf(m))
		return nil
	}
	return errorf("pg: unsupported dst: %T", dst.Interface())
}

func decodeBytes(f []byte) ([]byte, error) {
	f = f[2:] // Trim off "\\x".
	b := make([]byte, hex.DecodedLen(len(f)))
	_, err := hex.Decode(b, f)
	if err != nil {
		return nil, err
	}
	return b, nil
}

func decodeTime(f []byte) (time.Time, error) {
	switch l := len(f); {
	case l <= len(dateFormat):
		return time.Parse(dateFormat, string(f))
	case l <= len(timeFormat):
		return time.Parse(timeFormat, string(f))
	default:
		if c := f[len(f)-5]; c == '+' || c == '-' {
			return time.Parse(timestampWithTzFormat, string(f))
		}
		if c := f[len(f)-3]; c == '+' || c == '-' {
			return time.Parse(timestampWithTzFormat2, string(f))
		}
		return time.Parse(timestampFormat, string(f))
	}
}

func decodeIntSlice(f []byte) ([]int, error) {
	p := newArrayParser(f[1 : len(f)-1])
	s := make([]int, 0)
	for p.Valid() {
		elem, err := p.NextElem()
		if err != nil {
			return nil, err
		}
		if elem == nil {
			return nil, errorf("pg: unexpected NULL: %q", f)
		}
		n, err := strconv.Atoi(string(elem))
		if err != nil {
			return nil, err
		}
		s = append(s, n)
	}
	return s, nil
}

func decodeInt64Slice(f []byte) ([]int64, error) {
	p := newArrayParser(f[1 : len(f)-1])
	s := make([]int64, 0)
	for p.Valid() {
		elem, err := p.NextElem()
		if err != nil {
			return nil, err
		}
		if elem == nil {
			return nil, errorf("pg: unexpected NULL: %q", f)
		}
		n, err := strconv.ParseInt(string(elem), 10, 64)
		if err != nil {
			return nil, err
		}
		s = append(s, n)
	}
	return s, nil
}

func decodeStringSlice(f []byte) ([]string, error) {
	p := newArrayParser(f[1 : len(f)-1])
	s := make([]string, 0)
	for p.Valid() {
		elem, err := p.NextElem()
		if err != nil {
			return nil, err
		}
		if elem == nil {
			return nil, errorf("pg: unexpected NULL: %q", f)
		}
		s = append(s, string(elem))
	}
	return s, nil
}

func decodeStringStringMap(f []byte) (map[string]string, error) {
	p := newHstoreParser(f)
	m := make(map[string]string)
	for p.Valid() {
		key, err := p.NextKey()
		if err != nil {
			return nil, err
		}
		if key == nil {
			return nil, errorf("pg: unexpected NULL: %q", f)
		}
		value, err := p.NextValue()
		if err != nil {
			return nil, err
		}
		if value == nil {
			return nil, errorf("pg: unexpected NULL: %q", f)
		}
		m[string(key)] = string(value)
	}
	return m, nil
}
<|endoftext|>


<commit_before>package sparta

import (
	"os"
	"testing"
)

func TestDescribe(t *testing.T) {
	logger, err := NewLogger("info")
	output, err := os.Create("./graph.html")
	if nil != err {
		t.Fatalf(err.Error())
		return
	}
	defer output.Close()
	err = Describe("SampleService", "SampleService Description", testLambdaData(), nil, nil, output, logger)
	if nil != err {
		t.Errorf("Failed to describe: %s", err)
	}
}
<commit_msg>Update Describe signature to include nil WorkflowHooks<commit_after>package sparta

import (
	"os"
	"testing"
)

func TestDescribe(t *testing.T) {
	logger, err := NewLogger("info")
	output, err := os.Create("./graph.html")
	if nil != err {
		t.Fatalf(err.Error())
		return
	}
	defer output.Close()
	err = Describe("SampleService",
		"SampleService Description",
		testLambdaData(),
		nil,
		nil,
		output,
		nil,
		logger)
	if nil != err {
		t.Errorf("Failed to describe: %s", err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2011, Bryan Matsuo. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main
/*
 *  Filename:    goline-questions.go
 *  Author:      Bryan Matsuo <bmatsuo@soe.ucsc.edu>
 *  Created:     Sat Aug 13 21:39:57 PDT 2011
 *  Description:
 *  Usage:       goline-questions [options] ARGUMENT ...
 */
import (
    "goline"
    "fmt"
    "os"
)

var opt = parseFlags()

func main() {
    cont := true
    for cont {
        // "Read a byte" but short-circuit the prompt.
        var a uint8
        goline.Ask(&a, "This should not appear:  ", func(a *goline.Answer) {
            a.FirstAnswer = uint(0xFF)
            fmt.Println(a.FirstAnswer)
            a.In(goline.UintRange{200, 255})
            a.Panic = func(err os.Error) {
                fmt.Printf("Error: %s\n", err.String())
            }
        })
        fmt.Printf("byte 0x%X\n", a)

        // Read a bounded integer.
        var b int32
        goline.Ask(&b, "Enter an int:  ", func(a *goline.Answer) {
            a.Responses[goline.AskOnError] = a.Question
            a.Default = 13
            a.In(goline.IntRange{26, 62})
            a.Panic = func(err os.Error) {
                fmt.Printf("Error: %s\n", err.String())
            }
        })
        fmt.Printf("Integer %d\n", b)

        // Read a string contained in a set of possible values.
        var broken bool
        cont = !goline.Confirm("Exit?  ", true, func(a *goline.Answer) {
            a.Panic = func(err os.Error) {
                if err == os.EOF {
                    broken = true
                }
                fmt.Printf("Error: %s\n", err.String())
            }
        })
        cont = cont && !broken
    }
}
<commit_msg>Fix API changes in the question example.<commit_after>// Copyright 2011, Bryan Matsuo. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main
/*
 *  Filename:    goline-questions.go
 *  Author:      Bryan Matsuo <bmatsuo@soe.ucsc.edu>
 *  Created:     Sat Aug 13 21:39:57 PDT 2011
 *  Description:
 *  Usage:       goline-questions [options] ARGUMENT ...
 */
import (
    "goline"
    "fmt"
    "os"
)

var opt = parseFlags()

func main() {
    cont := true
    for cont {
        // "Read a byte" but short-circuit the prompt.
        var a uint8
        goline.Ask(&a, "This should not appear:  ", func(a *goline.Question) {
            a.FirstAnswer = uint(0xFF)
            fmt.Println(a.FirstAnswer)
            a.In(goline.UintRange{200, 255})
            a.Panic = func(err os.Error) {
                fmt.Printf("Error: %s\n", err.String())
            }
        })
        fmt.Printf("byte 0x%X\n", a)

        // Read a bounded integer.
        var b int32
        goline.Ask(&b, "Enter an int:  ", func(a *goline.Question) {
            a.Responses[goline.AskOnError] = a.Question
            a.Default = 13
            a.In(goline.IntRange{26, 62})
            a.Panic = func(err os.Error) {
                fmt.Printf("Error: %s\n", err.String())
            }
        })
        fmt.Printf("Integer %d\n", b)

        // Read a string contained in a set of possible values.
        var broken bool
        cont = !goline.Confirm("Exit?  ", true, func(a *goline.Question) {
            a.Panic = func(err os.Error) {
                if err == os.EOF {
                    broken = true
                }
                fmt.Printf("Error: %s\n", err.String())
            }
        })
        cont = cont && !broken
    }
}
<|endoftext|>


<commit_before>package merge

import (
	"fmt"
	"reflect"

	"github.com/Sirupsen/logrus"
)

type Options struct {
	Overwrite  bool
	MergeFuncs *funcSelector
}

func NewOptions() *Options {
	return &Options{
		Overwrite:  true,
		MergeFuncs: newFuncSelector(),
	}
}

// helper to wrap type assertion
func MergeMapStrIface(target, src map[string]interface{}, opt *Options) (map[string]interface{}, error) {
	val, err := Merge(target, src, opt)
	if err != nil {
		return nil, err
	}

	valMap, ok := val.(map[string]interface{})
	if ok {
		return valMap, nil
	}

	return nil, fmt.Errorf("Merge failed. Expected map[string]interface{} but got %v", reflect.TypeOf(val))
}

// public wrapper
func Merge(target, source interface{}, opt *Options) (interface{}, error) {
	// use defaults if none are provided
	if opt == nil {
		opt = NewOptions()
	}
	logrus.Debugf("OPT: %v", opt)

	return merge(target, source, opt)
}

func merge(target, src interface{}, opt *Options) (interface{}, error) {
	valS := reflect.ValueOf(src)
	valT := reflect.ValueOf(target)

	//logrus.Debugf("MERGE T<>S :: %v (%v) <> %v (%v)", target, typeT, src, typeS)

	// if source is nil, skip
	if src == nil ||
		valS.Kind() == reflect.Ptr && valS.IsNil() {
		return target, nil
	}

	// if target is nil write to it
	if target == nil ||
		valT.Kind() == reflect.Ptr && valT.IsNil() {
		return src, nil
	}

	typeS := reflect.TypeOf(src)
	typeT := reflect.TypeOf(target)

	// if types do not match, bail
	if typeT != typeS {
		return nil, fmt.Errorf("Types do not match: %v, %v", typeT, typeS)
	}

	// look for a merge function
	f := opt.MergeFuncs.GetFunc(target)
	val, err := f(target, src, opt)
	if err != nil {
		return nil, err
	}

	return val, nil
}
<commit_msg>add context to opts<commit_after>package merge

import (
	"context"
	"fmt"
	"reflect"

	"github.com/Sirupsen/logrus"
)

type Options struct {
	Overwrite  bool
	MergeFuncs *funcSelector

	// to be used by merge functions to pass values down into recursive calls freely
	Context context.Context
}

func NewOptions() *Options {
	return &Options{
		Overwrite:  true,
		MergeFuncs: newFuncSelector(),
	}
}

// helper to wrap type assertion
func MergeMapStrIface(target, src map[string]interface{}, opt *Options) (map[string]interface{}, error) {
	val, err := Merge(target, src, opt)
	if err != nil {
		return nil, err
	}

	valMap, ok := val.(map[string]interface{})
	if ok {
		return valMap, nil
	}

	return nil, fmt.Errorf("Merge failed. Expected map[string]interface{} but got %v", reflect.TypeOf(val))
}

// public wrapper
func Merge(target, source interface{}, opt *Options) (interface{}, error) {
	// use defaults if none are provided
	if opt == nil {
		opt = NewOptions()
	}
	logrus.Debugf("OPT: %v", opt)

	return merge(target, source, opt)
}

func merge(target, src interface{}, opt *Options) (interface{}, error) {
	valS := reflect.ValueOf(src)
	valT := reflect.ValueOf(target)

	//logrus.Debugf("MERGE T<>S :: %v (%v) <> %v (%v)", target, typeT, src, typeS)

	// if source is nil, skip
	if src == nil ||
		valS.Kind() == reflect.Ptr && valS.IsNil() {
		return target, nil
	}

	// if target is nil write to it
	if target == nil ||
		valT.Kind() == reflect.Ptr && valT.IsNil() {
		return src, nil
	}

	typeS := reflect.TypeOf(src)
	typeT := reflect.TypeOf(target)

	// if types do not match, bail
	if typeT != typeS {
		return nil, fmt.Errorf("Types do not match: %v, %v", typeT, typeS)
	}

	// look for a merge function
	f := opt.MergeFuncs.GetFunc(target)
	val, err := f(target, src, opt)
	if err != nil {
		return nil, err
	}

	return val, nil
}
<|endoftext|>


<commit_before>package funnel

import (
	"bytes"
	"io/ioutil"
	"os"
	"path"
	"strings"
	"testing"
)

func TestRollover(t *testing.T) {
	dir, c := setupTest(t)
	defer os.RemoveAll(dir)

	f, err := os.Open("testdata/file_84lines")
	if err != nil {
		t.Fatal(err)
		return
	}
	defer f.Close()
	c.Start(f)

	// testing results
	files := readTestDir(t, dir)

	sep := []byte{'\n'}
	if len(files) != 3 {
		t.Errorf("Incorrect no. of files created. Expected 3, Got %d", len(files))
	}
	for i, file := range files {
		data, err := ioutil.ReadFile(path.Join(dir, file.Name()))
		if err != nil {
			t.Fatal(err)
			continue
		}
		numLines := bytes.Count(data, sep)
		// First 2 files will have 40 lines
		// last one will have 4 lines
		if i < 2 {
			if numLines != 40 {
				t.Errorf("Incorrect no. of lines created in file #%d. Expected 40, Got %d", i, numLines)
			}
		} else {
			if numLines != 4 {
				t.Errorf("Incorrect no. of lines created in file #%d. Expected 4, Got %d", i, numLines)
			}
		}
	}
}

func TestHugeLine(t *testing.T) {
	dir, c := setupTest(t)
	defer os.RemoveAll(dir)

	// This file also contains arabic, indian and tibetan characters
	// to test any ascii-utf8 codec incompatibility
	targetBytes, err := ioutil.ReadFile("testdata/file_bigline")
	if err != nil {
		t.Fatal(err)
		return
	}
	r := bytes.NewReader(targetBytes)
	c.Start(r)

	// testing results
	files := readTestDir(t, dir)
	if len(files) != 1 {
		t.Errorf("Incorrect no. of files created. Expected 1, Got %d", len(files))
	}
	for _, file := range files {
		data, err := ioutil.ReadFile(path.Join(dir, file.Name()))
		if err != nil {
			t.Fatal(err)
			continue
		}
		// removing the newline character at the end
		cmp := bytes.Compare(data, targetBytes)
		if cmp != 0 {
			t.Errorf("Incorrect string found. Expected- %s, Found- %s", string(targetBytes), string(data))
		}
	}
}

func TestNewLines(t *testing.T) {
	dir, c := setupTest(t)
	defer os.RemoveAll(dir)

	// This file also contains arabic, indian and tibetan characters
	// to test any ascii-utf8 codec incompatibility
	targetBytes, err := ioutil.ReadFile("testdata/file_newline")
	if err != nil {
		t.Fatal(err)
		return
	}
	r := bytes.NewReader(targetBytes)
	c.Start(r)

	// testing results
	files := readTestDir(t, dir)
	if len(files) != 1 {
		t.Errorf("Incorrect no. of files created. Expected 1, Got %d", len(files))
	}
	for _, file := range files {
		data, err := ioutil.ReadFile(path.Join(dir, file.Name()))
		if err != nil {
			t.Fatal(err)
			continue
		}
		cmp := bytes.Compare(data, targetBytes)
		if cmp != 0 {
			t.Errorf("Incorrect string found. Expected- %s, Found- %s", string(targetBytes), string(data))
		}
	}
}

func TestEmptyString(t *testing.T) {
	dir, c := setupTest(t)
	defer os.RemoveAll(dir)

	r := strings.NewReader("")
	c.Start(r)

	// testing results
	files := readTestDir(t, dir)
	if len(files) != 1 {
		t.Errorf("Incorrect no. of files created. Expected 1, Got %d", len(files))
	}
	for _, file := range files {
		data, err := ioutil.ReadFile(path.Join(dir, file.Name()))
		if err != nil {
			t.Fatal(err)
			continue
		}
		cmp := bytes.Compare(data, []byte(""))
		if cmp != 0 {
			t.Errorf("Incorrect string found. Expected- %s, Found- %s", "", string(data))
		}
	}
}

func TestRolloverSerial(t *testing.T) {
	dir, c := setupTest(t)
	c.Config.FileRenamePolicy = "serial"
	defer os.RemoveAll(dir)

	f, err := os.Open("testdata/file_84lines")
	if err != nil {
		t.Fatal(err)
		return
	}
	defer f.Close()
	c.Start(f)

	// testing results
	files := readTestDir(t, dir)

	sep := []byte{'\n'}
	if len(files) != 3 {
		t.Errorf("Incorrect no. of files created. Expected 3, Got %d", len(files))
	}
	for i, file := range files {
		data, err := ioutil.ReadFile(path.Join(dir, file.Name()))
		if err != nil {
			t.Fatal(err)
			continue
		}
		numLines := bytes.Count(data, sep)
		// First file will have 4 lines
		// rest will have 40 lines
		if i == 0 {
			if numLines != 4 {
				t.Errorf("Incorrect no. of lines created in file #%d. Expected 4, Got %d", i, numLines)
			}
		} else {
			if numLines != 40 {
				t.Errorf("Incorrect no. of lines created in file #%d. Expected 40, Got %d", i, numLines)
			}
		}
	}
}

func TestSendInterruptSerial(t *testing.T) {
	// TODO
}

// Internal helper functions
func setupTest(t *testing.T) (string, *Consumer) {
	dir, err := ioutil.TempDir("", "test")
	if err != nil {
		t.Fatal(err)
		return "", nil
	}

	c := &Consumer{
		Config: &Config{
			DirName:                  dir,
			ActiveFileName:           "out.log",
			RotationMaxLines:         40,
			RotationMaxBytes:         1000000,
			FlushingTimeIntervalSecs: 5,
			FileRenamePolicy:         "timestamp",
			MaxAge:                   int64(1 * 60 * 60),
			MaxCount:                 500,
			Target:                   "file",
		},
		LineProcessor: &NoProcessor{},
	}
	return dir, c
}

func readTestDir(t *testing.T, dir string) []os.FileInfo {
	files, err := ioutil.ReadDir(dir)
	if err != nil {
		t.Fatal(err)
		return nil
	}
	return files
}
<commit_msg>Adding file write benchmarks<commit_after>package funnel

import (
	"bytes"
	"io/ioutil"
	"math/rand"
	"os"
	"path"
	"strings"
	"syscall"
	"testing"
)

func TestRollover(t *testing.T) {
	dir, c := setupTest(t)
	defer os.RemoveAll(dir)

	f, err := os.Open("testdata/file_84lines")
	if err != nil {
		t.Fatal(err)
		return
	}
	defer f.Close()
	c.Start(f)

	// testing results
	files := readTestDir(t, dir)

	sep := []byte{'\n'}
	if len(files) != 3 {
		t.Errorf("Incorrect no. of files created. Expected 3, Got %d", len(files))
	}
	for i, file := range files {
		data, err := ioutil.ReadFile(path.Join(dir, file.Name()))
		if err != nil {
			t.Fatal(err)
			continue
		}
		numLines := bytes.Count(data, sep)
		// First 2 files will have 40 lines
		// last one will have 4 lines
		if i < 2 {
			if numLines != 40 {
				t.Errorf("Incorrect no. of lines created in file #%d. Expected 40, Got %d", i, numLines)
			}
		} else {
			if numLines != 4 {
				t.Errorf("Incorrect no. of lines created in file #%d. Expected 4, Got %d", i, numLines)
			}
		}
	}
}

func TestHugeLine(t *testing.T) {
	dir, c := setupTest(t)
	defer os.RemoveAll(dir)

	// This file also contains arabic, indian and tibetan characters
	// to test any ascii-utf8 codec incompatibility
	targetBytes, err := ioutil.ReadFile("testdata/file_bigline")
	if err != nil {
		t.Fatal(err)
		return
	}
	r := bytes.NewReader(targetBytes)
	c.Start(r)

	// testing results
	files := readTestDir(t, dir)
	if len(files) != 1 {
		t.Errorf("Incorrect no. of files created. Expected 1, Got %d", len(files))
	}
	for _, file := range files {
		data, err := ioutil.ReadFile(path.Join(dir, file.Name()))
		if err != nil {
			t.Fatal(err)
			continue
		}
		// removing the newline character at the end
		cmp := bytes.Compare(data, targetBytes)
		if cmp != 0 {
			t.Errorf("Incorrect string found. Expected- %s, Found- %s", string(targetBytes), string(data))
		}
	}
}

func TestNewLines(t *testing.T) {
	dir, c := setupTest(t)
	defer os.RemoveAll(dir)

	// This file also contains arabic, indian and tibetan characters
	// to test any ascii-utf8 codec incompatibility
	targetBytes, err := ioutil.ReadFile("testdata/file_newline")
	if err != nil {
		t.Fatal(err)
		return
	}
	r := bytes.NewReader(targetBytes)
	c.Start(r)

	// testing results
	files := readTestDir(t, dir)
	if len(files) != 1 {
		t.Errorf("Incorrect no. of files created. Expected 1, Got %d", len(files))
	}
	for _, file := range files {
		data, err := ioutil.ReadFile(path.Join(dir, file.Name()))
		if err != nil {
			t.Fatal(err)
			continue
		}
		cmp := bytes.Compare(data, targetBytes)
		if cmp != 0 {
			t.Errorf("Incorrect string found. Expected- %s, Found- %s", string(targetBytes), string(data))
		}
	}
}

func TestEmptyString(t *testing.T) {
	dir, c := setupTest(t)
	defer os.RemoveAll(dir)

	r := strings.NewReader("")
	c.Start(r)

	// testing results
	files := readTestDir(t, dir)
	if len(files) != 1 {
		t.Errorf("Incorrect no. of files created. Expected 1, Got %d", len(files))
	}
	for _, file := range files {
		data, err := ioutil.ReadFile(path.Join(dir, file.Name()))
		if err != nil {
			t.Fatal(err)
			continue
		}
		cmp := bytes.Compare(data, []byte(""))
		if cmp != 0 {
			t.Errorf("Incorrect string found. Expected- %s, Found- %s", "", string(data))
		}
	}
}

func TestRolloverSerial(t *testing.T) {
	dir, c := setupTest(t)
	c.Config.FileRenamePolicy = "serial"
	defer os.RemoveAll(dir)

	f, err := os.Open("testdata/file_84lines")
	if err != nil {
		t.Fatal(err)
		return
	}
	defer f.Close()
	c.Start(f)

	// testing results
	files := readTestDir(t, dir)

	sep := []byte{'\n'}
	if len(files) != 3 {
		t.Errorf("Incorrect no. of files created. Expected 3, Got %d", len(files))
	}
	for i, file := range files {
		data, err := ioutil.ReadFile(path.Join(dir, file.Name()))
		if err != nil {
			t.Fatal(err)
			continue
		}
		numLines := bytes.Count(data, sep)
		// First file will have 4 lines
		// rest will have 40 lines
		if i == 0 {
			if numLines != 4 {
				t.Errorf("Incorrect no. of lines created in file #%d. Expected 4, Got %d", i, numLines)
			}
		} else {
			if numLines != 40 {
				t.Errorf("Incorrect no. of lines created in file #%d. Expected 40, Got %d", i, numLines)
			}
		}
	}
}

func TestSendInterruptSerial(t *testing.T) {
	// TODO
}

// Benchmarking different file creation and status flags to check write speed
func benchmarkFileIO(b *testing.B, flags int) {
	dir, _ := ioutil.TempDir("", "test")
	f, _ := os.OpenFile(path.Join(dir, "testspeed"),
		flags,
		0644)
	defer f.Sync()
	defer f.Close()
	defer os.Remove(f.Name())
	defer os.RemoveAll(dir)

	for n := 0; n < b.N; n++ {
		f.Write(randStringBytes(50))
	}
}

func BenchmarkFileIO_Append(b *testing.B) {
	benchmarkFileIO(b, os.O_CREATE|os.O_WRONLY|os.O_APPEND)
}

func BenchmarkFileIO_Normal(b *testing.B) {
	benchmarkFileIO(b, os.O_CREATE|os.O_WRONLY)
}

func BenchmarkFileIO_Sync(b *testing.B) {
	benchmarkFileIO(b, os.O_CREATE|os.O_WRONLY|os.O_SYNC)
}

func BenchmarkFileIO_DSync(b *testing.B) {
	benchmarkFileIO(b, os.O_CREATE|os.O_WRONLY|syscall.O_DSYNC)
}

func BenchmarkFileIO_AppendSync(b *testing.B) {
	benchmarkFileIO(b, os.O_CREATE|os.O_WRONLY|os.O_APPEND|os.O_SYNC)
}

func BenchmarkFileIO_AppendDSync(b *testing.B) {
	benchmarkFileIO(b, os.O_CREATE|os.O_WRONLY|os.O_APPEND|syscall.O_DSYNC)
}

// Internal helper functions
func setupTest(t *testing.T) (string, *Consumer) {
	dir, err := ioutil.TempDir("", "test")
	if err != nil {
		t.Fatal(err)
		return "", nil
	}

	c := &Consumer{
		Config: &Config{
			DirName:                  dir,
			ActiveFileName:           "out.log",
			RotationMaxLines:         40,
			RotationMaxBytes:         1000000,
			FlushingTimeIntervalSecs: 5,
			FileRenamePolicy:         "timestamp",
			MaxAge:                   int64(1 * 60 * 60),
			MaxCount:                 500,
			Target:                   "file",
		},
		LineProcessor: &NoProcessor{},
	}
	return dir, c
}

func readTestDir(t *testing.T, dir string) []os.FileInfo {
	files, err := ioutil.ReadDir(dir)
	if err != nil {
		t.Fatal(err)
		return nil
	}
	return files
}

const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"

func randStringBytes(n int) []byte {
	b := make([]byte, n)
	for i := 0; i < n-1; i++ {
		b[i] = letterBytes[rand.Intn(len(letterBytes))]
	}
	b[n-1] = '\n'
	return b
}
<|endoftext|>


<commit_before>package cmd

var (
	defaultEnv *Environment
)

func init() {
	defaultEnv = NewEnvironment()
}
<commit_msg>Implement the default (global) environment.<commit_after>package cmd

import "context"

var (
	defaultEnv *Environment
)

func init() {
	defaultEnv = NewEnvironment()
}

// Context returns the base context of the global environment.
func Context() context.Context {
	return defaultEnv.Context()
}

// Stop cancels the base context of the global environment.
//
// Passed err will be returned by Wait().
// Once stopped, Go() will not start new goroutines.
//
// This returns true if the caller is the first that calls Stop.
// For second and later calls, Stop does nothing and returns false.
func Stop(err error) bool {
	return defaultEnv.Stop(err)
}

// Wait waits for Stop being called.
//
// The returned err is the one passed to Stop.
// err can be tested by IsSignaled to determine whether the
// program got SIGINT or SIGTERM.
func Wait() error {
	return defaultEnv.Wait()
}

// Go starts a goroutine that executes f in the global environment.
//
// Goroutines started by this function will be waited for by
// Wait until all such goroutines return.
//
// If f returns non-nil error, Stop is called with that error.
//
// f should watch ctx.Done() channel and return quickly when the
// channel is canceled.
func Go(f func(ctx context.Context) error) {
	defaultEnv.Go(f)
}
<|endoftext|>


<commit_before>package oak

import (
	"image"
	"time"

	"github.com/oakmound/oak/v3/alg/intgeom"
	"github.com/oakmound/oak/v3/event"
	"github.com/oakmound/oak/v3/render"
	"github.com/oakmound/oak/v3/scene"
)

var defaultController = NewController()

func Init(scene string, configOptions ...ConfigOption) error {
	defaultController.DrawStack = render.GlobalDrawStack
	defaultController.logicHandler = event.DefaultBus
	return defaultController.Init(scene, configOptions...)
}

func AddScene(name string, sc scene.Scene) error {
	return defaultController.AddScene(name, sc)
}

func IsDown(key string) bool {
	return defaultController.IsDown(key)
}

func IsHeld(key string) (bool, time.Duration) {
	return defaultController.IsHeld(key)
}

func SetUp(key string) {
	defaultController.SetUp(key)
}

func SetDown(key string) {
	defaultController.SetDown(key)
}

func SetViewportBounds(rect intgeom.Rect2) {
	defaultController.SetViewportBounds(rect)
}

func ShiftScreen(x, y int) {
	defaultController.ShiftScreen(x, y)
}

func SetScreen(x, y int) {
	defaultController.SetScreen(x, y)
}

func SetFullScreen(fs bool) error {
	return defaultController.SetFullScreen(fs)
}

func SetBorderless(bs bool) error {
	return defaultController.SetBorderless(bs)
}

func ScreenShot() *image.RGBA {
	return defaultController.ScreenShot()
}

func SetLoadingRenderable(r render.Renderable) {
	defaultController.SetLoadingRenderable(r)
}

func SetBackground(b Background) {
	defaultController.SetBackground(b)
}

func SetColorBackground(img image.Image) {
	defaultController.SetColorBackground(img)
}

func GetBackgroundImage() image.Image {
	return defaultController.GetBackgroundImage()
}

func Width() int {
	return defaultController.Width()
}

func Height() int {
	return defaultController.Height()
}

func HideCursor() error {
	return defaultController.HideCursor()
}
<commit_msg>oak: lazily initialize the default controller<commit_after>package oak

import (
	"image"
	"sync"
	"time"

	"github.com/oakmound/oak/v3/alg/intgeom"
	"github.com/oakmound/oak/v3/event"
	"github.com/oakmound/oak/v3/render"
	"github.com/oakmound/oak/v3/scene"
)

var defaultController *Controller

var initDefaultControllerOnce sync.Once

func initDefaultController() {
	initDefaultControllerOnce.Do(func() {
		defaultController = NewController()
	})
}

func Init(scene string, configOptions ...ConfigOption) error {
	initDefaultController()
	defaultController.DrawStack = render.GlobalDrawStack
	defaultController.logicHandler = event.DefaultBus
	return defaultController.Init(scene, configOptions...)
}

func AddScene(name string, sc scene.Scene) error {
	initDefaultController()
	return defaultController.AddScene(name, sc)
}

func IsDown(key string) bool {
	initDefaultController()
	return defaultController.IsDown(key)
}

func IsHeld(key string) (bool, time.Duration) {
	initDefaultController()
	return defaultController.IsHeld(key)
}

func SetUp(key string) {
	initDefaultController()
	defaultController.SetUp(key)
}

func SetDown(key string) {
	initDefaultController()
	defaultController.SetDown(key)
}

func SetViewportBounds(rect intgeom.Rect2) {
	initDefaultController()
	defaultController.SetViewportBounds(rect)
}

func ShiftScreen(x, y int) {
	initDefaultController()
	defaultController.ShiftScreen(x, y)
}

func SetScreen(x, y int) {
	initDefaultController()
	defaultController.SetScreen(x, y)
}

func SetFullScreen(fs bool) error {
	initDefaultController()
	return defaultController.SetFullScreen(fs)
}

func SetBorderless(bs bool) error {
	initDefaultController()
	return defaultController.SetBorderless(bs)
}

func ScreenShot() *image.RGBA {
	initDefaultController()
	return defaultController.ScreenShot()
}

func SetLoadingRenderable(r render.Renderable) {
	initDefaultController()
	defaultController.SetLoadingRenderable(r)
}

func SetBackground(b Background) {
	initDefaultController()
	defaultController.SetBackground(b)
}

func SetColorBackground(img image.Image) {
	initDefaultController()
	defaultController.SetColorBackground(img)
}

func GetBackgroundImage() image.Image {
	initDefaultController()
	return defaultController.GetBackgroundImage()
}

func Width() int {
	initDefaultController()
	return defaultController.Width()
}

func Height() int {
	initDefaultController()
	return defaultController.Height()
}

func HideCursor() error {
	initDefaultController()
	return defaultController.HideCursor()
}
<|endoftext|>