<commit_before>package mobi

import (
	"crypto/sha1"
	"fmt"
	"image"
	"io"
	"os"
	"path/filepath"

	"github.com/geek1011/BookBrowser/booklist"
	"github.com/geek1011/BookBrowser/formats"
	"github.com/pkg/errors"

	mobirdr "github.com/sblinch/mobi"
)

type mobi struct {
	book *booklist.Book
	coverstart int64
	coverlength int64
}

func (e *mobi) Book() *booklist.Book {
	return e.book
}

func (e *mobi) HasCover() bool {
	return e.coverstart > 0
}

func (e *mobi) GetCover() (i image.Image, err error) {
	if !e.HasCover() {
		return nil, errors.New("no cover")
	}

	f, err := os.Open(e.book.FilePath)
	if err != nil {
		return nil, errors.Wrap(err, "unable to open book file")
	}
	defer f.Close()

	if _, err := f.Seek(e.coverstart, 0); err != nil {
		return nil, errors.Wrap(err, "unable to see to cover offset")
	}

	ltd := io.LimitReader(f,e.coverlength)
	if i, _, err = image.Decode(ltd); err != nil {
		return nil, errors.Wrap(err, "unable to decode book cover")
	}

	return i, nil
}

func load(filename string) (bi formats.BookInfo, ferr error) {
	defer func() {
		if r := recover(); r != nil {
			bi = nil
			ferr = fmt.Errorf("unknown error: %s", r)
		}
	}()

	m := &mobi{book: &booklist.Book{}}

	f, err := os.Open(filename)
	if err != nil {
		return nil, err
	}

	fi, err := f.Stat()
	if err != nil {
		f.Close()
		return nil, errors.Wrapf(err, "could not stat book")
	}
	m.book.FilePath = filename
	m.book.FileSize = fi.Size()
	m.book.ModTime = fi.ModTime()

	s := sha1.New()
	i, err := io.Copy(s, f)
	if err == nil && i != fi.Size() {
		err = errors.New("could not read whole file")
	}
	if err != nil {
		f.Close()
		return nil, errors.Wrap(err, "could not hash book")
	}
	m.book.Hash = fmt.Sprintf("%x", s.Sum(nil))
	f.Close()

	r, err := mobirdr.NewReader(filename)
	if err != nil {
		return nil, err
	}
	defer r.Close()

	m.coverstart, m.coverlength = r.CoverOffsetLength()

	m.book.Title = r.BestTitle()

	authors := r.Authors()
	if len(authors)>0 {
		m.book.Author = authors[0]
	}

	m.book.Description = r.Description()
	m.book.Publisher = r.Publisher()

	/* // uncomment and import github.com/moraes/isbn after merging ISBN/publishing date pull request :)
	isbnStr := r.Isbn()
	if len(isbnStr)>0 && isbn.Validate(isbnStr) {
		m.book.ISBN = isbnStr
	}

	m.book.PublishDate = parsePublishDate(r.PublishingDate())
	*/

	if len(m.book.Title)==0 {
		m.book.Title = filepath.Base(filename)
	}

	return m, nil
}

func init() {
	formats.Register("mobi", load)
	formats.Register("azw", load)
	formats.Register("azw3", load)
}

/* // uncomment after merging ISBN/publishing date pull request :)
func parsePublishDate(s string) time.Time {
	// handle the various dumb decisions people make when encoding dates
	format := ""
	switch len(s) {
	case 32:
		//2012-02-13T20:20:58.175203+00:00
		format = "2006-01-02T15:04:05.000000-07:00"
	case 25:
		//2000-10-31 00:00:00-06:00
		//2009-04-19T22:00:00+00:00
		format = "2006-01-02" + string(s[10]) + "15:04:05-07:00"
	case 20:
		//2016-08-11T14:09:25Z
		format = "2006-01-02T15:04:05Z"
	case 19:
		//2008-01-28T07:00:00
		//2000-10-31 00:00:00
		format = "2006-01-02" + string(s[10]) + "15:04:05"
	case 10:
		//1998-07-01
		format = "2006-01-02"
	default:
		return time.Time{}
	}

	t, err := time.Parse(format,s)
	if err != nil {
		t = time.Time{}
	}
	return t
}
*/<commit_msg>Uncommented ISBN/date code<commit_after>package mobi

import (
	"crypto/sha1"
	"fmt"
	"image"
	"io"
	"os"
	"path/filepath"
	"time"

	"github.com/geek1011/BookBrowser/booklist"
	"github.com/geek1011/BookBrowser/formats"
	"github.com/moraes/isbn"
	"github.com/pkg/errors"

	mobirdr "github.com/sblinch/mobi"
)

type mobi struct {
	book *booklist.Book
	coverstart int64
	coverlength int64
}

func (e *mobi) Book() *booklist.Book {
	return e.book
}

func (e *mobi) HasCover() bool {
	return e.coverstart > 0
}

func (e *mobi) GetCover() (i image.Image, err error) {
	if !e.HasCover() {
		return nil, errors.New("no cover")
	}

	f, err := os.Open(e.book.FilePath)
	if err != nil {
		return nil, errors.Wrap(err, "unable to open book file")
	}
	defer f.Close()

	if _, err := f.Seek(e.coverstart, 0); err != nil {
		return nil, errors.Wrap(err, "unable to see to cover offset")
	}

	ltd := io.LimitReader(f,e.coverlength)
	if i, _, err = image.Decode(ltd); err != nil {
		return nil, errors.Wrap(err, "unable to decode book cover")
	}

	return i, nil
}

func load(filename string) (bi formats.BookInfo, ferr error) {
	defer func() {
		if r := recover(); r != nil {
			bi = nil
			ferr = fmt.Errorf("unknown error: %s", r)
		}
	}()

	m := &mobi{book: &booklist.Book{}}

	f, err := os.Open(filename)
	if err != nil {
		return nil, err
	}

	fi, err := f.Stat()
	if err != nil {
		f.Close()
		return nil, errors.Wrapf(err, "could not stat book")
	}
	m.book.FilePath = filename
	m.book.FileSize = fi.Size()
	m.book.ModTime = fi.ModTime()

	s := sha1.New()
	i, err := io.Copy(s, f)
	if err == nil && i != fi.Size() {
		err = errors.New("could not read whole file")
	}
	if err != nil {
		f.Close()
		return nil, errors.Wrap(err, "could not hash book")
	}
	m.book.Hash = fmt.Sprintf("%x", s.Sum(nil))
	f.Close()

	r, err := mobirdr.NewReader(filename)
	if err != nil {
		return nil, err
	}
	defer r.Close()

	m.coverstart, m.coverlength = r.CoverOffsetLength()

	m.book.Title = r.BestTitle()

	authors := r.Authors()
	if len(authors)>0 {
		m.book.Author = authors[0]
	}

	m.book.Description = r.Description()
	m.book.Publisher = r.Publisher()

	isbnStr := r.Isbn()
	if len(isbnStr)>0 && isbn.Validate(isbnStr) {
		m.book.ISBN = isbnStr
	}

	m.book.PublishDate = parsePublishDate(r.PublishingDate())

	if len(m.book.Title)==0 {
		m.book.Title = filepath.Base(filename)
	}

	return m, nil
}

func init() {
	formats.Register("mobi", load)
	formats.Register("azw", load)
	formats.Register("azw3", load)
}

func parsePublishDate(s string) time.Time {
	// handle the various dumb decisions people make when encoding dates
	format := ""
	switch len(s) {
	case 32:
		//2012-02-13T20:20:58.175203+00:00
		format = "2006-01-02T15:04:05.000000-07:00"
	case 25:
		//2000-10-31 00:00:00-06:00
		//2009-04-19T22:00:00+00:00
		format = "2006-01-02" + string(s[10]) + "15:04:05-07:00"
	case 20:
		//2016-08-11T14:09:25Z
		format = "2006-01-02T15:04:05Z"
	case 19:
		//2008-01-28T07:00:00
		//2000-10-31 00:00:00
		format = "2006-01-02" + string(s[10]) + "15:04:05"
	case 10:
		//1998-07-01
		format = "2006-01-02"
	default:
		return time.Time{}
	}

	t, err := time.Parse(format,s)
	if err != nil {
		t = time.Time{}
	}
	return t
}
<|endoftext|>


<commit_before>package types

type Type int

const (
	String Type = iota + 1
	Int
	Ident
	StringList
)
<commit_msg>Implement fmt.Stringer for Type<commit_after>package types

type Type int

const (
	String Type = iota + 1
	Int
	Ident
	StringList
)

func (t Type) String() string {
	switch t {
	case String:
		return "String"
	case Int:
		return "Int"
	case Ident:
		return "Ident"
	case StringList:
		return "List String"
	}
	panic("unreachable")
}
<|endoftext|>


<commit_before>package index

import (
	"compress/gzip"
	"encoding/gob"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"sync"
	"time"
	"unicode/utf8"

	"github.com/etsy/hound/codesearch/index"
	"github.com/etsy/hound/codesearch/regexp"
)

const (
	matchLimit               = 5000
	manifestFilename         = "metadata.gob"
	excludedFileJsonFilename = "excluded_files.json"
	filePeekSize             = 2048
)

const (
	reasonDotFile     = "Dot files are excluded."
	reasonInvalidMode = "Invalid file mode."
	reasonNotText     = "Not a text file."
)

type Index struct {
	Ref *IndexRef
	idx *index.Index
	lck sync.RWMutex
}

type IndexOptions struct {
	ExcludeDotFiles bool
	SpecialFiles    []string
}

type SearchOptions struct {
	IgnoreCase     bool
	LinesOfContext uint
	FileRegexp     string
	Offset         int
	Limit          int
}

type Match struct {
	Line       string
	LineNumber int
	Before     []string
	After      []string
}

type SearchResponse struct {
	Matches        []*FileMatch
	FilesWithMatch int
	FilesOpened    int           `json:"-"`
	Duration       time.Duration `json:"-"`
	Revision       string
}

type FileMatch struct {
	Filename string
	Matches  []*Match
}

type ExcludedFile struct {
	Filename string
	Reason   string
}

type IndexRef struct {
	Url  string
	Rev  string
	Time time.Time
	dir  string
}

func (r *IndexRef) Dir() string {
	return r.dir
}

func (r *IndexRef) writeManifest() error {
	w, err := os.Create(filepath.Join(r.dir, manifestFilename))
	if err != nil {
		return err
	}
	defer w.Close()

	return gob.NewEncoder(w).Encode(r)
}

func (r *IndexRef) Open() (*Index, error) {
	return &Index{
		Ref: r,
		idx: index.Open(filepath.Join(r.dir, "tri")),
	}, nil
}

func (r *IndexRef) Remove() error {
	return os.RemoveAll(r.dir)
}

func (n *Index) Close() error {
	n.lck.Lock()
	defer n.lck.Unlock()
	return n.idx.Close()
}

func (n *Index) Destroy() error {
	n.lck.Lock()
	defer n.lck.Unlock()
	if err := n.idx.Close(); err != nil {
		return err
	}
	return n.Ref.Remove()
}

func (n *Index) GetDir() string {
	return n.Ref.dir
}

func toStrings(lines [][]byte) []string {
	strs := make([]string, len(lines))
	for i, n := 0, len(lines); i < n; i++ {
		strs[i] = string(lines[i])
	}
	return strs
}

func GetRegexpPattern(pat string, ignoreCase bool) string {
	if ignoreCase {
		return "(?i)(?m)" + pat
	}
	return "(?m)" + pat
}

func (n *Index) Search(pat string, opt *SearchOptions) (*SearchResponse, error) {
	startedAt := time.Now()

	n.lck.RLock()
	defer n.lck.RUnlock()

	re, err := regexp.Compile(GetRegexpPattern(pat, opt.IgnoreCase))
	if err != nil {
		return nil, err
	}

	var (
		g                grepper
		results          []*FileMatch
		filesOpened      int
		filesFound       int
		filesCollected   int
		matchesCollected int
	)

	var fre *regexp.Regexp
	if opt.FileRegexp != "" {
		fre, err = regexp.Compile(opt.FileRegexp)
		if err != nil {
			return nil, err
		}
	}

	files := n.idx.PostingQuery(index.RegexpQuery(re.Syntax))
	for _, file := range files {
		var matches []*Match
		name := n.idx.Name(file)
		hasMatch := false

		// reject files that do not match the file pattern
		if fre != nil && fre.MatchString(name, true, true) < 0 {
			continue
		}

		filesOpened++
		if err := g.grep2File(filepath.Join(n.Ref.dir, "raw", name), re, int(opt.LinesOfContext),
			func(line []byte, lineno int, before [][]byte, after [][]byte) (bool, error) {

				hasMatch = true
				if filesFound < opt.Offset || (opt.Limit > 0 && filesCollected >= opt.Limit) {
					return false, nil
				}

				matchesCollected++
				matches = append(matches, &Match{
					Line:       string(line),
					LineNumber: lineno,
					Before:     toStrings(before),
					After:      toStrings(after),
				})

				if matchesCollected > matchLimit {
					return false, fmt.Errorf("search exceeds limit on matches: %d", matchLimit)
				}

				return true, nil
			}); err != nil {
			return nil, err
		}

		if !hasMatch {
			continue
		}

		filesFound++
		if len(matches) > 0 {
			filesCollected++
			results = append(results, &FileMatch{
				Filename: name,
				Matches:  matches,
			})
		}
	}

	return &SearchResponse{
		Matches:        results,
		FilesWithMatch: filesFound,
		FilesOpened:    filesOpened,
		Duration:       time.Now().Sub(startedAt),
		Revision:       n.Ref.Rev,
	}, nil
}

func isTextFile(filename string) (bool, error) {
	buf := make([]byte, filePeekSize)
	r, err := os.Open(filename)
	if err != nil {
		return false, err
	}
	defer r.Close()

	n, err := io.ReadFull(r, buf)
	if err != nil && err != io.ErrUnexpectedEOF && err != io.EOF {
		return false, err
	}

	buf = buf[:n]

	res, validSize := validUTF8(buf)
	// if remaining non valid less than utf8.UTFMax probably we are read part of rune just assume it's valid
	if n == filePeekSize && n-validSize < utf8.UTFMax {
		res = true
	}
	return res, nil
}

// copy paste from go source https://golang.org/src/unicode/utf8/utf8.go?s=9677:9702#L387, with valid bytes size
func validUTF8(p []byte) (bool, int) {
	i := 0
	for i < len(p) {
		if p[i] < utf8.RuneSelf {
			i++
		} else {
			_, size := utf8.DecodeRune(p[i:])
			if size == 1 {
				// All valid runes of size 1 (those
				// below RuneSelf) were handled above.
				// This must be a RuneError.
				return false, i
			}
			i += size
		}
	}
	return true, i
}

func addFileToIndex(ix *index.IndexWriter, dst, src, path string) (string, error) {
	rel, err := filepath.Rel(src, path)
	if err != nil {
		return "", err
	}

	r, err := os.Open(path)
	if err != nil {
		return "", err
	}
	defer r.Close()

	dup := filepath.Join(dst, "raw", rel)
	w, err := os.Create(dup)
	if err != nil {
		return "", err
	}
	defer w.Close()

	g := gzip.NewWriter(w)
	defer g.Close()

	return ix.Add(rel, io.TeeReader(r, g)), nil
}

func addDirToIndex(dst, src, path string) error {
	rel, err := filepath.Rel(src, path)
	if err != nil {
		return err
	}

	if rel == "." {
		return nil
	}

	dup := filepath.Join(dst, "raw", rel)
	return os.Mkdir(dup, os.ModePerm)
}

// write the list of excluded files to the given filename.
func writeExcludedFilesJson(filename string, files []*ExcludedFile) error {
	w, err := os.Create(filename)
	if err != nil {
		return err
	}
	defer w.Close()

	return json.NewEncoder(w).Encode(files)
}

func containsString(haystack []string, needle string) bool {
	for i, n := 0, len(haystack); i < n; i++ {
		if haystack[i] == needle {
			return true
		}
	}
	return false
}

func indexAllFiles(opt *IndexOptions, dst, src string) error {
	ix := index.Create(filepath.Join(dst, "tri"))
	defer ix.Close()

	excluded := []*ExcludedFile{}

	// Make a file to store the excluded files for this repo
	fileHandle, err := os.Create(filepath.Join(dst, "excluded_files.json"))
	if err != nil {
		return err
	}
	defer fileHandle.Close()

	if err := filepath.Walk(src, func(path string, info os.FileInfo, err error) error {
		name := info.Name()
		rel, err := filepath.Rel(src, path)
		if err != nil {
			return err
		}

		// Is this file considered "special", this means it's not even a part
		// of the source repository (like .git or .svn).
		if containsString(opt.SpecialFiles, name) {
			if info.IsDir() {
				return filepath.SkipDir
			}
			return nil
		}

		if opt.ExcludeDotFiles && name[0] == '.' {
			if info.IsDir() {
				return filepath.SkipDir
			}

			excluded = append(excluded, &ExcludedFile{
				rel,
				reasonDotFile,
			})
			return nil
		}

		if info.IsDir() {
			return addDirToIndex(dst, src, path)
		}

		if info.Mode()&os.ModeType != 0 {
			excluded = append(excluded, &ExcludedFile{
				rel,
				reasonInvalidMode,
			})
			return nil
		}

		txt, err := isTextFile(path)
		if err != nil {
			return err
		}

		if !txt {
			excluded = append(excluded, &ExcludedFile{
				rel,
				reasonNotText,
			})
			return nil
		}

		reasonForExclusion, err := addFileToIndex(ix, dst, src, path)
		if err != nil {
			return err
		}
		if reasonForExclusion != "" {
			excluded = append(excluded, &ExcludedFile{rel, reasonForExclusion})
		}

		return nil
	}); err != nil {
		return err
	}

	if err := writeExcludedFilesJson(
		filepath.Join(dst, excludedFileJsonFilename),
		excluded); err != nil {
		return err
	}

	ix.Flush()

	return nil
}

// Read the metadata for the index directory. Note that even if this
// returns a non-nil error, a Metadata object will be returned with
// all the information that is known about the index (this might
// include only the path)
func Read(dir string) (*IndexRef, error) {
	m := &IndexRef{
		dir: dir,
	}

	r, err := os.Open(filepath.Join(dir, manifestFilename))
	if err != nil {
		return m, err
	}
	defer r.Close()

	if err := gob.NewDecoder(r).Decode(m); err != nil {
		return m, err
	}

	return m, nil
}

func Build(opt *IndexOptions, dst, src, url, rev string) (*IndexRef, error) {
	if _, err := os.Stat(dst); err != nil {
		if err := os.MkdirAll(dst, os.ModePerm); err != nil {
			return nil, err
		}
	}

	if err := os.Mkdir(filepath.Join(dst, "raw"), os.ModePerm); err != nil {
		return nil, err
	}

	if err := indexAllFiles(opt, dst, src); err != nil {
		return nil, err
	}

	r := &IndexRef{
		Url:  url,
		Rev:  rev,
		Time: time.Now(),
		dir:  dst,
	}

	if err := r.writeManifest(); err != nil {
		return nil, err
	}

	return r, nil
}

// Open the index in dir for searching.
func Open(dir string) (*Index, error) {
	r, err := Read(dir)
	if err != nil {
		return nil, err
	}

	return r.Open()
}
<commit_msg>Make utf8 validity check function self-contained.<commit_after>package index

import (
	"compress/gzip"
	"encoding/gob"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"sync"
	"time"
	"unicode/utf8"

	"github.com/etsy/hound/codesearch/index"
	"github.com/etsy/hound/codesearch/regexp"
)

const (
	matchLimit               = 5000
	manifestFilename         = "metadata.gob"
	excludedFileJsonFilename = "excluded_files.json"
	filePeekSize             = 2048
)

const (
	reasonDotFile     = "Dot files are excluded."
	reasonInvalidMode = "Invalid file mode."
	reasonNotText     = "Not a text file."
)

type Index struct {
	Ref *IndexRef
	idx *index.Index
	lck sync.RWMutex
}

type IndexOptions struct {
	ExcludeDotFiles bool
	SpecialFiles    []string
}

type SearchOptions struct {
	IgnoreCase     bool
	LinesOfContext uint
	FileRegexp     string
	Offset         int
	Limit          int
}

type Match struct {
	Line       string
	LineNumber int
	Before     []string
	After      []string
}

type SearchResponse struct {
	Matches        []*FileMatch
	FilesWithMatch int
	FilesOpened    int           `json:"-"`
	Duration       time.Duration `json:"-"`
	Revision       string
}

type FileMatch struct {
	Filename string
	Matches  []*Match
}

type ExcludedFile struct {
	Filename string
	Reason   string
}

type IndexRef struct {
	Url  string
	Rev  string
	Time time.Time
	dir  string
}

func (r *IndexRef) Dir() string {
	return r.dir
}

func (r *IndexRef) writeManifest() error {
	w, err := os.Create(filepath.Join(r.dir, manifestFilename))
	if err != nil {
		return err
	}
	defer w.Close()

	return gob.NewEncoder(w).Encode(r)
}

func (r *IndexRef) Open() (*Index, error) {
	return &Index{
		Ref: r,
		idx: index.Open(filepath.Join(r.dir, "tri")),
	}, nil
}

func (r *IndexRef) Remove() error {
	return os.RemoveAll(r.dir)
}

func (n *Index) Close() error {
	n.lck.Lock()
	defer n.lck.Unlock()
	return n.idx.Close()
}

func (n *Index) Destroy() error {
	n.lck.Lock()
	defer n.lck.Unlock()
	if err := n.idx.Close(); err != nil {
		return err
	}
	return n.Ref.Remove()
}

func (n *Index) GetDir() string {
	return n.Ref.dir
}

func toStrings(lines [][]byte) []string {
	strs := make([]string, len(lines))
	for i, n := 0, len(lines); i < n; i++ {
		strs[i] = string(lines[i])
	}
	return strs
}

func GetRegexpPattern(pat string, ignoreCase bool) string {
	if ignoreCase {
		return "(?i)(?m)" + pat
	}
	return "(?m)" + pat
}

func (n *Index) Search(pat string, opt *SearchOptions) (*SearchResponse, error) {
	startedAt := time.Now()

	n.lck.RLock()
	defer n.lck.RUnlock()

	re, err := regexp.Compile(GetRegexpPattern(pat, opt.IgnoreCase))
	if err != nil {
		return nil, err
	}

	var (
		g                grepper
		results          []*FileMatch
		filesOpened      int
		filesFound       int
		filesCollected   int
		matchesCollected int
	)

	var fre *regexp.Regexp
	if opt.FileRegexp != "" {
		fre, err = regexp.Compile(opt.FileRegexp)
		if err != nil {
			return nil, err
		}
	}

	files := n.idx.PostingQuery(index.RegexpQuery(re.Syntax))
	for _, file := range files {
		var matches []*Match
		name := n.idx.Name(file)
		hasMatch := false

		// reject files that do not match the file pattern
		if fre != nil && fre.MatchString(name, true, true) < 0 {
			continue
		}

		filesOpened++
		if err := g.grep2File(filepath.Join(n.Ref.dir, "raw", name), re, int(opt.LinesOfContext),
			func(line []byte, lineno int, before [][]byte, after [][]byte) (bool, error) {

				hasMatch = true
				if filesFound < opt.Offset || (opt.Limit > 0 && filesCollected >= opt.Limit) {
					return false, nil
				}

				matchesCollected++
				matches = append(matches, &Match{
					Line:       string(line),
					LineNumber: lineno,
					Before:     toStrings(before),
					After:      toStrings(after),
				})

				if matchesCollected > matchLimit {
					return false, fmt.Errorf("search exceeds limit on matches: %d", matchLimit)
				}

				return true, nil
			}); err != nil {
			return nil, err
		}

		if !hasMatch {
			continue
		}

		filesFound++
		if len(matches) > 0 {
			filesCollected++
			results = append(results, &FileMatch{
				Filename: name,
				Matches:  matches,
			})
		}
	}

	return &SearchResponse{
		Matches:        results,
		FilesWithMatch: filesFound,
		FilesOpened:    filesOpened,
		Duration:       time.Now().Sub(startedAt),
		Revision:       n.Ref.Rev,
	}, nil
}

func isTextFile(filename string) (bool, error) {
	buf := make([]byte, filePeekSize)
	r, err := os.Open(filename)
	if err != nil {
		return false, err
	}
	defer r.Close()

	n, err := io.ReadFull(r, buf)
	if err != nil && err != io.ErrUnexpectedEOF && err != io.EOF {
		return false, err
	}

	buf = buf[:n]

	if n < filePeekSize {
		// read the whole file, must be valid.
		return utf8.Valid(buf), nil
	}

	// read a prefix, allow trailing partial runes.
	return validUTF8IgnoringPartialTrailingRune(buf), nil

}

// Determines if the buffer contains valid UTF8 encoded string data. The buffer is assumed
// to be a prefix of a larger buffer so if the buffer ends with the start of a rune, it
// is still considered valid.
//
// Basic logic copied from https://golang.org/pkg/unicode/utf8/#Valid
func validUTF8IgnoringPartialTrailingRune(p []byte) bool {
	i := 0
	n := len(p)

	for i < n {
		if p[i] < utf8.RuneSelf {
			i++
		} else {
			_, size := utf8.DecodeRune(p[i:])
			if size == 1 {
				// All valid runes of size 1 (those below RuneSelf) were handled above. This must be a RuneError.
				// If we're encountering this error within UTFMax of the end and the current byte could be a
				// valid start, we'll just ignore the assumed partial rune.
				return n-i < utf8.UTFMax && utf8.RuneStart(p[i])
			}
			i += size
		}
	}
	return true
}

func addFileToIndex(ix *index.IndexWriter, dst, src, path string) (string, error) {
	rel, err := filepath.Rel(src, path)
	if err != nil {
		return "", err
	}

	r, err := os.Open(path)
	if err != nil {
		return "", err
	}
	defer r.Close()

	dup := filepath.Join(dst, "raw", rel)
	w, err := os.Create(dup)
	if err != nil {
		return "", err
	}
	defer w.Close()

	g := gzip.NewWriter(w)
	defer g.Close()

	return ix.Add(rel, io.TeeReader(r, g)), nil
}

func addDirToIndex(dst, src, path string) error {
	rel, err := filepath.Rel(src, path)
	if err != nil {
		return err
	}

	if rel == "." {
		return nil
	}

	dup := filepath.Join(dst, "raw", rel)
	return os.Mkdir(dup, os.ModePerm)
}

// write the list of excluded files to the given filename.
func writeExcludedFilesJson(filename string, files []*ExcludedFile) error {
	w, err := os.Create(filename)
	if err != nil {
		return err
	}
	defer w.Close()

	return json.NewEncoder(w).Encode(files)
}

func containsString(haystack []string, needle string) bool {
	for i, n := 0, len(haystack); i < n; i++ {
		if haystack[i] == needle {
			return true
		}
	}
	return false
}

func indexAllFiles(opt *IndexOptions, dst, src string) error {
	ix := index.Create(filepath.Join(dst, "tri"))
	defer ix.Close()

	excluded := []*ExcludedFile{}

	// Make a file to store the excluded files for this repo
	fileHandle, err := os.Create(filepath.Join(dst, "excluded_files.json"))
	if err != nil {
		return err
	}
	defer fileHandle.Close()

	if err := filepath.Walk(src, func(path string, info os.FileInfo, err error) error {
		name := info.Name()
		rel, err := filepath.Rel(src, path)
		if err != nil {
			return err
		}

		// Is this file considered "special", this means it's not even a part
		// of the source repository (like .git or .svn).
		if containsString(opt.SpecialFiles, name) {
			if info.IsDir() {
				return filepath.SkipDir
			}
			return nil
		}

		if opt.ExcludeDotFiles && name[0] == '.' {
			if info.IsDir() {
				return filepath.SkipDir
			}

			excluded = append(excluded, &ExcludedFile{
				rel,
				reasonDotFile,
			})
			return nil
		}

		if info.IsDir() {
			return addDirToIndex(dst, src, path)
		}

		if info.Mode()&os.ModeType != 0 {
			excluded = append(excluded, &ExcludedFile{
				rel,
				reasonInvalidMode,
			})
			return nil
		}

		txt, err := isTextFile(path)
		if err != nil {
			return err
		}

		if !txt {
			excluded = append(excluded, &ExcludedFile{
				rel,
				reasonNotText,
			})
			return nil
		}

		reasonForExclusion, err := addFileToIndex(ix, dst, src, path)
		if err != nil {
			return err
		}
		if reasonForExclusion != "" {
			excluded = append(excluded, &ExcludedFile{rel, reasonForExclusion})
		}

		return nil
	}); err != nil {
		return err
	}

	if err := writeExcludedFilesJson(
		filepath.Join(dst, excludedFileJsonFilename),
		excluded); err != nil {
		return err
	}

	ix.Flush()

	return nil
}

// Read the metadata for the index directory. Note that even if this
// returns a non-nil error, a Metadata object will be returned with
// all the information that is known about the index (this might
// include only the path)
func Read(dir string) (*IndexRef, error) {
	m := &IndexRef{
		dir: dir,
	}

	r, err := os.Open(filepath.Join(dir, manifestFilename))
	if err != nil {
		return m, err
	}
	defer r.Close()

	if err := gob.NewDecoder(r).Decode(m); err != nil {
		return m, err
	}

	return m, nil
}

func Build(opt *IndexOptions, dst, src, url, rev string) (*IndexRef, error) {
	if _, err := os.Stat(dst); err != nil {
		if err := os.MkdirAll(dst, os.ModePerm); err != nil {
			return nil, err
		}
	}

	if err := os.Mkdir(filepath.Join(dst, "raw"), os.ModePerm); err != nil {
		return nil, err
	}

	if err := indexAllFiles(opt, dst, src); err != nil {
		return nil, err
	}

	r := &IndexRef{
		Url:  url,
		Rev:  rev,
		Time: time.Now(),
		dir:  dst,
	}

	if err := r.writeManifest(); err != nil {
		return nil, err
	}

	return r, nil
}

// Open the index in dir for searching.
func Open(dir string) (*Index, error) {
	r, err := Read(dir)
	if err != nil {
		return nil, err
	}

	return r.Open()
}
<|endoftext|>


<commit_before>/*
Package inflection pluralizes and singularizes English nouns.

		inflection.Plural("person") => "people"
		inflection.Plural("Person") => "People"
		inflection.Plural("PERSON") => "PEOPLE"

		inflection.Singular("people") => "person"
		inflection.Singular("People") => "Person"
		inflection.Singular("PEOPLE") => "PERSON"

		inflection.Plural("FancyPerson") => "FancydPeople"
		inflection.Singular("FancyPeople") => "FancydPerson"

Standard rules are from Rails's ActiveSupport (https://github.com/rails/rails/blob/master/activesupport/lib/active_support/inflections.rb)

If you want to register more rules, follow:

		inflection.AddUncountable("fish")
		inflection.AddIrregular("person", "people")
		inflection.AddPlural("(bu)s$", "${1}ses") # "bus" => "buses" / "BUS" => "BUSES" / "Bus" => "Buses"
		inflection.AddSingular("(bus)(es)?$", "${1}") # "buses" => "bus" / "Buses" => "Bus" / "BUSES" => "BUS"
*/
package inflection

import (
	"regexp"
	"strings"
)

type inflection struct {
	regexp  *regexp.Regexp
	replace string
}

// Regular is a regexp find replace inflection
type Regular struct {
	find    string
	replace string
}

// Irregular is a hard replace inflection,
// containing both singular and plural forms
type Irregular struct {
	singular string
	plural   string
}

// RegularSlice is a slice of Regular inflections
type RegularSlice []Regular

// IrregularSlice is a slice of Irregular inflections
type IrregularSlice []Irregular

var pluralInflections = RegularSlice{
	{"([a-z])$", "${1}s"},
	{"s$", "s"},
	{"^(ax|test)is$", "${1}es"},
	{"(octop|vir)us$", "${1}i"},
	{"(octop|vir)i$", "${1}i"},
	{"(alias|status)$", "${1}es"},
	{"(bu)s$", "${1}ses"},
	{"(buffal|tomat)o$", "${1}oes"},
	{"([ti])um$", "${1}a"},
	{"([ti])a$", "${1}a"},
	{"sis$", "ses"},
	{"(?:([^f])fe|([lr])f)$", "${1}${2}ves"},
	{"(hive)$", "${1}s"},
	{"([^aeiouy]|qu)y$", "${1}ies"},
	{"(x|ch|ss|sh)$", "${1}es"},
	{"(matr|vert|ind)(?:ix|ex)$", "${1}ices"},
	{"^(m|l)ouse$", "${1}ice"},
	{"^(m|l)ice$", "${1}ice"},
	{"^(ox)$", "${1}en"},
	{"^(oxen)$", "${1}"},
	{"(quiz)$", "${1}zes"},
}

var singularInflections = RegularSlice{
	{"s$", ""},
	{"(ss)$", "${1}"},
	{"(n)ews$", "${1}ews"},
	{"([ti])a$", "${1}um"},
	{"((a)naly|(b)a|(d)iagno|(p)arenthe|(p)rogno|(s)ynop|(t)he)(sis|ses)$", "${1}sis"},
	{"(^analy)(sis|ses)$", "${1}sis"},
	{"([^f])ves$", "${1}fe"},
	{"(hive)s$", "${1}"},
	{"(tive)s$", "${1}"},
	{"([lr])ves$", "${1}f"},
	{"([^aeiouy]|qu)ies$", "${1}y"},
	{"(s)eries$", "${1}eries"},
	{"(m)ovies$", "${1}ovie"},
	{"(c)ookies$", "${1}ookie"},
	{"(x|ch|ss|sh)es$", "${1}"},
	{"^(m|l)ice$", "${1}ouse"},
	{"(bus)(es)?$", "${1}"},
	{"(o)es$", "${1}"},
	{"(shoe)s$", "${1}"},
	{"(cris|test)(is|es)$", "${1}is"},
	{"^(a)x[ie]s$", "${1}xis"},
	{"(octop|vir)(us|i)$", "${1}us"},
	{"(alias|status)(es)?$", "${1}"},
	{"^(ox)en", "${1}"},
	{"(vert|ind)ices$", "${1}ex"},
	{"(matr)ices$", "${1}ix"},
	{"(quiz)zes$", "${1}"},
	{"(database)s$", "${1}"},
}

var irregularInflections = IrregularSlice{
	{"person", "people"},
	{"man", "men"},
	{"child", "children"},
	{"sex", "sexes"},
	{"move", "moves"},
	{"mombie", "mombies"},
}

var uncountableInflections = []string{"equipment", "information", "rice", "money", "species", "series", "fish", "sheep", "jeans", "police", "evidence"}

var compiledPluralMaps []inflection
var compiledSingularMaps []inflection

func compile() {
	compiledPluralMaps = []inflection{}
	compiledSingularMaps = []inflection{}
	for _, uncountable := range uncountableInflections {
		inf := inflection{
			regexp:  regexp.MustCompile("^(?i)(" + uncountable + ")$"),
			replace: "${1}",
		}
		compiledPluralMaps = append(compiledPluralMaps, inf)
		compiledSingularMaps = append(compiledSingularMaps, inf)
	}

	for _, value := range irregularInflections {
		infs := []inflection{
			inflection{regexp: regexp.MustCompile(strings.ToUpper(value.singular) + "$"), replace: strings.ToUpper(value.plural)},
			inflection{regexp: regexp.MustCompile(strings.Title(value.singular) + "$"), replace: strings.Title(value.plural)},
			inflection{regexp: regexp.MustCompile(value.singular + "$"), replace: value.plural},
		}
		compiledPluralMaps = append(compiledPluralMaps, infs...)
	}

	for _, value := range irregularInflections {
		infs := []inflection{
			inflection{regexp: regexp.MustCompile(strings.ToUpper(value.plural) + "$"), replace: strings.ToUpper(value.singular)},
			inflection{regexp: regexp.MustCompile(strings.Title(value.plural) + "$"), replace: strings.Title(value.singular)},
			inflection{regexp: regexp.MustCompile(value.plural + "$"), replace: value.singular},
		}
		compiledSingularMaps = append(compiledSingularMaps, infs...)
	}

	for i := len(pluralInflections) - 1; i >= 0; i-- {
		value := pluralInflections[i]
		infs := []inflection{
			inflection{regexp: regexp.MustCompile(strings.ToUpper(value.find)), replace: strings.ToUpper(value.replace)},
			inflection{regexp: regexp.MustCompile(value.find), replace: value.replace},
			inflection{regexp: regexp.MustCompile("(?i)" + value.find), replace: value.replace},
		}
		compiledPluralMaps = append(compiledPluralMaps, infs...)
	}

	for i := len(singularInflections) - 1; i >= 0; i-- {
		value := singularInflections[i]
		infs := []inflection{
			inflection{regexp: regexp.MustCompile(strings.ToUpper(value.find)), replace: strings.ToUpper(value.replace)},
			inflection{regexp: regexp.MustCompile(value.find), replace: value.replace},
			inflection{regexp: regexp.MustCompile("(?i)" + value.find), replace: value.replace},
		}
		compiledSingularMaps = append(compiledSingularMaps, infs...)
	}
}

func init() {
	compile()
}

// AddPlural adds a plural inflection
func AddPlural(find, replace string) {
	pluralInflections = append(pluralInflections, Regular{find, replace})
	compile()
}

// AddSingular adds a singular inflection
func AddSingular(find, replace string) {
	singularInflections = append(singularInflections, Regular{find, replace})
	compile()
}

// AddIrregular adds an irregular inflection
func AddIrregular(singular, plural string) {
	irregularInflections = append(irregularInflections, Irregular{singular, plural})
	compile()
}

// AddUncountable adds an uncountable inflection
func AddUncountable(values ...string) {
	uncountableInflections = append(uncountableInflections, values...)
	compile()
}

// GetPlural retrieves the plural inflection values
func GetPlural() RegularSlice {
	plurals := make(RegularSlice, len(pluralInflections))
	copy(plurals, pluralInflections)
	return plurals
}

// GetSingular retrieves the singular inflection values
func GetSingular() RegularSlice {
	singulars := make(RegularSlice, len(singularInflections))
	copy(singulars, singularInflections)
	return singulars
}

// GetIrregular retrieves the irregular inflection values
func GetIrregular() IrregularSlice {
	irregular := make(IrregularSlice, len(irregularInflections))
	copy(irregular, irregularInflections)
	return irregular
}

// GetUncountable retrieves the uncountable inflection values
func GetUncountable() []string {
	uncountables := make([]string, len(uncountableInflections))
	copy(uncountables, uncountableInflections)
	return uncountables
}

// SetPlural sets the plural inflections slice
func SetPlural(inflections RegularSlice) {
	pluralInflections = inflections
	compile()
}

// SetSingular sets the singular inflections slice
func SetSingular(inflections RegularSlice) {
	singularInflections = inflections
	compile()
}

// SetIrregular sets the irregular inflections slice
func SetIrregular(inflections IrregularSlice) {
	irregularInflections = inflections
	compile()
}

// SetUncountable sets the uncountable inflections slice
func SetUncountable(inflections []string) {
	uncountableInflections = inflections
	compile()
}

// Plural converts a word to its plural form
func Plural(str string) string {
	for _, inflection := range compiledPluralMaps {
		if inflection.regexp.MatchString(str) {
			return inflection.regexp.ReplaceAllString(str, inflection.replace)
		}
	}
	return str
}

// Singular converts a word to its singular form
func Singular(str string) string {
	for _, inflection := range compiledSingularMaps {
		if inflection.regexp.MatchString(str) {
			return inflection.regexp.ReplaceAllString(str, inflection.replace)
		}
	}
	return str
}
<commit_msg>Added "foot"/"feet" to irregularInflections<commit_after>/*
Package inflection pluralizes and singularizes English nouns.

		inflection.Plural("person") => "people"
		inflection.Plural("Person") => "People"
		inflection.Plural("PERSON") => "PEOPLE"

		inflection.Singular("people") => "person"
		inflection.Singular("People") => "Person"
		inflection.Singular("PEOPLE") => "PERSON"

		inflection.Plural("FancyPerson") => "FancydPeople"
		inflection.Singular("FancyPeople") => "FancydPerson"

Standard rules are from Rails's ActiveSupport (https://github.com/rails/rails/blob/master/activesupport/lib/active_support/inflections.rb)

If you want to register more rules, follow:

		inflection.AddUncountable("fish")
		inflection.AddIrregular("person", "people")
		inflection.AddPlural("(bu)s$", "${1}ses") # "bus" => "buses" / "BUS" => "BUSES" / "Bus" => "Buses"
		inflection.AddSingular("(bus)(es)?$", "${1}") # "buses" => "bus" / "Buses" => "Bus" / "BUSES" => "BUS"
*/
package inflection

import (
	"regexp"
	"strings"
)

type inflection struct {
	regexp  *regexp.Regexp
	replace string
}

// Regular is a regexp find replace inflection
type Regular struct {
	find    string
	replace string
}

// Irregular is a hard replace inflection,
// containing both singular and plural forms
type Irregular struct {
	singular string
	plural   string
}

// RegularSlice is a slice of Regular inflections
type RegularSlice []Regular

// IrregularSlice is a slice of Irregular inflections
type IrregularSlice []Irregular

var pluralInflections = RegularSlice{
	{"([a-z])$", "${1}s"},
	{"s$", "s"},
	{"^(ax|test)is$", "${1}es"},
	{"(octop|vir)us$", "${1}i"},
	{"(octop|vir)i$", "${1}i"},
	{"(alias|status)$", "${1}es"},
	{"(bu)s$", "${1}ses"},
	{"(buffal|tomat)o$", "${1}oes"},
	{"([ti])um$", "${1}a"},
	{"([ti])a$", "${1}a"},
	{"sis$", "ses"},
	{"(?:([^f])fe|([lr])f)$", "${1}${2}ves"},
	{"(hive)$", "${1}s"},
	{"([^aeiouy]|qu)y$", "${1}ies"},
	{"(x|ch|ss|sh)$", "${1}es"},
	{"(matr|vert|ind)(?:ix|ex)$", "${1}ices"},
	{"^(m|l)ouse$", "${1}ice"},
	{"^(m|l)ice$", "${1}ice"},
	{"^(ox)$", "${1}en"},
	{"^(oxen)$", "${1}"},
	{"(quiz)$", "${1}zes"},
}

var singularInflections = RegularSlice{
	{"s$", ""},
	{"(ss)$", "${1}"},
	{"(n)ews$", "${1}ews"},
	{"([ti])a$", "${1}um"},
	{"((a)naly|(b)a|(d)iagno|(p)arenthe|(p)rogno|(s)ynop|(t)he)(sis|ses)$", "${1}sis"},
	{"(^analy)(sis|ses)$", "${1}sis"},
	{"([^f])ves$", "${1}fe"},
	{"(hive)s$", "${1}"},
	{"(tive)s$", "${1}"},
	{"([lr])ves$", "${1}f"},
	{"([^aeiouy]|qu)ies$", "${1}y"},
	{"(s)eries$", "${1}eries"},
	{"(m)ovies$", "${1}ovie"},
	{"(c)ookies$", "${1}ookie"},
	{"(x|ch|ss|sh)es$", "${1}"},
	{"^(m|l)ice$", "${1}ouse"},
	{"(bus)(es)?$", "${1}"},
	{"(o)es$", "${1}"},
	{"(shoe)s$", "${1}"},
	{"(cris|test)(is|es)$", "${1}is"},
	{"^(a)x[ie]s$", "${1}xis"},
	{"(octop|vir)(us|i)$", "${1}us"},
	{"(alias|status)(es)?$", "${1}"},
	{"^(ox)en", "${1}"},
	{"(vert|ind)ices$", "${1}ex"},
	{"(matr)ices$", "${1}ix"},
	{"(quiz)zes$", "${1}"},
	{"(database)s$", "${1}"},
}

var irregularInflections = IrregularSlice{
	{"person", "people"},
	{"man", "men"},
	{"child", "children"},
	{"sex", "sexes"},
	{"move", "moves"},
	{"mombie", "mombies"},
	{"foot", "feet"},
}

var uncountableInflections = []string{"equipment", "information", "rice", "money", "species", "series", "fish", "sheep", "jeans", "police", "evidence"}

var compiledPluralMaps []inflection
var compiledSingularMaps []inflection

func compile() {
	compiledPluralMaps = []inflection{}
	compiledSingularMaps = []inflection{}
	for _, uncountable := range uncountableInflections {
		inf := inflection{
			regexp:  regexp.MustCompile("^(?i)(" + uncountable + ")$"),
			replace: "${1}",
		}
		compiledPluralMaps = append(compiledPluralMaps, inf)
		compiledSingularMaps = append(compiledSingularMaps, inf)
	}

	for _, value := range irregularInflections {
		infs := []inflection{
			inflection{regexp: regexp.MustCompile(strings.ToUpper(value.singular) + "$"), replace: strings.ToUpper(value.plural)},
			inflection{regexp: regexp.MustCompile(strings.Title(value.singular) + "$"), replace: strings.Title(value.plural)},
			inflection{regexp: regexp.MustCompile(value.singular + "$"), replace: value.plural},
		}
		compiledPluralMaps = append(compiledPluralMaps, infs...)
	}

	for _, value := range irregularInflections {
		infs := []inflection{
			inflection{regexp: regexp.MustCompile(strings.ToUpper(value.plural) + "$"), replace: strings.ToUpper(value.singular)},
			inflection{regexp: regexp.MustCompile(strings.Title(value.plural) + "$"), replace: strings.Title(value.singular)},
			inflection{regexp: regexp.MustCompile(value.plural + "$"), replace: value.singular},
		}
		compiledSingularMaps = append(compiledSingularMaps, infs...)
	}

	for i := len(pluralInflections) - 1; i >= 0; i-- {
		value := pluralInflections[i]
		infs := []inflection{
			inflection{regexp: regexp.MustCompile(strings.ToUpper(value.find)), replace: strings.ToUpper(value.replace)},
			inflection{regexp: regexp.MustCompile(value.find), replace: value.replace},
			inflection{regexp: regexp.MustCompile("(?i)" + value.find), replace: value.replace},
		}
		compiledPluralMaps = append(compiledPluralMaps, infs...)
	}

	for i := len(singularInflections) - 1; i >= 0; i-- {
		value := singularInflections[i]
		infs := []inflection{
			inflection{regexp: regexp.MustCompile(strings.ToUpper(value.find)), replace: strings.ToUpper(value.replace)},
			inflection{regexp: regexp.MustCompile(value.find), replace: value.replace},
			inflection{regexp: regexp.MustCompile("(?i)" + value.find), replace: value.replace},
		}
		compiledSingularMaps = append(compiledSingularMaps, infs...)
	}
}

func init() {
	compile()
}

// AddPlural adds a plural inflection
func AddPlural(find, replace string) {
	pluralInflections = append(pluralInflections, Regular{find, replace})
	compile()
}

// AddSingular adds a singular inflection
func AddSingular(find, replace string) {
	singularInflections = append(singularInflections, Regular{find, replace})
	compile()
}

// AddIrregular adds an irregular inflection
func AddIrregular(singular, plural string) {
	irregularInflections = append(irregularInflections, Irregular{singular, plural})
	compile()
}

// AddUncountable adds an uncountable inflection
func AddUncountable(values ...string) {
	uncountableInflections = append(uncountableInflections, values...)
	compile()
}

// GetPlural retrieves the plural inflection values
func GetPlural() RegularSlice {
	plurals := make(RegularSlice, len(pluralInflections))
	copy(plurals, pluralInflections)
	return plurals
}

// GetSingular retrieves the singular inflection values
func GetSingular() RegularSlice {
	singulars := make(RegularSlice, len(singularInflections))
	copy(singulars, singularInflections)
	return singulars
}

// GetIrregular retrieves the irregular inflection values
func GetIrregular() IrregularSlice {
	irregular := make(IrregularSlice, len(irregularInflections))
	copy(irregular, irregularInflections)
	return irregular
}

// GetUncountable retrieves the uncountable inflection values
func GetUncountable() []string {
	uncountables := make([]string, len(uncountableInflections))
	copy(uncountables, uncountableInflections)
	return uncountables
}

// SetPlural sets the plural inflections slice
func SetPlural(inflections RegularSlice) {
	pluralInflections = inflections
	compile()
}

// SetSingular sets the singular inflections slice
func SetSingular(inflections RegularSlice) {
	singularInflections = inflections
	compile()
}

// SetIrregular sets the irregular inflections slice
func SetIrregular(inflections IrregularSlice) {
	irregularInflections = inflections
	compile()
}

// SetUncountable sets the uncountable inflections slice
func SetUncountable(inflections []string) {
	uncountableInflections = inflections
	compile()
}

// Plural converts a word to its plural form
func Plural(str string) string {
	for _, inflection := range compiledPluralMaps {
		if inflection.regexp.MatchString(str) {
			return inflection.regexp.ReplaceAllString(str, inflection.replace)
		}
	}
	return str
}

// Singular converts a word to its singular form
func Singular(str string) string {
	for _, inflection := range compiledSingularMaps {
		if inflection.regexp.MatchString(str) {
			return inflection.regexp.ReplaceAllString(str, inflection.replace)
		}
	}
	return str
}
<|endoftext|>


<commit_before>// Copyright (c) 2012-2014 Jeremy Latt
// Copyright (c) 2014-2015 Edmund Huber
// Copyright (c) 2016- Daniel Oaks <daniel@danieloaks.net>
// released under the MIT license

package irc

import (
	"regexp"
	"strings"

	"golang.org/x/text/unicode/norm"
)

var (
	// regexps
	// these get replaced with real regexes at server load time

	ChannelNameExpr = regexp.MustCompile("^$")
	NicknameExpr    = regexp.MustCompile("^$")
)

// Names are normalized and canonicalized to remove formatting marks
// and simplify usage. They are things like hostnames and usermasks.
type Name string

func NewName(str string) Name {
	return Name(norm.NFKC.String(str))
}

func NewNames(strs []string) []Name {
	names := make([]Name, len(strs))
	for index, str := range strs {
		names[index] = NewName(str)
	}
	return names
}

// tests

func (name Name) IsChannel() bool {
	return ChannelNameExpr.MatchString(name.String())
}

func (name Name) IsNickname() bool {
	namestr := name.String()
	// * is used for unregistered clients
	// * is used for mask matching
	// ? is used for mask matching
	// , is used as a separator by the protocol
	// ! separates username from nickname
	// @ separates nick+user from hostname
	// # is a channel prefix
	// @+ are channel membership prefixes
	if strings.Contains(namestr, "*") || strings.Contains(namestr, "?") ||
		strings.Contains(namestr, ",") || strings.Contains(namestr, "!") ||
		strings.Contains(namestr, "@") || strings.Contains("#@+", string(namestr[0])) {
		return false
	}
	// names that look like hostnames are restricted to servers, as with other ircds
	if IsHostname(namestr) {
		return false
	}
	return NicknameExpr.MatchString(namestr)
}

// conversions

func (name Name) String() string {
	return string(name)
}

func (name Name) ToLower() Name {
	return Name(strings.ToLower(name.String()))
}

// It's safe to coerce a Name to Text. Name is a strict subset of Text.
func (name Name) Text() Text {
	return Text(name)
}

// Text is PRIVMSG, NOTICE, or TOPIC data. It's canonicalized UTF8
// data to simplify but keeps all formatting.
type Text string

func NewText(str string) Text {
	return Text(norm.NFC.String(str))
}

func (text Text) String() string {
	return string(text)
}

// CTCPText is text suitably escaped for CTCP.
type CTCPText string

var ctcpEscaper = strings.NewReplacer("\x00", "\x200", "\n", "\x20n", "\r", "\x20r")

func NewCTCPText(str string) CTCPText {
	return CTCPText(ctcpEscaper.Replace(str))
}
<commit_msg>strings: Don't allow nicks to start with dashes or 0-9<commit_after>// Copyright (c) 2012-2014 Jeremy Latt
// Copyright (c) 2014-2015 Edmund Huber
// Copyright (c) 2016- Daniel Oaks <daniel@danieloaks.net>
// released under the MIT license

package irc

import (
	"regexp"
	"strings"

	"golang.org/x/text/unicode/norm"
)

var (
	// regexps
	// these get replaced with real regexes at server load time

	ChannelNameExpr = regexp.MustCompile("^$")
	NicknameExpr    = regexp.MustCompile("^$")
)

// Names are normalized and canonicalized to remove formatting marks
// and simplify usage. They are things like hostnames and usermasks.
type Name string

func NewName(str string) Name {
	return Name(norm.NFKC.String(str))
}

func NewNames(strs []string) []Name {
	names := make([]Name, len(strs))
	for index, str := range strs {
		names[index] = NewName(str)
	}
	return names
}

// tests

func (name Name) IsChannel() bool {
	return ChannelNameExpr.MatchString(name.String())
}

func (name Name) IsNickname() bool {
	namestr := name.String()
	// * is used for unregistered clients
	// * is used for mask matching
	// ? is used for mask matching
	// , is used as a separator by the protocol
	// ! separates username from nickname
	// @ separates nick+user from hostname
	// # is a channel prefix
	// @+ are channel membership prefixes
	// - is typically disallowed from first char of nicknames
	// nicknames can't start with digits
	if strings.Contains(namestr, "*") || strings.Contains(namestr, "?") ||
		strings.Contains(namestr, ",") || strings.Contains(namestr, "!") ||
		strings.Contains(namestr, "@") || strings.Contains("#@+-1234567890", string(namestr[0])) {
		return false
	}
	// names that look like hostnames are restricted to servers, as with other ircds
	if IsHostname(namestr) {
		return false
	}
	return NicknameExpr.MatchString(namestr)
}

// conversions

func (name Name) String() string {
	return string(name)
}

func (name Name) ToLower() Name {
	return Name(strings.ToLower(name.String()))
}

// It's safe to coerce a Name to Text. Name is a strict subset of Text.
func (name Name) Text() Text {
	return Text(name)
}

// Text is PRIVMSG, NOTICE, or TOPIC data. It's canonicalized UTF8
// data to simplify but keeps all formatting.
type Text string

func NewText(str string) Text {
	return Text(norm.NFC.String(str))
}

func (text Text) String() string {
	return string(text)
}

// CTCPText is text suitably escaped for CTCP.
type CTCPText string

var ctcpEscaper = strings.NewReplacer("\x00", "\x200", "\n", "\x20n", "\r", "\x20r")

func NewCTCPText(str string) CTCPText {
	return CTCPText(ctcpEscaper.Replace(str))
}
<|endoftext|>


<commit_before>package isitup

import (
	"reflect"
	"testing"
)

func TestGetDomain(t *testing.T) {
	v := GetDomain("example.com", "go-isitup Unit Tests")
	if v != "example.com" {
		t.Error("Expected example.com, got", v)
	}
}

func TestGetIP(t *testing.T) {
	v := GetIP("example.com", "go-isitup Unit Tests")
	if v != "93.184.216.34" {
		t.Error("Expected 93.184.216.34, got", v)
	}
}

func TestGetPort(t *testing.T) {
	v := GetPort("example.com", "go-isitup Unit Tests")
	if v != 80 {
		t.Error("Expected 80, got", v)
	}
}

func TestGetStatusCode(t *testing.T) {
	v := GetStatusCode("example.com", "go-isitup Unit Tests")
	if v != 200 {
		t.Error("Expected HTTP code 200, got", v)
	}
}

func TestGetResponseTime(t *testing.T) {
	v := GetResponseTime("example.com", "go-isitup Unit Tests")
	typ := reflect.TypeOf(v)

	if typ.Kind() != reflect.Float64 {
		t.Error("Expected type of `float64`, got", reflect.TypeOf(v))
	}
}
<commit_msg>tests: lowercase 'T' in user agent header<commit_after>/*
*  Copyright 2018 Dom Rodriguez (shymega)
*
*  Licensed under the Apache License, Version 2.0 (the "License");
*  you may not use this file except in compliance with the License.
*  You may obtain a copy of the License at
*
*      http://www.apache.org/licenses/LICENSE-2.0
*
*  Unless required by applicable law or agreed to in writing, software
*  distributed under the License is distributed on an "AS IS" BASIS,
*  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*  See the License for the specific language governing permissions and
*  limitations under the License.
*/

package isitup

import (
	"reflect"
	"testing"
)

func TestGetDomain(t *testing.T) {
	v := GetDomain("example.com", "go-isitup Unit tests")
	if v != "example.com" {
		t.Error("Expected example.com, got", v)
	}
}

func TestGetIP(t *testing.T) {
	v := GetIP("example.com", "go-isitup Unit tests")
	if v != "93.184.216.34" {
		t.Error("Expected 93.184.216.34, got", v)
	}
}

func TestGetPort(t *testing.T) {
	v := GetPort("example.com", "go-isitup Unit tests")
	if v != 80 {
		t.Error("Expected 80, got", v)
	}
}

func TestGetStatusCode(t *testing.T) {
	v := GetStatusCode("example.com", "go-isitup Unit tests")
	if v != 200 {
		t.Error("Expected HTTP code 200, got", v)
	}
}

func TestGetResponseTime(t *testing.T) {
	v := GetResponseTime("example.com", "go-isitup Unit tests")
	typ := reflect.TypeOf(v)

	if typ.Kind() != reflect.Float64 {
		t.Error("Expected type of `float64`, got", reflect.TypeOf(v))
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2014 Pagoda Box Inc.
//
// This Source Code Form is subject to the terms of the Mozilla Public License,
// v. 2.0. If a copy of the MPL was not distributed with this file, You can
// obtain one at http://mozilla.org/MPL/2.0/.

//
package jobs

//
import (
	"fmt"
	"reflect"
	"strings"

	"github.com/nanobox-io/nanobox-boxfile"
	"github.com/nanobox-io/nanobox-golang-stylish"
	// "github.com/nanobox-io/nanobox-logtap"
	"github.com/nanobox-io/nanobox-router"
	"github.com/nanobox-io/nanobox-server/config"
	"github.com/nanobox-io/nanobox-server/util"
	"github.com/nanobox-io/nanobox-server/util/docker"
	"github.com/nanobox-io/nanobox-server/util/fs"
	"github.com/nanobox-io/nanobox-server/util/script"
	"github.com/nanobox-io/nanobox-server/util/worker"
)

//
type Deploy struct {
	ID    string
	Reset bool
	Run   bool

	payload map[string]interface{}
}

// Proccess syncronies your docker containers with the boxfile specification
func (j *Deploy) Process() {
	// add a lock so the service wont go down whil im running
	util.Lock()
	defer util.Unlock()

	// set routing to watch logs
	router.ErrorHandler = router.DeployInProgress{}

	// remove all code containers
	util.LogInfo(stylish.Bullet("Cleaning containers"))
	if err := j.RemoveOldContainers(); err != nil {
		util.UpdateStatus(j, "errored")
		return
	}

	if err := j.SetupFS(); err != nil {
		util.UpdateStatus(j, "errored")
		return
	}

	// parse the boxfile
	util.LogDebug(stylish.Bullet("Parsing Boxfile"))
	box := UserBoxfile(true)

	if err := j.CreateBuildContainer(box.Node("build")); err != nil {
		util.UpdateStatus(j, "errored")
		return
	}

	// define the deploy payload
	j.payload = map[string]interface{}{
		"platform":    "local",
		"app":         config.App(),
		"dns":         []string{config.App() + ".dev"},
		"port":        "8080",
		"boxfile":     box.Node("build").Parsed,
		"logtap_host": config.LogtapHost,
	}

	j.payload["env"] = DefaultEVars(*box)

	if err := j.SetupBuild(); err != nil {
		util.UpdateStatus(j, "errored")
		return
	}

	// get a cached copy of the combined boxfile
	oldCombinedBox := CombinedBoxfile(false)

	// make sure to grab the new engine boxfile
	EngineBoxfile(true)
	// grab a new boxfile
	box = CombinedBoxfile(true)
	// add the missing storage nodes to the boxfile
	box.AddStorageNode()

	j.payload["boxfile"] = box.Node("build").Parsed

	// remove any containers no longer in the boxfile
	// this will also remove any services where the boxfile has been modified since last deploy
	util.LogDebug(stylish.Bullet("Removing old containers..."))
	serviceContainers, _ := docker.ListContainers("service")
	for _, container := range serviceContainers {
		if !box.Node(container.Config.Labels["uid"]).Valid {
			util.LogDebug(stylish.SubBullet("- removing " + container.Config.Labels["uid"]))
			util.RemoveForward(container.NetworkSettings.IPAddress)
			docker.RemoveContainer(container.ID)
		}
		if !reflect.DeepEqual(box.Node(container.Config.Labels["uid"]), oldCombinedBox.Node(container.Config.Labels["uid"])) {
			util.LogDebug(stylish.SubBullet("- replacing " + container.Config.Labels["uid"]))
			util.RemoveForward(container.NetworkSettings.IPAddress)
			docker.RemoveContainer(container.ID)
		}
	}

	worker := worker.New()
	worker.Blocking = true

	//
	serviceStarts := []*ServiceStart{}

	// build service containers according to boxfile
	for _, node := range box.Nodes("service") {
		if _, err := docker.GetContainer(node); err != nil {
			// container doesn't exist so we need to create it
			s := ServiceStart{
				Boxfile: box.Node(node),
				UID:     node,
				EVars:   map[string]string{},
			}

			serviceStarts = append(serviceStarts, &s)

			worker.Queue(&s)
		}
	}

	if worker.Count() > 0 {
		util.LogInfo(stylish.Bullet("Launching data services"))
	}

	// we dont want service starts to be concurrent here for messaging
	worker.Concurrent = false
	worker.Process()
	// make the worker concurrent from here on
	worker.Concurrent = true

	failedStart := false
	// ensure all services started correctly before continuing
	for _, starts := range serviceStarts {
		if !starts.Success {
			util.HandleError(stylish.ErrorHead("Failed to start %v", starts.UID))
			util.HandleError(stylish.ErrorBody(""))
			failedStart = true
		}
	}
	if failedStart {
		util.UpdateStatus(j, "errored")
		return
	}

	// grab the environment data from all service containers
	evars := j.payload["env"].(map[string]string)

	// clear out the old ports from the previous deploy
	clearPorts()

	//
	serviceEnvs := []*ServiceEnv{}

	serviceContainers, _ = docker.ListContainers("service")
	for _, container := range serviceContainers {

		s := ServiceEnv{UID: container.Config.Labels["uid"]}
		for _, serviceStart := range serviceStarts {
			if serviceStart.UID == s.UID {
				s.FirstTime = true
			}
		}
		serviceEnvs = append(serviceEnvs, &s)

		worker.Queue(&s)
	}

	worker.Process()

	failedEnv := false
	for _, env := range serviceEnvs {
		if !env.Success {
			util.HandleError(stylish.ErrorHead("Failed to configure %v's environment variables", env.UID))
			util.HandleError(stylish.ErrorBody(""))
			failedEnv = true
			continue
		}

		for key, val := range env.EVars {
			evars[strings.ToUpper(env.UID+"_"+key)] = val
		}
	}
	if failedEnv {
		util.UpdateStatus(j, "errored")

		return
	}

	j.payload["env"] = evars

	if err := j.RunBuild(); err != nil {
		util.UpdateStatus(j, "errored")
		return
	}

	// we will only create new code nodes if we are
	// supposed to be running
	if j.Run {

		// build new code containers
		codeServices := []*ServiceStart{}
		for _, node := range box.Nodes("code") {
			if _, err := docker.GetContainer(node); err != nil {
				// container doesn't exist so we need to create it
				s := ServiceStart{
					Boxfile: box.Node(node),
					UID:     node,
					EVars:   evars,
				}

				codeServices = append(codeServices, &s)

				worker.Queue(&s)
			}
			if worker.Count() > 0 {
				util.LogInfo(stylish.Bullet("Launching Code services"))
			}
		}

		worker.Process()

		for _, serv := range codeServices {
			if !serv.Success {
				util.HandleError("A Service was not started correctly (" + serv.UID + ")")
				util.UpdateStatus(j, "errored")
				return
			}
		}
	}

	util.LogDebug(stylish.Bullet("Running before deploy scripts..."))

	if err := j.RunDeployScripts("before", *box); err != nil {
		util.UpdateStatus(j, "errored")
		return
	}

	// configure the port forwards per service
	if err := configurePorts(*box); err != nil {
		util.HandleError(stylish.Error("Failed to configure Ports", err.Error()))
		util.UpdateStatus(j, "errored")
		return
	}

	// configure the routing mesh for any web services
	if err := configureRoutes(*box); err != nil {
		util.HandleError(stylish.Error("Failed to configure Routes", err.Error()))
		util.UpdateStatus(j, "errored")
		return
	}

	//
	util.LogDebug(stylish.Bullet("Running after deploy hooks..."))

	if err := j.RunDeployScripts("after", *box); err != nil {
		util.UpdateStatus(j, "errored")
		return
	}

	util.UpdateStatus(j, "complete")
}

func (j *Deploy) RemoveOldContainers() error {
	// might as well remove bootstraps and execs too
	containers, _ := docker.ListContainers("code", "build", "bootstrap", "dev", "tcp", "udp")
	for _, container := range containers {
		util.RemoveForward(container.NetworkSettings.IPAddress)
		if err := docker.RemoveContainer(container.ID); err != nil {
			util.HandleError(stylish.Error("Failed to remove old containers", err.Error()))
			return err
		}
	}
	return nil
}

func (j *Deploy) SetupFS() error {
	// Make sure we have the directories
	if err := fs.CreateDirs(); err != nil {
		util.HandleError(stylish.Error("Failed to create dirs", err.Error()))
		return err
	}

	// wipe the previous deploy data if reset == true
	if j.Reset {
		util.LogInfo(stylish.Bullet("Emptying cache"))
		if err := fs.Clean(); err != nil {
			util.HandleError(stylish.Warning("Failed to reset cache and code directories:\n%v", err.Error()))
			return err
		}
	}
	return nil
}

func (j *Deploy) CreateBuildContainer(box boxfile.Boxfile) error {
	image := "nanobox/build"

	if stab := box.StringValue("stability"); stab != "" {
		image = image + ":" + stab
	}

	// if the build image doesn't exist it needs to be downloaded
	if !docker.ImageExists(image) {
		util.LogInfo(stylish.Bullet("Pulling the latest build image (this may take awhile)... "))
		docker.InstallImage(image)
	}

	util.LogDebug(stylish.Bullet("image name: %v", image))

	// create a build container
	util.LogInfo(stylish.Bullet("Creating build container"))

	_, err := docker.CreateContainer(docker.CreateConfig{Image: image, Category: "build", UID: "build1"})
	if err != nil {
		util.HandleError(stylish.Error("Failed to create build container", err.Error()))
		return err
	}
	return nil
}

func (j *Deploy) SetupBuild() error {
	// run the default-user hook to get ssh keys setup
	if _, err := script.Exec("default-user", "build1", fs.UserPayload()); err != nil {
		return err
	}

	if _, err := script.Exec("default-configure", "build1", j.payload); err != nil {
		return err
	}

	if _, err := script.Exec("default-detect", "build1", j.payload); err != nil {
		return err
	}

	if _, err := script.Exec("default-sync", "build1", j.payload); err != nil {
		return err
	}

	if _, err := script.Exec("default-setup", "build1", j.payload); err != nil {
		return err
	}
	return nil
}

func (j *Deploy) RunBuild() error {
	// run prepare script (blocking)
	if _, err := script.Exec("default-prepare", "build1", j.payload); err != nil {
		return err
	}

	if j.Run {
		// run build script (blocking)
		if _, err := script.Exec("default-build", "build1", j.payload); err != nil {
			return err
		}

		// run publish script (blocking)
		if _, err := script.Exec("default-publish", "build1", j.payload); err != nil {
			return err
		}
	}

	// run cleanup script (blocking)
	if _, err := script.Exec("default-cleanup", "build1", j.payload); err != nil {
		return err
	}
	return nil
}

func (j *Deploy) RunDeployScripts(stage string, box boxfile.Boxfile) error {
	// run before deploy scripts
	for _, node := range box.Nodes() {
		bd := box.Node(node).Value(stage + "_deploy")
		bda := box.Node(node).Value(stage + "_deploy_all")
		if bd != nil || bda != nil {

			// run before deploy script (blocking)
			if _, err := script.Exec(fmt.Sprintf("default-%s_deploy", stage), node, map[string]interface{}{stage + "_deploy": bd, stage + "_deploy_all": bda}); err != nil {
				return err
			}
		}
	}
	return nil
}
<commit_msg>add in a run flag in the payload<commit_after>// Copyright (c) 2014 Pagoda Box Inc.
//
// This Source Code Form is subject to the terms of the Mozilla Public License,
// v. 2.0. If a copy of the MPL was not distributed with this file, You can
// obtain one at http://mozilla.org/MPL/2.0/.

//
package jobs

//
import (
	"fmt"
	"reflect"
	"strings"

	"github.com/nanobox-io/nanobox-boxfile"
	"github.com/nanobox-io/nanobox-golang-stylish"
	// "github.com/nanobox-io/nanobox-logtap"
	"github.com/nanobox-io/nanobox-router"
	"github.com/nanobox-io/nanobox-server/config"
	"github.com/nanobox-io/nanobox-server/util"
	"github.com/nanobox-io/nanobox-server/util/docker"
	"github.com/nanobox-io/nanobox-server/util/fs"
	"github.com/nanobox-io/nanobox-server/util/script"
	"github.com/nanobox-io/nanobox-server/util/worker"
)

//
type Deploy struct {
	ID    string
	Reset bool
	Run   bool

	payload map[string]interface{}
}

// Proccess syncronies your docker containers with the boxfile specification
func (j *Deploy) Process() {
	// add a lock so the service wont go down whil im running
	util.Lock()
	defer util.Unlock()

	// set routing to watch logs
	router.ErrorHandler = router.DeployInProgress{}

	// remove all code containers
	util.LogInfo(stylish.Bullet("Cleaning containers"))
	if err := j.RemoveOldContainers(); err != nil {
		util.UpdateStatus(j, "errored")
		return
	}

	if err := j.SetupFS(); err != nil {
		util.UpdateStatus(j, "errored")
		return
	}

	// parse the boxfile
	util.LogDebug(stylish.Bullet("Parsing Boxfile"))
	box := UserBoxfile(true)

	if err := j.CreateBuildContainer(box.Node("build")); err != nil {
		util.UpdateStatus(j, "errored")
		return
	}

	// define the deploy payload
	j.payload = map[string]interface{}{
		"platform":    "local",
		"run":         j.Run,
		"app":         config.App(),
		"dns":         []string{config.App() + ".dev"},
		"port":        "8080",
		"boxfile":     box.Node("build").Parsed,
		"logtap_host": config.LogtapHost,
	}

	j.payload["env"] = DefaultEVars(*box)

	if err := j.SetupBuild(); err != nil {
		util.UpdateStatus(j, "errored")
		return
	}

	// get a cached copy of the combined boxfile
	oldCombinedBox := CombinedBoxfile(false)

	// make sure to grab the new engine boxfile
	EngineBoxfile(true)
	// grab a new boxfile
	box = CombinedBoxfile(true)
	// add the missing storage nodes to the boxfile
	box.AddStorageNode()

	j.payload["boxfile"] = box.Node("build").Parsed

	// remove any containers no longer in the boxfile
	// this will also remove any services where the boxfile has been modified since last deploy
	util.LogDebug(stylish.Bullet("Removing old containers..."))
	serviceContainers, _ := docker.ListContainers("service")
	for _, container := range serviceContainers {
		if !box.Node(container.Config.Labels["uid"]).Valid {
			util.LogDebug(stylish.SubBullet("- removing " + container.Config.Labels["uid"]))
			util.RemoveForward(container.NetworkSettings.IPAddress)
			docker.RemoveContainer(container.ID)
		}
		if !reflect.DeepEqual(box.Node(container.Config.Labels["uid"]), oldCombinedBox.Node(container.Config.Labels["uid"])) {
			util.LogDebug(stylish.SubBullet("- replacing " + container.Config.Labels["uid"]))
			util.RemoveForward(container.NetworkSettings.IPAddress)
			docker.RemoveContainer(container.ID)
		}
	}

	worker := worker.New()
	worker.Blocking = true

	//
	serviceStarts := []*ServiceStart{}

	// build service containers according to boxfile
	for _, node := range box.Nodes("service") {
		if _, err := docker.GetContainer(node); err != nil {
			// container doesn't exist so we need to create it
			s := ServiceStart{
				Boxfile: box.Node(node),
				UID:     node,
				EVars:   map[string]string{},
			}

			serviceStarts = append(serviceStarts, &s)

			worker.Queue(&s)
		}
	}

	if worker.Count() > 0 {
		util.LogInfo(stylish.Bullet("Launching data services"))
	}

	// we dont want service starts to be concurrent here for messaging
	worker.Concurrent = false
	worker.Process()
	// make the worker concurrent from here on
	worker.Concurrent = true

	failedStart := false
	// ensure all services started correctly before continuing
	for _, starts := range serviceStarts {
		if !starts.Success {
			util.HandleError(stylish.ErrorHead("Failed to start %v", starts.UID))
			util.HandleError(stylish.ErrorBody(""))
			failedStart = true
		}
	}
	if failedStart {
		util.UpdateStatus(j, "errored")
		return
	}

	// grab the environment data from all service containers
	evars := j.payload["env"].(map[string]string)

	// clear out the old ports from the previous deploy
	clearPorts()

	//
	serviceEnvs := []*ServiceEnv{}

	serviceContainers, _ = docker.ListContainers("service")
	for _, container := range serviceContainers {

		s := ServiceEnv{UID: container.Config.Labels["uid"]}
		for _, serviceStart := range serviceStarts {
			if serviceStart.UID == s.UID {
				s.FirstTime = true
			}
		}
		serviceEnvs = append(serviceEnvs, &s)

		worker.Queue(&s)
	}

	worker.Process()

	failedEnv := false
	for _, env := range serviceEnvs {
		if !env.Success {
			util.HandleError(stylish.ErrorHead("Failed to configure %v's environment variables", env.UID))
			util.HandleError(stylish.ErrorBody(""))
			failedEnv = true
			continue
		}

		for key, val := range env.EVars {
			evars[strings.ToUpper(env.UID+"_"+key)] = val
		}
	}
	if failedEnv {
		util.UpdateStatus(j, "errored")

		return
	}

	j.payload["env"] = evars

	if err := j.RunBuild(); err != nil {
		util.UpdateStatus(j, "errored")
		return
	}

	// we will only create new code nodes if we are
	// supposed to be running
	if j.Run {

		// build new code containers
		codeServices := []*ServiceStart{}
		for _, node := range box.Nodes("code") {
			if _, err := docker.GetContainer(node); err != nil {
				// container doesn't exist so we need to create it
				s := ServiceStart{
					Boxfile: box.Node(node),
					UID:     node,
					EVars:   evars,
				}

				codeServices = append(codeServices, &s)

				worker.Queue(&s)
			}
			if worker.Count() > 0 {
				util.LogInfo(stylish.Bullet("Launching Code services"))
			}
		}

		worker.Process()

		for _, serv := range codeServices {
			if !serv.Success {
				util.HandleError("A Service was not started correctly (" + serv.UID + ")")
				util.UpdateStatus(j, "errored")
				return
			}
		}
	}

	util.LogDebug(stylish.Bullet("Running before deploy scripts..."))

	if err := j.RunDeployScripts("before", *box); err != nil {
		util.UpdateStatus(j, "errored")
		return
	}

	// configure the port forwards per service
	if err := configurePorts(*box); err != nil {
		util.HandleError(stylish.Error("Failed to configure Ports", err.Error()))
		util.UpdateStatus(j, "errored")
		return
	}

	// configure the routing mesh for any web services
	if err := configureRoutes(*box); err != nil {
		util.HandleError(stylish.Error("Failed to configure Routes", err.Error()))
		util.UpdateStatus(j, "errored")
		return
	}

	//
	util.LogDebug(stylish.Bullet("Running after deploy hooks..."))

	if err := j.RunDeployScripts("after", *box); err != nil {
		util.UpdateStatus(j, "errored")
		return
	}

	util.UpdateStatus(j, "complete")
}

func (j *Deploy) RemoveOldContainers() error {
	// might as well remove bootstraps and execs too
	containers, _ := docker.ListContainers("code", "build", "bootstrap", "dev", "tcp", "udp")
	for _, container := range containers {
		util.RemoveForward(container.NetworkSettings.IPAddress)
		if err := docker.RemoveContainer(container.ID); err != nil {
			util.HandleError(stylish.Error("Failed to remove old containers", err.Error()))
			return err
		}
	}
	return nil
}

func (j *Deploy) SetupFS() error {
	// Make sure we have the directories
	if err := fs.CreateDirs(); err != nil {
		util.HandleError(stylish.Error("Failed to create dirs", err.Error()))
		return err
	}

	// wipe the previous deploy data if reset == true
	if j.Reset {
		util.LogInfo(stylish.Bullet("Emptying cache"))
		if err := fs.Clean(); err != nil {
			util.HandleError(stylish.Warning("Failed to reset cache and code directories:\n%v", err.Error()))
			return err
		}
	}
	return nil
}

func (j *Deploy) CreateBuildContainer(box boxfile.Boxfile) error {
	image := "nanobox/build"

	if stab := box.StringValue("stability"); stab != "" {
		image = image + ":" + stab
	}

	// if the build image doesn't exist it needs to be downloaded
	if !docker.ImageExists(image) {
		util.LogInfo(stylish.Bullet("Pulling the latest build image (this may take awhile)... "))
		docker.InstallImage(image)
	}

	util.LogDebug(stylish.Bullet("image name: %v", image))

	// create a build container
	util.LogInfo(stylish.Bullet("Creating build container"))

	_, err := docker.CreateContainer(docker.CreateConfig{Image: image, Category: "build", UID: "build1"})
	if err != nil {
		util.HandleError(stylish.Error("Failed to create build container", err.Error()))
		return err
	}
	return nil
}

func (j *Deploy) SetupBuild() error {
	// run the default-user hook to get ssh keys setup
	if _, err := script.Exec("default-user", "build1", fs.UserPayload()); err != nil {
		return err
	}

	if _, err := script.Exec("default-configure", "build1", j.payload); err != nil {
		return err
	}

	if _, err := script.Exec("default-detect", "build1", j.payload); err != nil {
		return err
	}

	if _, err := script.Exec("default-sync", "build1", j.payload); err != nil {
		return err
	}

	if _, err := script.Exec("default-setup", "build1", j.payload); err != nil {
		return err
	}
	return nil
}

func (j *Deploy) RunBuild() error {
	// run prepare script (blocking)
	if _, err := script.Exec("default-prepare", "build1", j.payload); err != nil {
		return err
	}

	if j.Run {
		// run build script (blocking)
		if _, err := script.Exec("default-build", "build1", j.payload); err != nil {
			return err
		}

		// run publish script (blocking)
		if _, err := script.Exec("default-publish", "build1", j.payload); err != nil {
			return err
		}
	}

	// run cleanup script (blocking)
	if _, err := script.Exec("default-cleanup", "build1", j.payload); err != nil {
		return err
	}
	return nil
}

func (j *Deploy) RunDeployScripts(stage string, box boxfile.Boxfile) error {
	// run before deploy scripts
	for _, node := range box.Nodes() {
		bd := box.Node(node).Value(stage + "_deploy")
		bda := box.Node(node).Value(stage + "_deploy_all")
		if bd != nil || bda != nil {

			// run before deploy script (blocking)
			if _, err := script.Exec(fmt.Sprintf("default-%s_deploy", stage), node, map[string]interface{}{stage + "_deploy": bd, stage + "_deploy_all": bda}); err != nil {
				return err
			}
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package fastly

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/google/jsonapi"
	"io"
	"io/ioutil"
	"net/http"
	"reflect"
	"strconv"
	"strings"
)

// Events represents an event_logs item response from the Fastly API.
type Event struct {
	ID          string                 `jsonapi:"primary,event"`
	CustomerID  string                 `jsonapi:"attr,customer_id"`
	Description string                 `jsonapi:"attr,description"`
	EventType   string                 `jsonapi:"attr,event_type"`
	IP          string                 `jsonapi:"attr,ip"`
	Metadata    map[string]interface{} `jsonapi:"attr,metadata,omitempty"`
	ServiceID   string                 `jsonapi:"attr,service_id"`
	UserID      string                 `jsonapi:"attr,user_id"`
	CreatedAt   string                 `jsonapi:"attr,created_at"`
	Admin       bool                   `jsonapi:"attr,admin"`
}

// GetAPIEventsFilter is used as input to the GetAPIEvents function.
type GetAPIEventsFilterInput struct {
	// CustomerID to Limit the returned events to a specific customer.
	CustomerID string

	// ServiceID to Limit the returned events to a specific service.
	ServiceID string

	// EventType to Limit the returned events to a specific event type. See above for event codes.
	EventType string

	// UserID to Limit the returned events to a specific user.
	UserID string

	// Number is the Pagination page number.
	PageNumber int

	// Size is the Number of items to return on each paginated page.
	MaxResults int
}

// eventLinksResponse is used to pull the "Links" pagination fields from
// a call to Fastly; these are excluded from the results of the jsonapi
// call to `UnmarshalManyPayload()`, so we have to fetch them separately.
// type EventLinksResponse struct {
// 	Links EventsPaginationInfo `json:"links"`
// }

// eventsPaginationInfo stores links to searches related to the current one, showing
// any information about additional results being stored on another page
type EventsPaginationInfo struct {
	First string `json:"first,omitempty"`
	Last  string `json:"last,omitempty"`
	Next  string `json:"next,omitempty"`
}

// GetAPIEventsResponse is the data returned to the user from a GetAPIEvents call
type GetAPIEventsResponse struct {
	Events []*Event
	Links  EventsPaginationInfo `json:"links"`
}

// GetAPIEvents gets the events for a particular customer
func (c *Client) GetAPIEvents(i *GetAPIEventsFilterInput) (GetAPIEventsResponse, error) {
	eventsResponse := GetAPIEventsResponse{
		Events: []*Event{},
		Links:  EventsPaginationInfo{},
	}

	path := fmt.Sprintf("/events")

	filters := &RequestOptions{Params: i.formatEventFilters()}

	resp, err := c.Get(path, filters)
	fmt.Println(resp.Request.URL)
	if err != nil {
		return eventsResponse, err
	}
	// if i.PageNumber != 0 {
	// data, err := jsonapi.UnmarshalManyPayload(resp.Body, reflect.TypeOf(new(Event)))
	// if err != nil {
	// 	return eventsResponse, err
	// }

	// 	return eventsResponse, err
	// }
	err = c.interpretAPIEventsPage(&eventsResponse, i.PageNumber, resp)
	// NOTE: It's possible for eventsResponse to be partially completed before an error
	// was encountered, so the presence of a statusResponse doesn't preclude the presence of
	// an error.
	// }
	return eventsResponse, err
}

// GetAPIEventInput is used as input to the GetAPIEvent function.
type GetAPIEventInput struct {
	// EventID is the ID of the event and is required.
	EventID string
}

func (c *Client) GetAPIEvent(i *GetAPIEventInput) (*Event, error) {
	if i.EventID == "" {
		return nil, ErrMissingEventID
	}

	path := fmt.Sprintf("/events/%s", i.EventID)
	resp, err := c.Get(path, nil)
	if err != nil {
		return nil, err
	}

	var event Event
	if err := jsonapi.UnmarshalPayload(resp.Body, &event); err != nil {
		return nil, err
	}
	return &event, nil
}

// interpretAPIEventsPage accepts a Fastly response for a set of WAF rule statuses
// and unmarshals the results. If there are more pages of results, it fetches the next
// page, adds that response to the array of results, and repeats until all results have
// been fetched.
func (c *Client) interpretAPIEventsPage(answer *GetAPIEventsResponse, pageNum int, received *http.Response) error {
	// before we pull the status info out of the response body, fetch
	// pagination info from it:
	pages, body, err := getEventsPages(received.Body)
	if err != nil {
		return err
	}
	answer.Links = pages
	data, err := jsonapi.UnmarshalManyPayload(body, reflect.TypeOf(new(Event)))
	if err != nil {
		return err
	}

	for i := range data {
		typed, ok := data[i].(*Event)
		if !ok {
			return fmt.Errorf("got back response of unexpected type")
		}
		answer.Events = append(answer.Events, typed)
	}
	if pageNum == 0 {
		if pages.Next != "" {
			// NOTE: pages.Next URL includes filters already
			resp, err := c.SimpleGet(pages.Next)
			if err != nil {
				return err
			}
			c.interpretAPIEventsPage(answer, pageNum, resp)
		}
		return nil
	}
	return nil
}

// getEventsPages parses a response to get the pagination data without destroying
// the reader we receive as "resp.Body"; this essentially copies resp.Body
// and returns it so we can use it again.
func getEventsPages(body io.Reader) (EventsPaginationInfo, io.Reader, error) {
	var buf bytes.Buffer
	tee := io.TeeReader(body, &buf)

	bodyBytes, err := ioutil.ReadAll(tee)
	if err != nil {
		return EventsPaginationInfo{}, nil, err
	}

	var pages GetAPIEventsResponse
	json.Unmarshal(bodyBytes, &pages)
	return pages.Links, bytes.NewReader(buf.Bytes()), nil
}

// formatEventFilters converts user input into query parameters for filtering
// Fastly results for rules in an Event.
func (i *GetAPIEventsFilterInput) formatEventFilters() map[string]string {
	result := map[string]string{}
	pairings := map[string]interface{}{
		"filter[customer_id]": i.CustomerID,
		"filter[service_id]":  i.ServiceID,
		"filter[event_type]":  i.EventType,
		"filter[user_id]":     i.UserID,
		"page[size]":          i.MaxResults,
		"page[number]":        i.PageNumber, // starts at 1, not 0
	}
	// NOTE: This setup means we will not be able to send the zero value
	// of any of these filters. It doesn't appear we would need to at present.

	for key, value := range pairings {
		switch t := reflect.TypeOf(value).String(); t {
		case "string":
			if value != "" {
				result[key] = value.(string)
			}
		case "int":
			if value != 0 {
				result[key] = strconv.Itoa(value.(int))
			}
		case "[]int":
			// convert ints to strings
			toStrings := []string{}
			values := value.([]int)
			for _, i := range values {
				toStrings = append(toStrings, strconv.Itoa(i))
			}
			// concat strings
			if len(values) > 0 {
				result[key] = strings.Join(toStrings, ",")
			}
		}

	}
	return result
}
<commit_msg>adding page value to event<commit_after>package fastly

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/google/jsonapi"
	"io"
	"io/ioutil"
	"net/http"
	"reflect"
	"strconv"
	"strings"
)

// Events represents an event_logs item response from the Fastly API.
type Event struct {
	ID          string                 `jsonapi:"primary,event"`
	CustomerID  string                 `jsonapi:"attr,customer_id"`
	Description string                 `jsonapi:"attr,description"`
	EventType   string                 `jsonapi:"attr,event_type"`
	IP          string                 `jsonapi:"attr,ip"`
	Metadata    map[string]interface{} `jsonapi:"attr,metadata,omitempty"`
	ServiceID   string                 `jsonapi:"attr,service_id"`
	UserID      string                 `jsonapi:"attr,user_id"`
	CreatedAt   string                 `jsonapi:"attr,created_at"`
	Admin       bool                   `jsonapi:"attr,admin"`
}

// GetAPIEventsFilter is used as input to the GetAPIEvents function.
type GetAPIEventsFilterInput struct {
	// CustomerID to Limit the returned events to a specific customer.
	CustomerID string

	// ServiceID to Limit the returned events to a specific service.
	ServiceID string

	// EventType to Limit the returned events to a specific event type. See above for event codes.
	EventType string

	// UserID to Limit the returned events to a specific user.
	UserID string

	// Number is the Pagination page number.
	PageNumber int

	// Size is the Number of items to return on each paginated page.
	MaxResults int
}

// eventLinksResponse is used to pull the "Links" pagination fields from
// a call to Fastly; these are excluded from the results of the jsonapi
// call to `UnmarshalManyPayload()`, so we have to fetch them separately.
// type EventLinksResponse struct {
// 	Links EventsPaginationInfo `json:"links"`
// }

// eventsPaginationInfo stores links to searches related to the current one, showing
// any information about additional results being stored on another page
type EventsPaginationInfo struct {
	First string `json:"first,omitempty"`
	Last  string `json:"last,omitempty"`
	Next  string `json:"next,omitempty"`
}

// GetAPIEventsResponse is the data returned to the user from a GetAPIEvents call
type GetAPIEventsResponse struct {
	Events []*Event
	Links  EventsPaginationInfo `json:"links"`
}

// GetAPIEvents gets the events for a particular customer
func (c *Client) GetAPIEvents(i *GetAPIEventsFilterInput) (GetAPIEventsResponse, error) {
	eventsResponse := GetAPIEventsResponse{
		Events: []*Event{},
		Links:  EventsPaginationInfo{},
	}

	path := fmt.Sprintf("/events")

	filters := &RequestOptions{Params: i.formatEventFilters()}

	resp, err := c.Get(path, filters)
	fmt.Println(resp.Request.URL)
	if err != nil {
		return eventsResponse, err
	}
	// if i.PageNumber != 0 {
	// data, err := jsonapi.UnmarshalManyPayload(resp.Body, reflect.TypeOf(new(Event)))
	// if err != nil {
	// 	return eventsResponse, err
	// }

	// 	return eventsResponse, err
	// }
	err = c.interpretAPIEventsPage(&eventsResponse, i.PageNumber, resp)
	// NOTE: It's possible for eventsResponse to be partially completed before an error
	// was encountered, so the presence of a statusResponse doesn't preclude the presence of
	// an error.
	// }
	return eventsResponse, err
}

// GetAPIEventInput is used as input to the GetAPIEvent function.
type GetAPIEventInput struct {
	// EventID is the ID of the event and is required.
	EventID string
}

func (c *Client) GetAPIEvent(i *GetAPIEventInput) (*Event, error) {
	if i.EventID == "" {
		return nil, ErrMissingEventID
	}

	path := fmt.Sprintf("/events/%s", i.EventID)
	resp, err := c.Get(path, nil)
	if err != nil {
		return nil, err
	}

	var event Event
	if err := jsonapi.UnmarshalPayload(resp.Body, &event); err != nil {
		return nil, err
	}
	return &event, nil
}

// interpretAPIEventsPage accepts a Fastly response for a set of WAF rule statuses
// and unmarshals the results. If there are more pages of results, it fetches the next
// page, adds that response to the array of results, and repeats until all results have
// been fetched.
func (c *Client) interpretAPIEventsPage(answer *GetAPIEventsResponse, pageNum int, received *http.Response) error {
	// before we pull the status info out of the response body, fetch
	// pagination info from it:
	pages, body, err := getEventsPages(received.Body)
	if err != nil {
		return err
	}
	answer.Links = pages
	data, err := jsonapi.UnmarshalManyPayload(body, reflect.TypeOf(new(Event)))
	if err != nil {
		return err
	}

	for i := range data {
		typed, ok := data[i].(*Event)
		if !ok {
			return fmt.Errorf("got back response of unexpected type")
		}
		answer.Events = append(answer.Events, typed)
	}
	if pageNum == 0 {
		if pages.Next != "" {
			// NOTE: pages.Next URL includes filters already
			resp, err := c.SimpleGet(pages.Next)
			if err != nil {
				return err
			}
			c.interpretAPIEventsPage(answer, pageNum, resp)
		}
		return nil
	}
	return nil
}

// getEventsPages parses a response to get the pagination data without destroying
// the reader we receive as "resp.Body"; this essentially copies resp.Body
// and returns it so we can use it again.
func getEventsPages(body io.Reader) (EventsPaginationInfo, io.Reader, error) {
	var buf bytes.Buffer
	tee := io.TeeReader(body, &buf)

	bodyBytes, err := ioutil.ReadAll(tee)
	if err != nil {
		return EventsPaginationInfo{}, nil, err
	}

	var pages *GetAPIEventsResponse
	json.Unmarshal(bodyBytes, &pages)
	return pages.Links, bytes.NewReader(buf.Bytes()), nil
}

// formatEventFilters converts user input into query parameters for filtering
// Fastly results for rules in an Event.
func (i *GetAPIEventsFilterInput) formatEventFilters() map[string]string {
	result := map[string]string{}
	pairings := map[string]interface{}{
		"filter[customer_id]": i.CustomerID,
		"filter[service_id]":  i.ServiceID,
		"filter[event_type]":  i.EventType,
		"filter[user_id]":     i.UserID,
		"page[size]":          i.MaxResults,
		"page[number]":        i.PageNumber, // starts at 1, not 0
	}
	// NOTE: This setup means we will not be able to send the zero value
	// of any of these filters. It doesn't appear we would need to at present.

	for key, value := range pairings {
		switch t := reflect.TypeOf(value).String(); t {
		case "string":
			if value != "" {
				result[key] = value.(string)
			}
		case "int":
			if value != 0 {
				result[key] = strconv.Itoa(value.(int))
			}
		case "[]int":
			// convert ints to strings
			toStrings := []string{}
			values := value.([]int)
			for _, i := range values {
				toStrings = append(toStrings, strconv.Itoa(i))
			}
			// concat strings
			if len(values) > 0 {
				result[key] = strings.Join(toStrings, ",")
			}
		}

	}
	return result
}
<|endoftext|>


<commit_before>// fujinews is a stupid scraper.
//
// There isn't any **direct** and **no BS** channel to recieve latest firmware
// update news for Fujifilm X series (and possibly new cameras and lenses) as
// far as i know.  Even though scraping a site to fill this gap is not a
// brilliant idea, it does the job for me.
package main

// TODO(ig): Be a good citizen and extract `news` category from http://feeds.feedburner.com/fujilove

import (
	"fmt"
	"log"

	"github.com/PuerkitoBio/goquery"
)

const baseURL = "http://fujilove.com/category/news/"

func main() {
	doc, err := goquery.NewDocument(baseURL)
	if err != nil {
		log.Fatal(err)
	}

	doc.Find("#sidebar a").Each(func(i int, s *goquery.Selection) {
		if val, ok := s.Attr("target"); ok {
			if val == "_self" {
				href, _ := s.Attr("href")
				fmt.Printf("- %v\n\t%v\n", s.Text(), href)
			}
		}
	})
}
<commit_msg>set a http.client timeout<commit_after>// fujinews is a stupid scraper.
//
// There isn't any **direct** and **no BS** channel to recieve latest firmware
// update news for Fujifilm X series (and possibly new cameras and lenses) as
// far as i know.  Even though scraping a site to fill this gap is not a
// brilliant idea, it does the job for me.
package main

// TODO(ig): Be a good citizen and extract `news` category from http://feeds.feedburner.com/fujilove

import (
	"fmt"
	"log"
	"net/http"
	"time"

	"github.com/PuerkitoBio/goquery"
	rss "github.com/jteeuwen/go-pkg-rss"
)

const (
	cacheTimeout = 24 * 60
	webURL       = "http://fujilove.com/category/news/"
	rssURL       = "http://feeds.feedburner.com/fujilove"
)

var c = http.Client{Timeout: 1 * time.Second}

func scrape() {
	resp, err := c.Get(webURL)
	if err != nil {
		log.Fatal(resp)
	}
	defer resp.Body.Close()

	doc, err := goquery.NewDocumentFromResponse(resp)
	if err != nil {
		log.Fatal(err)
	}

	doc.Find("#sidebar a").Each(func(i int, s *goquery.Selection) {
		if val, ok := s.Attr("target"); ok {
			if val == "_self" {
				href, _ := s.Attr("href")
				fmt.Printf("- %v\n\t%v\n", s.Text(), href)
			}
		}
	})
}

func itemHandler(f *rss.Feed, ch *rss.Channel, items []*rss.Item) {
	for _, item := range items {
		for _, c := range item.Categories {
			// Firmware and new lens/camera updates are in `News` category.
			if c.Text != "News" {
				continue
			}

			fmt.Println(item.Title)
		}
	}
}

func main() {
	scrape()
}
<|endoftext|>