<commit_before>package multipartfs

import (
	"io"
	"io/ioutil"
	"mime/multipart"
	"net/http"
	"path"
	"time"

	fs "github.com/ungerik/go-fs"
	"github.com/ungerik/go-fs/fsimpl"
)

const (
	// Prefix for the MultipartFileSystem
	Prefix = "multipart://"

	// Separator used in MultipartFileSystem paths
	Separator = "/"
)

// MultipartFileSystem wraps the files in a MIME multipart message as fs.FileSystem
type MultipartFileSystem struct {
	fs.ReadOnlyBase

	prefix string
	Form   *multipart.Form
}

// FromRequestForm returns a MultipartFileSystem from a http.Request
func FromRequestForm(request *http.Request, maxMemory int64) (*MultipartFileSystem, error) {
	err := request.ParseMultipartForm(maxMemory)
	if err != nil {
		return nil, err
	}
	mpfs := &MultipartFileSystem{
		prefix: Prefix + fsimpl.RandomString(),
		Form:   request.MultipartForm,
	}
	fs.Register(mpfs)
	return mpfs, err
}

func (mpfs *MultipartFileSystem) Close() error {
	fs.Unregister(mpfs)
	return mpfs.Form.RemoveAll()
}

// FormFile returns the first file uploaded under name
// or ErrDoesNotExist if there is no file under name.
func (mpfs *MultipartFileSystem) FormFile(name string) (fs.File, error) {
	ff, _ := mpfs.Form.File[name]
	if len(ff) == 0 {
		return "", fs.NewErrDoesNotExist(mpfs.File(name))
	}
	return mpfs.JoinCleanFile(name, ff[0].Filename), nil
}

// FormFiles returns the uploaded files under name.
func (mpfs *MultipartFileSystem) FormFiles(name string) (files []fs.File) {
	ff, _ := mpfs.Form.File[name]
	if len(ff) == 0 {
		return nil
	}
	files = make([]fs.File, len(ff))
	for i, f := range ff {
		files[i] = mpfs.JoinCleanFile(name, f.Filename)
	}
	return files
}

func (mpfs *MultipartFileSystem) GetMultipartFileHeader(filePath string) (*multipart.FileHeader, error) {
	parts := mpfs.SplitPath(filePath)
	if len(parts) != 2 {
		return nil, fs.NewErrDoesNotExist(mpfs.File(filePath))
	}
	dir, filename := parts[0], parts[1]
	ff, _ := mpfs.Form.File[dir]
	for _, f := range ff {
		if f.Filename == filename {
			return f, nil
		}
	}
	return nil, fs.NewErrDoesNotExist(mpfs.File(filePath))
}

func (mpfs *MultipartFileSystem) Root() fs.File {
	return fs.File(mpfs.prefix + Separator)
}

func (mpfs *MultipartFileSystem) ID() (string, error) {
	return mpfs.prefix, nil
}

// Prefix for the MultipartFileSystem
func (mpfs *MultipartFileSystem) Prefix() string {
	return mpfs.prefix
}

func (mpfs *MultipartFileSystem) Name() string {
	return "multipart file system " + path.Base(mpfs.prefix)
}

func (mpfs *MultipartFileSystem) String() string {
	return mpfs.Name() + " with prefix " + mpfs.Prefix()
}

func (mpfs *MultipartFileSystem) File(filePath string) fs.File {
	return mpfs.JoinCleanFile(filePath)
}

func (mpfs *MultipartFileSystem) JoinCleanFile(uriParts ...string) fs.File {
	return fs.File(mpfs.prefix + mpfs.JoinCleanPath(uriParts...))
}

func (mpfs *MultipartFileSystem) URL(cleanPath string) string {
	return mpfs.prefix + cleanPath
}

func (mpfs *MultipartFileSystem) JoinCleanPath(uriParts ...string) string {
	return fsimpl.JoinCleanPath(uriParts, mpfs.prefix, Separator)
}

func (mpfs *MultipartFileSystem) SplitPath(filePath string) []string {
	return fsimpl.SplitPath(filePath, mpfs.prefix, Separator)
}

func (*MultipartFileSystem) Separator() string {
	return Separator
}

// MatchAnyPattern returns true if name matches any of patterns,
// or if len(patterns) == 0.
// The match per pattern works like path.Match or filepath.Match
func (*MultipartFileSystem) MatchAnyPattern(name string, patterns []string) (bool, error) {
	return fsimpl.MatchAnyPattern(name, patterns)
}

func (*MultipartFileSystem) DirAndName(filePath string) (dir, name string) {
	return fsimpl.DirAndName(filePath, 0, Separator)
}

func (*MultipartFileSystem) VolumeName(filePath string) string {
	return ""
}

func (mpfs *MultipartFileSystem) IsAbsPath(filePath string) bool {
	return path.IsAbs(filePath)
}

func (mpfs *MultipartFileSystem) AbsPath(filePath string) string {
	if mpfs.IsAbsPath(filePath) {
		return filePath
	}
	return Separator + filePath
}

// Stat returns FileInfo
func (mpfs *MultipartFileSystem) Stat(filePath string) (info fs.FileInfo) {
	parts := mpfs.SplitPath(filePath)
	switch len(parts) {
	case 1:
		dir := parts[0]
		exists := len(mpfs.Form.File[dir]) > 0
		if exists {
			info.Name = dir
			info.Exists = true
			info.IsDir = true
		}
	case 2:
		dir, filename := parts[0], parts[1]
		ff, _ := mpfs.Form.File[dir]
		for _, f := range ff {
			if f.Filename == filename {
				info.Name = filename
				info.Exists = true
				break
			}
		}
	}
	if info.Exists {
		info.IsRegular = true
		info.Size = -1
		// TODO get time from header if exists
		info.ModTime = time.Now()
		info.Permissions = fs.AllRead
	}
	return info
}

func (mpfs *MultipartFileSystem) IsHidden(filePath string) bool {
	name := path.Base(filePath)
	return len(name) > 0 && name[0] == '.'
}

func (mpfs *MultipartFileSystem) IsSymbolicLink(filePath string) bool {
	return false
}

func (mpfs *MultipartFileSystem) ListDirInfo(dirPath string, callback func(fs.File, fs.FileInfo) error, patterns []string) (err error) {
	parts := mpfs.SplitPath(dirPath)
	switch len(parts) {
	case 0:
		for fileDir := range mpfs.Form.File {
			file := mpfs.File(fileDir)
			info := mpfs.Stat(fileDir)
			err = callback(file, info)
			if err != nil {
				return err
			}
		}
	case 1:
		dir := parts[0]
		ff, _ := mpfs.Form.File[dir]
		if len(ff) > 0 {
			for _, f := range ff {
				file := mpfs.JoinCleanFile(dir, f.Filename)
				info := mpfs.Stat(file.Path())
				err = callback(file, info)
				if err != nil {
					return err
				}
			}
		} else {
			err = fs.NewErrDoesNotExist(mpfs.File(dirPath))
		}
	case 2:
		err = fs.NewErrIsNotDirectory(mpfs.File(dirPath))
	default:
		err = fs.NewErrDoesNotExist(mpfs.File(dirPath))
	}
	return err
}

func (mpfs *MultipartFileSystem) ListDirInfoRecursive(dirPath string, callback func(fs.File, fs.FileInfo) error, patterns []string) error {
	return fs.ListDirInfoRecursiveImpl(mpfs, dirPath, callback, patterns)
}

func (mpfs *MultipartFileSystem) ListDirMax(dirPath string, max int, patterns []string) (files []fs.File, err error) {
	return fs.ListDirMaxImpl(max, func(callback func(fs.File) error) error {
		return mpfs.ListDirInfo(dirPath, fs.FileCallback(callback).FileInfoCallback, patterns)
	})
}

func (*MultipartFileSystem) User(filePath string) string {
	return ""
}

func (*MultipartFileSystem) Group(filePath string) string {
	return ""
}

func (mpfs *MultipartFileSystem) ReadAll(filePath string) ([]byte, error) {
	file, err := mpfs.OpenReader(filePath)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	return ioutil.ReadAll(file)
}

func (mpfs *MultipartFileSystem) OpenReader(filePath string) (io.ReadCloser, error) {
	file, err := mpfs.GetMultipartFileHeader(filePath)
	if err != nil {
		return nil, err
	}
	return file.Open()
}
<commit_msg>escape file path in MultipartFileSystem<commit_after>package multipartfs

import (
	"io"
	"io/ioutil"
	"mime/multipart"
	"net/http"
	"net/url"
	"path"
	"time"

	fs "github.com/ungerik/go-fs"
	"github.com/ungerik/go-fs/fsimpl"
)

const (
	// Prefix for the MultipartFileSystem
	Prefix = "multipart://"

	// Separator used in MultipartFileSystem paths
	Separator = "/"
)

// MultipartFileSystem wraps the files in a MIME multipart message as fs.FileSystem
type MultipartFileSystem struct {
	fs.ReadOnlyBase

	prefix string
	Form   *multipart.Form
}

// FromRequestForm returns a MultipartFileSystem from a http.Request
func FromRequestForm(request *http.Request, maxMemory int64) (*MultipartFileSystem, error) {
	err := request.ParseMultipartForm(maxMemory)
	if err != nil {
		return nil, err
	}
	mpfs := &MultipartFileSystem{
		prefix: Prefix + fsimpl.RandomString(),
		Form:   request.MultipartForm,
	}
	fs.Register(mpfs)
	return mpfs, err
}

func (mpfs *MultipartFileSystem) Close() error {
	fs.Unregister(mpfs)
	return mpfs.Form.RemoveAll()
}

// FormFile returns the first file uploaded under name
// or ErrDoesNotExist if there is no file under name.
func (mpfs *MultipartFileSystem) FormFile(name string) (fs.File, error) {
	ff, _ := mpfs.Form.File[name]
	if len(ff) == 0 {
		return "", fs.NewErrDoesNotExist(mpfs.File(name))
	}
	return mpfs.JoinCleanFile(name, ff[0].Filename), nil
}

// FormFiles returns the uploaded files under name.
func (mpfs *MultipartFileSystem) FormFiles(name string) (files []fs.File) {
	ff, _ := mpfs.Form.File[name]
	if len(ff) == 0 {
		return nil
	}
	files = make([]fs.File, len(ff))
	for i, f := range ff {
		files[i] = mpfs.JoinCleanFile(name, f.Filename)
	}
	return files
}

func (mpfs *MultipartFileSystem) GetMultipartFileHeader(filePath string) (*multipart.FileHeader, error) {
	parts := mpfs.SplitPath(filePath)
	if len(parts) != 2 {
		return nil, fs.NewErrDoesNotExist(mpfs.File(filePath))
	}
	dir, filename := parts[0], parts[1]
	ff, _ := mpfs.Form.File[dir]
	for _, f := range ff {
		if f.Filename == filename {
			return f, nil
		}
	}
	return nil, fs.NewErrDoesNotExist(mpfs.File(filePath))
}

func (mpfs *MultipartFileSystem) Root() fs.File {
	return fs.File(mpfs.prefix + Separator)
}

func (mpfs *MultipartFileSystem) ID() (string, error) {
	return mpfs.prefix, nil
}

// Prefix for the MultipartFileSystem
func (mpfs *MultipartFileSystem) Prefix() string {
	return mpfs.prefix
}

func (mpfs *MultipartFileSystem) Name() string {
	return "multipart file system " + path.Base(mpfs.prefix)
}

func (mpfs *MultipartFileSystem) String() string {
	return mpfs.Name() + " with prefix " + mpfs.Prefix()
}

func (mpfs *MultipartFileSystem) File(filePath string) fs.File {
	return mpfs.JoinCleanFile(filePath)
}

func (mpfs *MultipartFileSystem) JoinCleanFile(uriParts ...string) fs.File {
	return fs.File(mpfs.prefix + mpfs.JoinCleanPath(uriParts...))
}

func (mpfs *MultipartFileSystem) URL(cleanPath string) string {
	return mpfs.prefix + cleanPath
}

func (mpfs *MultipartFileSystem) JoinCleanPath(uriParts ...string) string {
	return fsimpl.JoinCleanPath(uriParts, mpfs.prefix, Separator)
}

func (mpfs *MultipartFileSystem) SplitPath(filePath string) []string {
	return fsimpl.SplitPath(filePath, mpfs.prefix, Separator)
}

func (*MultipartFileSystem) Separator() string {
	return Separator
}

// MatchAnyPattern returns true if name matches any of patterns,
// or if len(patterns) == 0.
// The match per pattern works like path.Match or filepath.Match
func (*MultipartFileSystem) MatchAnyPattern(name string, patterns []string) (bool, error) {
	return fsimpl.MatchAnyPattern(name, patterns)
}

func (*MultipartFileSystem) DirAndName(filePath string) (dir, name string) {
	return fsimpl.DirAndName(filePath, 0, Separator)
}

func (*MultipartFileSystem) VolumeName(filePath string) string {
	return ""
}

func (mpfs *MultipartFileSystem) IsAbsPath(filePath string) bool {
	return path.IsAbs(filePath)
}

func (mpfs *MultipartFileSystem) AbsPath(filePath string) string {
	if mpfs.IsAbsPath(filePath) {
		return filePath
	}
	return Separator + filePath
}

// Stat returns FileInfo
func (mpfs *MultipartFileSystem) Stat(filePath string) (info fs.FileInfo) {
	parts := mpfs.SplitPath(filePath)
	switch len(parts) {
	case 1:
		dir := parts[0]
		exists := len(mpfs.Form.File[dir]) > 0
		if exists {
			info.Name = dir
			info.Exists = true
			info.IsDir = true
		}
	case 2:
		dir, filename := parts[0], parts[1]
		ff, _ := mpfs.Form.File[dir]
		for _, f := range ff {
			if f.Filename == filename {
				info.Name = filename
				info.Exists = true
				break
			}
		}
	}
	if info.Exists {
		info.IsRegular = true
		info.Size = -1
		// TODO get time from header if exists
		info.ModTime = time.Now()
		info.Permissions = fs.AllRead
	}
	return info
}

func (mpfs *MultipartFileSystem) IsHidden(filePath string) bool {
	name := path.Base(filePath)
	return len(name) > 0 && name[0] == '.'
}

func (mpfs *MultipartFileSystem) IsSymbolicLink(filePath string) bool {
	return false
}

func (mpfs *MultipartFileSystem) ListDirInfo(dirPath string, callback func(fs.File, fs.FileInfo) error, patterns []string) (err error) {
	parts := mpfs.SplitPath(dirPath)
	switch len(parts) {
	case 0:
		for fileDir := range mpfs.Form.File {
			file := mpfs.File(fileDir)
			info := mpfs.Stat(fileDir)
			err = callback(file, info)
			if err != nil {
				return err
			}
		}
	case 1:
		dir := parts[0]
		ff, _ := mpfs.Form.File[dir]
		if len(ff) > 0 {
			for _, f := range ff {
				file := mpfs.JoinCleanFile(dir, f.Filename)
				info := mpfs.Stat(file.Path())
				err = callback(file, info)
				if err != nil {
					return err
				}
			}
		} else {
			err = fs.NewErrDoesNotExist(mpfs.File(dirPath))
		}
	case 2:
		err = fs.NewErrIsNotDirectory(mpfs.File(dirPath))
	default:
		err = fs.NewErrDoesNotExist(mpfs.File(dirPath))
	}
	return err
}

func (mpfs *MultipartFileSystem) ListDirInfoRecursive(dirPath string, callback func(fs.File, fs.FileInfo) error, patterns []string) error {
	return fs.ListDirInfoRecursiveImpl(mpfs, dirPath, callback, patterns)
}

func (mpfs *MultipartFileSystem) ListDirMax(dirPath string, max int, patterns []string) (files []fs.File, err error) {
	return fs.ListDirMaxImpl(max, func(callback func(fs.File) error) error {
		return mpfs.ListDirInfo(dirPath, fs.FileCallback(callback).FileInfoCallback, patterns)
	})
}

func (*MultipartFileSystem) User(filePath string) string {
	return ""
}

func (*MultipartFileSystem) Group(filePath string) string {
	return ""
}

func (mpfs *MultipartFileSystem) ReadAll(filePath string) ([]byte, error) {
	file, err := mpfs.OpenReader(filePath)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	return ioutil.ReadAll(file)
}

func (mpfs *MultipartFileSystem) OpenReader(filePath string) (io.ReadCloser, error) {
	filePath, err := EscapePath(filePath)
	if err != nil {
		return nil, err
	}
	file, err := mpfs.GetMultipartFileHeader(filePath)
	if err != nil {
		return nil, err
	}
	return file.Open()
}

func EscapePath(filePath string) (string, error) {
	parsedFilePath, err := url.Parse(filePath)
	if err != nil {
		return "", err
	}
	return parsedFilePath.EscapedPath(), nil
}
<|endoftext|>


<commit_before>package glob

import (
	"path/filepath"
	"strings"
)

func Dir(glob string) string {
	glob = filepath.Dir(glob)
	for strings.IndexAny(glob, "*?[") > 0 {
		glob = filepath.Dir(glob)
	}
	return glob
}

func Base(glob string) string {
	base, _ := filepath.Rel(Dir(glob), glob)
	return base
}

func Match(pattern, name string) (bool, error) {

	negative := pattern != "" && pattern[0] == '!'
	if negative {
		pattern = pattern[1:]
	}

	m, err := filepath.Match(pattern, name)
	return m != negative, err
}

type MatchPair struct {
	Glob string
	Name string
}

type pattern struct {
	Glob     string
	Negative bool
}

func Excluded(patterns []pattern, name string) bool {

	for _, pattern := range patterns {
		if !pattern.Negative {
			continue
		}
		if m, _ := filepath.Match(pattern.Glob, name); m {
			return true
		}
	}

	return false
}

func Glob(globs ...string) (<-chan MatchPair, error) {

	//defer close(out)

	patterns := []pattern{}

	for _, glob := range globs {
		negative, err := Match(glob, "")
		if err != nil {
			return nil, err
		}

		if negative {
			glob = glob[1:]
		}

		patterns = append(patterns, pattern{glob, negative})
	}

	matches := make(chan MatchPair)
	go func() {

		seen := make(map[string]struct{})

		defer close(matches)
		for i, pattern := range patterns {

			if pattern.Negative {
				continue
			}
			//Patterns already checked and fs errors are ignored. so no error handling here.
			files, _ := filepath.Glob(pattern.Glob)

			for _, file := range files {
				if _, seen := seen[file]; seen || Excluded(patterns[i:], file) {
					continue
				}

				seen[file] = struct{}{}
				matches <- MatchPair{pattern.Glob, file}
			}
		}
	}()

	return matches, nil
}
<commit_msg>Add recursive globbing support<commit_after>package glob

import (
	"os"
	"path/filepath"
	"strings"
)

func Dir(glob string) string {
	glob = filepath.Dir(glob)
	for strings.IndexAny(glob, "*?[") > 0 {
		glob = filepath.Dir(glob)
	}
	return glob
}

func Base(glob string) string {
	base, _ := filepath.Rel(Dir(glob), glob)
	return base
}

func Match(pattern, name string) (bool, error) {

	negative := pattern != "" && pattern[0] == '!'
	if negative {
		pattern = pattern[1:]
	}

	m, err := filepath.Match(pattern, name)
	return m != negative, err
}

type MatchPair struct {
	Glob string
	Name string
}

type pattern struct {
	Glob     string
	Negative bool
}

func Excluded(patterns []pattern, name string) bool {

	for _, pattern := range patterns {
		if !pattern.Negative {
			continue
		}
		if m, _ := filepath.Match(pattern.Glob, name); m {
			return true
		}
	}

	return false
}

// If the glob contains "/**/" (backslashes on Windows), then a filepath.Walk()
// is performed starting at the directory determined by the path before it,
// and each file name is checked against the path after it.
// Otherwise, a standard filepath.Glob() is used.
func doGlob(glob string) ([]string, error) {
	const recurse = string(filepath.Separator)+"**"+string(filepath.Separator)
	if index := strings.Index(glob, recurse); index != -1 {
		var (
			g       = glob[index+4:]
			results = make([]string, 0)
		)
		if err := filepath.Walk(glob[:index], func(path string, info os.FileInfo, _ error) error {
			m, err := Match(g, info.Name())
			if err != nil {
				return err
			}
			if m {
				results = append(results, path)
			}
			return nil
		}); err != nil {
			return nil, err
		}
		return results, nil
	}
	// Otherwise, standard globbing.
	return filepath.Glob(glob)
}

func Glob(globs ...string) (<-chan MatchPair, error) {

	//defer close(out)

	patterns := []pattern{}

	for _, glob := range globs {
		negative, err := Match(glob, "")
		if err != nil {
			return nil, err
		}

		if negative {
			glob = glob[1:]
		}

		patterns = append(patterns, pattern{glob, negative})
	}

	matches := make(chan MatchPair)
	go func() {

		seen := make(map[string]struct{})

		defer close(matches)
		for i, pattern := range patterns {

			if pattern.Negative {
				continue
			}
			//Patterns already checked and fs errors are ignored. so no error handling here.
			files, _ := doGlob(pattern.Glob)

			for _, file := range files {
				if _, seen := seen[file]; seen || Excluded(patterns[i:], file) {
					continue
				}

				seen[file] = struct{}{}
				matches <- MatchPair{pattern.Glob, file}
			}
		}
	}()

	return matches, nil
}
<|endoftext|>


<commit_before>package goparser

import (
	"go/ast"
	"go/parser"
	"go/token"
	"io"
	"log"
	"os"
	"strings"
)

type ParsedSource struct {
	PackageName   string
	PackageDocs   string
	Types         map[string]Type
	TopLevelFuncs map[string]Func
}

type Func struct {
	Doc  string
	Body string
}

type Type struct {
	Name    string
	Body    string
	Docs    string
	Methods map[string]Func
}

func NewParsedSource() *ParsedSource {
	return &ParsedSource{
		TopLevelFuncs: make(map[string]Func),
		Types:         make(map[string]Type),
	}
}

func Parse(filename string) *ParsedSource {
	var (
		parsedsource = NewParsedSource()
		fset         = token.NewFileSet()
	)

	fi, err := os.Open(filename)
	if err != nil {
		log.Fatal(err)
	}

	f, err := parser.ParseFile(fset, filename, nil, parser.ParseComments)
	if err != nil {
		log.Fatal(err)
	}

	parsedsource.PackageName = f.Name.String()
	parsedsource.PackageDocs = f.Doc.Text()

	ast.Inspect(f, func(n ast.Node) bool {
		switch x := n.(type) {
		case *ast.TypeSpec:
			start := int64(fset.Position(x.Pos()).Offset)
			end := int64(fset.Position(x.Type.End()).Offset)
			name := x.Name.String()
			body := "type " + readCodeBlock(start, end, fi)

			parsedsource.Types[name] = Type{
				Name:    name,
				Docs:    x.Doc.Text(),
				Body:    body,
				Methods: make(map[string]Func),
			}
		case *ast.FuncDecl:
			var (
				fname      = x.Name.String()
				start, end int64
				recv       string
			)

			if x.Recv == nil {
				start = int64(fset.Position(x.Type.Func).Offset)
				end = int64(fset.Position(x.Body.End()).Offset)
				// In this case we are not a method, but a
				// top level function.
				parsedsource.TopLevelFuncs[fname] = Func{
					Doc:  x.Doc.Text(),
					Body: readCodeBlock(start, end, fi),
				}
			} else {
				for _, y := range x.Recv.List {
					start = int64(fset.Position(y.Type.Pos()).Offset)
					end = int64(fset.Position(y.Type.End()).Offset)
					recv = readCodeBlock(start, end, fi)
					recv = strings.Replace(recv, "*", "", -1)
				}

				start = int64(fset.Position(x.Type.Func).Offset)
				end = int64(fset.Position(x.Body.End()).Offset)
				parsedsource.Types[recv].Methods[fname] = Func{
					Doc:  x.Doc.Text(),
					Body: readCodeBlock(start, end, fi),
				}
			}
		}

		return true
	})

	return parsedsource
}

func readCodeBlock(start, end int64, f *os.File) string {
	var (
		delta = end - start
		body  = make([]byte, delta)
		srdr  = io.NewSectionReader(f, start, delta)
	)

	srdr.Read(body)
	return string(body)
}
<commit_msg>Refactor Go parser<commit_after>package goparser

import (
	"go/ast"
	"go/parser"
	"go/token"
	"io"
	"log"
	"os"
	"strings"
)

type ParsedSource struct {
	PackageName   string
	PackageDocs   string
	Types         map[string]Type
	TopLevelFuncs map[string]Func
}

type Func struct {
	Doc  string
	Body string
}

type Type struct {
	Name    string
	Body    string
	Docs    string
	Methods map[string]Func
}

func NewParsedSource() *ParsedSource {
	return &ParsedSource{
		TopLevelFuncs: make(map[string]Func),
		Types:         make(map[string]Type),
	}
}

func Parse(filename string) *ParsedSource {
	var (
		parsedsource = NewParsedSource()
		fset         = token.NewFileSet()
	)

	fi, err := os.Open(filename)
	if err != nil {
		log.Fatal(err)
	}

	f, err := parser.ParseFile(fset, filename, nil, parser.ParseComments)
	if err != nil {
		log.Fatal(err)
	}

	parsedsource.PackageName = f.Name.String()
	parsedsource.PackageDocs = f.Doc.Text()

	ast.Inspect(f, func(n ast.Node) bool {
		switch x := n.(type) {
		case *ast.TypeSpec:
			parseType(parsedsource, x, fset, fi)
		case *ast.FuncDecl:
			parseFunc(parsedsource, x, fset, fi)
		}

		return true
	})

	return parsedsource
}

func parseFunc(parsedsource *ParsedSource, fdecl *ast.FuncDecl, fset *token.FileSet, fi *os.File) {
	var (
		fname      = fdecl.Name.String()
		start, end int64
		recv       string
	)

	if fdecl.Recv == nil {
		start = int64(fset.Position(fdecl.Type.Func).Offset)
		end = int64(fset.Position(fdecl.Body.End()).Offset)
		// In this case we are not a method, but a
		// top level function.
		parsedsource.TopLevelFuncs[fname] = Func{
			Doc:  fdecl.Doc.Text(),
			Body: readCodeBlock(start, end, fi),
		}
	} else {
		for _, y := range fdecl.Recv.List {
			start = int64(fset.Position(y.Type.Pos()).Offset)
			end = int64(fset.Position(y.Type.End()).Offset)
			recv = readCodeBlock(start, end, fi)
			recv = strings.Replace(recv, "*", "", -1)
		}

		start = int64(fset.Position(fdecl.Type.Func).Offset)
		end = int64(fset.Position(fdecl.Body.End()).Offset)
		parsedsource.Types[recv].Methods[fname] = Func{
			Doc:  fdecl.Doc.Text(),
			Body: readCodeBlock(start, end, fi),
		}
	}
}

func parseType(parsedsource *ParsedSource, tspec *ast.TypeSpec, fset *token.FileSet, fi *os.File) {
	start := int64(fset.Position(tspec.Pos()).Offset)
	end := int64(fset.Position(tspec.Type.End()).Offset)
	name := tspec.Name.String()
	body := "type " + readCodeBlock(start, end, fi)

	parsedsource.Types[name] = Type{
		Name:    name,
		Docs:    tspec.Doc.Text(),
		Body:    body,
		Methods: make(map[string]Func),
	}
}

func readCodeBlock(start, end int64, f *os.File) string {
	var (
		delta = end - start
		body  = make([]byte, delta)
		srdr  = io.NewSectionReader(f, start, delta)
	)

	srdr.Read(body)
	return string(body)
}
<|endoftext|>


<commit_before>package util

import (
	"fmt"
	"path/filepath"

	"github.com/inconshreveable/log15"
	"github.com/spacemonkeygo/errors"
	"github.com/spacemonkeygo/errors/try"

	"polydawn.net/repeatr/api/def"
	"polydawn.net/repeatr/rio"
)

// Run inputs
func ProvisionInputs(transmat rio.Transmat, inputs def.InputGroup, journal log15.Logger) map[string]rio.Arena {
	// quick describe what we're going to do
	nInputs := len(inputs)
	journal.Info(fmt.Sprintf("Need %d inputs to be ready", nInputs))

	// start having all filesystems
	// input names are used as keys, so must be unique
	fsGather := make(chan map[string]materializerReport)
	for name, in := range inputs {
		go func(name string, in *def.Input) {
			journal := journal.New(
				"input", name,
				"type", in.Type,
				"hash", in.Hash,
			)
			try.Do(func() {
				journal.Info("Starting materialize")
				// todo: create validity checking api for URIs, check them all before launching anything
				warehouses := make([]rio.SiloURI, len(in.Warehouses))
				for i, wh := range in.Warehouses {
					warehouses[i] = rio.SiloURI(wh)
				}
				// invoke transmat (blocking, potentially long time)
				arena := transmat.Materialize(
					rio.TransmatKind(in.Type),
					rio.CommitID(in.Hash),
					warehouses,
					journal,
				)
				// submit report
				journal.Info("Finished materialize")
				fsGather <- map[string]materializerReport{
					name: {Arena: arena},
				}
			}).Catch(rio.Error, func(err *errors.Error) {
				journal.Warn("Error during materialize",
					"error", err.Message(),
				)
				fsGather <- map[string]materializerReport{
					name: {Err: err},
				}
			}).Done()
		}(name, in)
	}

	// (we don't have any output setup at this point, but if we do in the future, that'll be here.)

	// gather materialized inputs
	// any errors are re-raised immediately (TODO: this currently doesn't fan out smooth cancellations)
	filesystems := make(map[string]rio.Arena, nInputs)
	for range inputs {
		for name, report := range <-fsGather {
			if report.Err != nil {
				panic(report.Err)
			}
			journal.Info(fmt.Sprintf("Input %d/%d ready", len(filesystems)+1, nInputs))
			filesystems[name] = report.Arena
		}
	}

	return filesystems
}

func AssembleFilesystem(
	assemblerFn rio.Assembler,
	rootPath string,
	inputs def.InputGroup,
	inputArenas map[string]rio.Arena,
	hostMounts []def.Mount,
	journal log15.Logger,
) rio.Assembly {
	journal.Info("All inputs acquired... starting assembly")
	// process inputs
	assemblyParts := make([]rio.AssemblyPart, 0, len(inputArenas))
	for name, arena := range inputArenas {
		assemblyParts = append(assemblyParts, rio.AssemblyPart{
			SourcePath: arena.Path(),
			TargetPath: inputs[name].MountPath,
			Writable:   true, // TODO input config should have a word about this
		})
	}
	// process mounts
	for _, mount := range hostMounts {
		assemblyParts = append(assemblyParts, rio.AssemblyPart{
			SourcePath: mount.SourcePath,
			TargetPath: mount.TargetPath,
			Writable:   mount.Writable,
			BareMount:  true,
		})
	}
	// assemmmmmmmmblllle
	assembly := assemblerFn(rootPath, assemblyParts)
	journal.Info("Assembly complete!")
	return assembly
}

type materializerReport struct {
	Arena rio.Arena     // if success
	Err   *errors.Error // subtype of input.Error.  (others are forbidden by contract and treated as fatal.)
}

func ProvisionOutputs(outputs def.OutputGroup, rootfs string, journal log15.Logger) {
	// We no longer make output MountPaths by default.
	// Originally, this seemed like a good idea, because it would be a
	//  consistent stance and allow us to use more complex (e.g. mount-powered)
	//   output shuttling concepts later without any fuss.
	// However, practically speaking, as a default, this has turned out not to work well.
	// Other tools in the world expect clear directories.
	// One example that makes this instantaneously game-over in the real
	//  world is git: `git clone [url] .` *won't work* if there are any other
	//   dirs already existing under `.`; and given how frequently projects
	//    put their build and test outputs in gitignored dirs under their
	//     project versioning root, we very frequently see outputs pointed there.
	//      That creates a lot of noise... so we're dropping the behavior.
}

// Run outputs
func PreserveOutputs(transmat rio.Transmat, outputs def.OutputGroup, rootfs string, journal log15.Logger) def.OutputGroup {
	// run commit on the outputs
	scanGather := make(chan map[string]scanReport)
	for name, out := range outputs {
		go func(name string, out *def.Output) {
			journal := journal.New(
				"output", name,
				"type", out.Type,
			)

			out.Filters = &def.Filters{}
			out.Filters.InitDefaultsOutput()
			filterOptions := rio.ConvertFilterConfig(*out.Filters)
			scanPath := filepath.Join(rootfs, out.MountPath)
			journal.Info("Starting scan")
			try.Do(func() {
				// todo: create validity checking api for URIs, check them all before launching anything
				warehouses := make([]rio.SiloURI, len(out.Warehouses))
				for i, wh := range out.Warehouses {
					warehouses[i] = rio.SiloURI(wh)
				}
				// invoke transmat (blocking, potentially long time)
				commitID := transmat.Scan(
					rio.TransmatKind(out.Type),
					scanPath,
					warehouses,
					journal,
					filterOptions...,
				)
				out.Hash = string(commitID)
				// submit report
				journal.Info("Finished scan")
				scanGather <- map[string]scanReport{
					name: {Output: out},
				}
			}).Catch(rio.Error, func(err *errors.Error) {
				journal.Warn("Error during scan",
					"error", err.Message(),
				)
				scanGather <- map[string]scanReport{
					name: {Err: err},
				}
			}).Done()
		}(name, out)
	}

	// gather reports
	results := def.OutputGroup{}
	for range outputs {
		for name, report := range <-scanGather {
			if report.Err != nil {
				panic(report.Err)
			}
			results[name] = report.Output
		}
	}

	return results
}

type scanReport struct {
	Output *def.Output   // now including the hash
	Err    *errors.Error // subtype of output.Error.  (others are forbidden by contract and treated as fatal.)
}
<commit_msg>Time elapsed during assembly phase.<commit_after>package util

import (
	"fmt"
	"path/filepath"
	"time"

	"github.com/inconshreveable/log15"
	"github.com/spacemonkeygo/errors"
	"github.com/spacemonkeygo/errors/try"

	"polydawn.net/repeatr/api/def"
	"polydawn.net/repeatr/rio"
)

// Run inputs
func ProvisionInputs(transmat rio.Transmat, inputs def.InputGroup, journal log15.Logger) map[string]rio.Arena {
	// quick describe what we're going to do
	nInputs := len(inputs)
	journal.Info(fmt.Sprintf("Need %d inputs to be ready", nInputs))

	// start having all filesystems
	// input names are used as keys, so must be unique
	fsGather := make(chan map[string]materializerReport)
	for name, in := range inputs {
		go func(name string, in *def.Input) {
			journal := journal.New(
				"input", name,
				"type", in.Type,
				"hash", in.Hash,
			)
			try.Do(func() {
				journal.Info("Starting materialize")
				// todo: create validity checking api for URIs, check them all before launching anything
				warehouses := make([]rio.SiloURI, len(in.Warehouses))
				for i, wh := range in.Warehouses {
					warehouses[i] = rio.SiloURI(wh)
				}
				// invoke transmat (blocking, potentially long time)
				arena := transmat.Materialize(
					rio.TransmatKind(in.Type),
					rio.CommitID(in.Hash),
					warehouses,
					journal,
				)
				// submit report
				journal.Info("Finished materialize")
				fsGather <- map[string]materializerReport{
					name: {Arena: arena},
				}
			}).Catch(rio.Error, func(err *errors.Error) {
				journal.Warn("Error during materialize",
					"error", err.Message(),
				)
				fsGather <- map[string]materializerReport{
					name: {Err: err},
				}
			}).Done()
		}(name, in)
	}

	// (we don't have any output setup at this point, but if we do in the future, that'll be here.)

	// gather materialized inputs
	// any errors are re-raised immediately (TODO: this currently doesn't fan out smooth cancellations)
	filesystems := make(map[string]rio.Arena, nInputs)
	for range inputs {
		for name, report := range <-fsGather {
			if report.Err != nil {
				panic(report.Err)
			}
			journal.Info(fmt.Sprintf("Input %d/%d ready", len(filesystems)+1, nInputs))
			filesystems[name] = report.Arena
		}
	}

	return filesystems
}

func AssembleFilesystem(
	assemblerFn rio.Assembler,
	rootPath string,
	inputs def.InputGroup,
	inputArenas map[string]rio.Arena,
	hostMounts []def.Mount,
	journal log15.Logger,
) rio.Assembly {
	started := time.Now()
	journal.Info("All inputs acquired... starting assembly")
	// process inputs
	assemblyParts := make([]rio.AssemblyPart, 0, len(inputArenas))
	for name, arena := range inputArenas {
		assemblyParts = append(assemblyParts, rio.AssemblyPart{
			SourcePath: arena.Path(),
			TargetPath: inputs[name].MountPath,
			Writable:   true, // TODO input config should have a word about this
		})
	}
	// process mounts
	for _, mount := range hostMounts {
		assemblyParts = append(assemblyParts, rio.AssemblyPart{
			SourcePath: mount.SourcePath,
			TargetPath: mount.TargetPath,
			Writable:   mount.Writable,
			BareMount:  true,
		})
	}
	// assemmmmmmmmblllle
	assembly := assemblerFn(rootPath, assemblyParts)
	journal.Info("Assembly complete!",
		"elapsed", time.Now().Sub(started).Seconds(),
	)
	return assembly
}

type materializerReport struct {
	Arena rio.Arena     // if success
	Err   *errors.Error // subtype of input.Error.  (others are forbidden by contract and treated as fatal.)
}

func ProvisionOutputs(outputs def.OutputGroup, rootfs string, journal log15.Logger) {
	// We no longer make output MountPaths by default.
	// Originally, this seemed like a good idea, because it would be a
	//  consistent stance and allow us to use more complex (e.g. mount-powered)
	//   output shuttling concepts later without any fuss.
	// However, practically speaking, as a default, this has turned out not to work well.
	// Other tools in the world expect clear directories.
	// One example that makes this instantaneously game-over in the real
	//  world is git: `git clone [url] .` *won't work* if there are any other
	//   dirs already existing under `.`; and given how frequently projects
	//    put their build and test outputs in gitignored dirs under their
	//     project versioning root, we very frequently see outputs pointed there.
	//      That creates a lot of noise... so we're dropping the behavior.
}

// Run outputs
func PreserveOutputs(transmat rio.Transmat, outputs def.OutputGroup, rootfs string, journal log15.Logger) def.OutputGroup {
	// run commit on the outputs
	scanGather := make(chan map[string]scanReport)
	for name, out := range outputs {
		go func(name string, out *def.Output) {
			journal := journal.New(
				"output", name,
				"type", out.Type,
			)

			out.Filters = &def.Filters{}
			out.Filters.InitDefaultsOutput()
			filterOptions := rio.ConvertFilterConfig(*out.Filters)
			scanPath := filepath.Join(rootfs, out.MountPath)
			journal.Info("Starting scan")
			try.Do(func() {
				// todo: create validity checking api for URIs, check them all before launching anything
				warehouses := make([]rio.SiloURI, len(out.Warehouses))
				for i, wh := range out.Warehouses {
					warehouses[i] = rio.SiloURI(wh)
				}
				// invoke transmat (blocking, potentially long time)
				commitID := transmat.Scan(
					rio.TransmatKind(out.Type),
					scanPath,
					warehouses,
					journal,
					filterOptions...,
				)
				out.Hash = string(commitID)
				// submit report
				journal.Info("Finished scan")
				scanGather <- map[string]scanReport{
					name: {Output: out},
				}
			}).Catch(rio.Error, func(err *errors.Error) {
				journal.Warn("Error during scan",
					"error", err.Message(),
				)
				scanGather <- map[string]scanReport{
					name: {Err: err},
				}
			}).Done()
		}(name, out)
	}

	// gather reports
	results := def.OutputGroup{}
	for range outputs {
		for name, report := range <-scanGather {
			if report.Err != nil {
				panic(report.Err)
			}
			results[name] = report.Output
		}
	}

	return results
}

type scanReport struct {
	Output *def.Output   // now including the hash
	Err    *errors.Error // subtype of output.Error.  (others are forbidden by contract and treated as fatal.)
}
<|endoftext|>


<commit_before>package container

import (
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"runtime"
	"sort"
	"strings"
	"testing"

	"github.com/docker/cli/cli/config/configfile"
	"github.com/docker/cli/internal/test"
	"github.com/docker/cli/internal/test/notary"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/container"
	"github.com/docker/docker/api/types/network"
	"github.com/google/go-cmp/cmp"
	"github.com/pkg/errors"
	"gotest.tools/assert"
	is "gotest.tools/assert/cmp"
	"gotest.tools/fs"
	"gotest.tools/golden"
)

func TestCIDFileNoOPWithNoFilename(t *testing.T) {
	file, err := newCIDFile("")
	assert.NilError(t, err)
	assert.DeepEqual(t, &cidFile{}, file, cmp.AllowUnexported(cidFile{}))

	assert.NilError(t, file.Write("id"))
	assert.NilError(t, file.Close())
}

func TestNewCIDFileWhenFileAlreadyExists(t *testing.T) {
	tempfile := fs.NewFile(t, "test-cid-file")
	defer tempfile.Remove()

	_, err := newCIDFile(tempfile.Path())
	assert.ErrorContains(t, err, "Container ID file found")
}

func TestCIDFileCloseWithNoWrite(t *testing.T) {
	tempdir := fs.NewDir(t, "test-cid-file")
	defer tempdir.Remove()

	path := tempdir.Join("cidfile")
	file, err := newCIDFile(path)
	assert.NilError(t, err)
	assert.Check(t, is.Equal(file.path, path))

	assert.NilError(t, file.Close())
	_, err = os.Stat(path)
	assert.Check(t, os.IsNotExist(err))
}

func TestCIDFileCloseWithWrite(t *testing.T) {
	tempdir := fs.NewDir(t, "test-cid-file")
	defer tempdir.Remove()

	path := tempdir.Join("cidfile")
	file, err := newCIDFile(path)
	assert.NilError(t, err)

	content := "id"
	assert.NilError(t, file.Write(content))

	actual, err := ioutil.ReadFile(path)
	assert.NilError(t, err)
	assert.Check(t, is.Equal(content, string(actual)))

	assert.NilError(t, file.Close())
	_, err = os.Stat(path)
	assert.NilError(t, err)
}

func TestCreateContainerPullsImageIfMissing(t *testing.T) {
	imageName := "does-not-exist-locally"
	responseCounter := 0
	containerID := "abcdef"

	client := &fakeClient{
		createContainerFunc: func(
			config *container.Config,
			hostConfig *container.HostConfig,
			networkingConfig *network.NetworkingConfig,
			containerName string,
		) (container.ContainerCreateCreatedBody, error) {
			defer func() { responseCounter++ }()
			switch responseCounter {
			case 0:
				return container.ContainerCreateCreatedBody{}, fakeNotFound{}
			case 1:
				return container.ContainerCreateCreatedBody{ID: containerID}, nil
			default:
				return container.ContainerCreateCreatedBody{}, errors.New("unexpected")
			}
		},
		imageCreateFunc: func(parentReference string, options types.ImageCreateOptions) (io.ReadCloser, error) {
			return ioutil.NopCloser(strings.NewReader("")), nil
		},
		infoFunc: func() (types.Info, error) {
			return types.Info{IndexServerAddress: "http://indexserver"}, nil
		},
	}
	cli := test.NewFakeCli(client)
	config := &containerConfig{
		Config: &container.Config{
			Image: imageName,
		},
		HostConfig: &container.HostConfig{},
	}
	body, err := createContainer(context.Background(), cli, config, &createOptions{
		name:      "name",
		platform:  runtime.GOOS,
		untrusted: true,
		pull:      PullImageMissing,
	})
	assert.NilError(t, err)
	expected := container.ContainerCreateCreatedBody{ID: containerID}
	assert.Check(t, is.DeepEqual(expected, *body))
	stderr := cli.ErrBuffer().String()
	assert.Check(t, is.Contains(stderr, "Unable to find image 'does-not-exist-locally:latest' locally"))
}

func TestCreateContainerNeverPullsImage(t *testing.T) {
	imageName := "does-not-exist-locally"
	responseCounter := 0
	pullCounter := 0

	client := &fakeClient{
		createContainerFunc: func(
			config *container.Config,
			hostConfig *container.HostConfig,
			networkingConfig *network.NetworkingConfig,
			containerName string,
		) (container.ContainerCreateCreatedBody, error) {
			defer func() { responseCounter++ }()
			switch responseCounter {
			case 0:
				return container.ContainerCreateCreatedBody{}, fakeNotFound{}
			default:
				return container.ContainerCreateCreatedBody{}, errors.New("unexpected")
			}
		},
		imageCreateFunc: func(parentReference string, options types.ImageCreateOptions) (io.ReadCloser, error) {
			defer func() { pullCounter++ }()
			switch pullCounter {
			case 0:
				return ioutil.NopCloser(strings.NewReader("")), nil
			default:
				return nil, errors.New("unexpected pull")
			}
		},
		infoFunc: func() (types.Info, error) {
			return types.Info{IndexServerAddress: "http://indexserver"}, nil
		},
	}
	cli := test.NewFakeCli(client)
	config := &containerConfig{
		Config: &container.Config{
			Image: imageName,
		},
		HostConfig: &container.HostConfig{},
	}
	_, err := createContainer(context.Background(), cli, config, &createOptions{
		name:      "name",
		platform:  runtime.GOOS,
		untrusted: true,
		pull:      PullImageNever,
	})
	assert.ErrorContains(t, err, "fake not found")
}

func TestCreateContainerAlwaysPullsImage(t *testing.T) {
	imageName := "does-not-exist-locally"
	pullTries := 7
	responseCounter := 0
	pullCounter := 0
	containerID := "abcdef"

	client := &fakeClient{
		createContainerFunc: func(
			config *container.Config,
			hostConfig *container.HostConfig,
			networkingConfig *network.NetworkingConfig,
			containerName string,
		) (container.ContainerCreateCreatedBody, error) {
			defer func() { responseCounter++ }()
			switch responseCounter {
			default:
				return container.ContainerCreateCreatedBody{ID: containerID}, nil
			}
		},
		imageCreateFunc: func(parentReference string, options types.ImageCreateOptions) (io.ReadCloser, error) {
			defer func() { pullCounter++ }()
			return ioutil.NopCloser(strings.NewReader("")), nil
		},
		infoFunc: func() (types.Info, error) {
			return types.Info{IndexServerAddress: "http://indexserver"}, nil
		},
	}
	cli := test.NewFakeCli(client)
	config := &containerConfig{
		Config: &container.Config{
			Image: imageName,
		},
		HostConfig: &container.HostConfig{},
	}
	for i := 0; i < pullTries; i++ {
		body, err := createContainer(context.Background(), cli, config, &createOptions{
			name:      "name",
			platform:  runtime.GOOS,
			untrusted: true,
			pull:      PullImageAlways,
		})
		assert.NilError(t, err)
		expected := container.ContainerCreateCreatedBody{ID: containerID}
		assert.Check(t, is.DeepEqual(expected, *body))
	}

	assert.Check(t, is.Equal(responseCounter, pullCounter))
}

func TestNewCreateCommandWithContentTrustErrors(t *testing.T) {
	testCases := []struct {
		name          string
		args          []string
		expectedError string
		notaryFunc    test.NotaryClientFuncType
	}{
		{
			name:          "offline-notary-server",
			notaryFunc:    notary.GetOfflineNotaryRepository,
			expectedError: "client is offline",
			args:          []string{"image:tag"},
		},
		{
			name:          "uninitialized-notary-server",
			notaryFunc:    notary.GetUninitializedNotaryRepository,
			expectedError: "remote trust data does not exist",
			args:          []string{"image:tag"},
		},
		{
			name:          "empty-notary-server",
			notaryFunc:    notary.GetEmptyTargetsNotaryRepository,
			expectedError: "No valid trust data for tag",
			args:          []string{"image:tag"},
		},
	}
	for _, tc := range testCases {
		cli := test.NewFakeCli(&fakeClient{
			createContainerFunc: func(config *container.Config,
				hostConfig *container.HostConfig,
				networkingConfig *network.NetworkingConfig,
				containerName string,
			) (container.ContainerCreateCreatedBody, error) {
				return container.ContainerCreateCreatedBody{}, fmt.Errorf("shouldn't try to pull image")
			},
		}, test.EnableContentTrust)
		cli.SetNotaryClient(tc.notaryFunc)
		cmd := NewCreateCommand(cli)
		cmd.SetOutput(ioutil.Discard)
		cmd.SetArgs(tc.args)
		err := cmd.Execute()
		assert.ErrorContains(t, err, tc.expectedError)
	}
}

func TestNewCreateCommandWithWarnings(t *testing.T) {
	testCases := []struct {
		name    string
		args    []string
		warning bool
	}{
		{
			name: "container-create-without-oom-kill-disable",
			args: []string{"image:tag"},
		},
		{
			name: "container-create-oom-kill-disable-false",
			args: []string{"--oom-kill-disable=false", "image:tag"},
		},
		{
			name:    "container-create-oom-kill-without-memory-limit",
			args:    []string{"--oom-kill-disable", "image:tag"},
			warning: true,
		},
		{
			name:    "container-create-oom-kill-true-without-memory-limit",
			args:    []string{"--oom-kill-disable=true", "image:tag"},
			warning: true,
		},
		{
			name: "container-create-oom-kill-true-with-memory-limit",
			args: []string{"--oom-kill-disable=true", "--memory=100M", "image:tag"},
		},
		{
			name:    "container-create-localhost-dns",
			args:    []string{"--dns=127.0.0.11", "image:tag"},
			warning: true,
		},
		{
			name:    "container-create-localhost-dns-ipv6",
			args:    []string{"--dns=::1", "image:tag"},
			warning: true,
		},
	}
	for _, tc := range testCases {
		t.Run(tc.name, func(t *testing.T) {
			cli := test.NewFakeCli(&fakeClient{
				createContainerFunc: func(config *container.Config,
					hostConfig *container.HostConfig,
					networkingConfig *network.NetworkingConfig,
					containerName string,
				) (container.ContainerCreateCreatedBody, error) {
					return container.ContainerCreateCreatedBody{}, nil
				},
			})
			cmd := NewCreateCommand(cli)
			cmd.SetOutput(ioutil.Discard)
			cmd.SetArgs(tc.args)
			err := cmd.Execute()
			assert.NilError(t, err)
			if tc.warning {
				golden.Assert(t, cli.ErrBuffer().String(), tc.name+".golden")
			} else {
				assert.Equal(t, cli.ErrBuffer().String(), "")
			}
		})
	}
}

func TestCreateContainerWithProxyConfig(t *testing.T) {
	expected := []string{
		"HTTP_PROXY=httpProxy",
		"http_proxy=httpProxy",
		"HTTPS_PROXY=httpsProxy",
		"https_proxy=httpsProxy",
		"NO_PROXY=noProxy",
		"no_proxy=noProxy",
		"FTP_PROXY=ftpProxy",
		"ftp_proxy=ftpProxy",
	}
	sort.Strings(expected)

	cli := test.NewFakeCli(&fakeClient{
		createContainerFunc: func(config *container.Config,
			hostConfig *container.HostConfig,
			networkingConfig *network.NetworkingConfig,
			containerName string,
		) (container.ContainerCreateCreatedBody, error) {
			sort.Strings(config.Env)
			assert.DeepEqual(t, config.Env, expected)
			return container.ContainerCreateCreatedBody{}, nil
		},
	})
	cli.SetConfigFile(&configfile.ConfigFile{
		Proxies: map[string]configfile.ProxyConfig{
			"default": {
				HTTPProxy:  "httpProxy",
				HTTPSProxy: "httpsProxy",
				NoProxy:    "noProxy",
				FTPProxy:   "ftpProxy",
			},
		},
	})
	cmd := NewCreateCommand(cli)
	cmd.SetOutput(ioutil.Discard)
	cmd.SetArgs([]string{"image:tag"})
	err := cmd.Execute()
	assert.NilError(t, err)
}

type fakeNotFound struct{}

func (f fakeNotFound) NotFound() bool { return true }
func (f fakeNotFound) Error() string  { return "error fake not found" }
<commit_msg>Use single table for all ContainerImagePullPolicy tests<commit_after>package container

import (
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"runtime"
	"sort"
	"strings"
	"testing"

	"github.com/docker/cli/cli/config/configfile"
	"github.com/docker/cli/internal/test"
	"github.com/docker/cli/internal/test/notary"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/container"
	"github.com/docker/docker/api/types/network"
	"github.com/google/go-cmp/cmp"
	"gotest.tools/assert"
	is "gotest.tools/assert/cmp"
	"gotest.tools/fs"
	"gotest.tools/golden"
)

func TestCIDFileNoOPWithNoFilename(t *testing.T) {
	file, err := newCIDFile("")
	assert.NilError(t, err)
	assert.DeepEqual(t, &cidFile{}, file, cmp.AllowUnexported(cidFile{}))

	assert.NilError(t, file.Write("id"))
	assert.NilError(t, file.Close())
}

func TestNewCIDFileWhenFileAlreadyExists(t *testing.T) {
	tempfile := fs.NewFile(t, "test-cid-file")
	defer tempfile.Remove()

	_, err := newCIDFile(tempfile.Path())
	assert.ErrorContains(t, err, "Container ID file found")
}

func TestCIDFileCloseWithNoWrite(t *testing.T) {
	tempdir := fs.NewDir(t, "test-cid-file")
	defer tempdir.Remove()

	path := tempdir.Join("cidfile")
	file, err := newCIDFile(path)
	assert.NilError(t, err)
	assert.Check(t, is.Equal(file.path, path))

	assert.NilError(t, file.Close())
	_, err = os.Stat(path)
	assert.Check(t, os.IsNotExist(err))
}

func TestCIDFileCloseWithWrite(t *testing.T) {
	tempdir := fs.NewDir(t, "test-cid-file")
	defer tempdir.Remove()

	path := tempdir.Join("cidfile")
	file, err := newCIDFile(path)
	assert.NilError(t, err)

	content := "id"
	assert.NilError(t, file.Write(content))

	actual, err := ioutil.ReadFile(path)
	assert.NilError(t, err)
	assert.Check(t, is.Equal(content, string(actual)))

	assert.NilError(t, file.Close())
	_, err = os.Stat(path)
	assert.NilError(t, err)
}

func TestCreateContainerImagePullPolicy(t *testing.T) {
	imageName := "does-not-exist-locally"
	containerID := "abcdef"
	config := &containerConfig{
		Config: &container.Config{
			Image: imageName,
		},
		HostConfig: &container.HostConfig{},
	}

	cases := []struct {
		PullPolicy      string
		ExpectedPulls   int
		ExpectedBody    container.ContainerCreateCreatedBody
		ExpectedErrMsg  string
		ResponseCounter int
	}{
		{
			PullPolicy:    PullImageMissing,
			ExpectedPulls: 1,
			ExpectedBody:  container.ContainerCreateCreatedBody{ID: containerID},
		}, {
			PullPolicy:      PullImageAlways,
			ExpectedPulls:   1,
			ExpectedBody:    container.ContainerCreateCreatedBody{ID: containerID},
			ResponseCounter: 1, // This lets us return a container on the first pull
		}, {
			PullPolicy:     PullImageNever,
			ExpectedPulls:  0,
			ExpectedErrMsg: "error fake not found",
		},
	}
	for _, c := range cases {
		pullCounter := 0

		client := &fakeClient{
			createContainerFunc: func(
				config *container.Config,
				hostConfig *container.HostConfig,
				networkingConfig *network.NetworkingConfig,
				containerName string,
			) (container.ContainerCreateCreatedBody, error) {
				defer func() { c.ResponseCounter++ }()
				switch c.ResponseCounter {
				case 0:
					return container.ContainerCreateCreatedBody{}, fakeNotFound{}
				default:
					return container.ContainerCreateCreatedBody{ID: containerID}, nil
				}
			},
			imageCreateFunc: func(parentReference string, options types.ImageCreateOptions) (io.ReadCloser, error) {
				defer func() { pullCounter++ }()
				return ioutil.NopCloser(strings.NewReader("")), nil
			},
			infoFunc: func() (types.Info, error) {
				return types.Info{IndexServerAddress: "http://indexserver"}, nil
			},
		}
		cli := test.NewFakeCli(client)
		body, err := createContainer(context.Background(), cli, config, &createOptions{
			name:      "name",
			platform:  runtime.GOOS,
			untrusted: true,
			pull:      c.PullPolicy,
		})

		if c.ExpectedErrMsg != "" {
			assert.ErrorContains(t, err, c.ExpectedErrMsg)
		} else {
			assert.NilError(t, err)
			assert.Check(t, is.DeepEqual(c.ExpectedBody, *body))
		}

		assert.Check(t, is.Equal(c.ExpectedPulls, pullCounter))
	}
}
func TestNewCreateCommandWithContentTrustErrors(t *testing.T) {
	testCases := []struct {
		name          string
		args          []string
		expectedError string
		notaryFunc    test.NotaryClientFuncType
	}{
		{
			name:          "offline-notary-server",
			notaryFunc:    notary.GetOfflineNotaryRepository,
			expectedError: "client is offline",
			args:          []string{"image:tag"},
		},
		{
			name:          "uninitialized-notary-server",
			notaryFunc:    notary.GetUninitializedNotaryRepository,
			expectedError: "remote trust data does not exist",
			args:          []string{"image:tag"},
		},
		{
			name:          "empty-notary-server",
			notaryFunc:    notary.GetEmptyTargetsNotaryRepository,
			expectedError: "No valid trust data for tag",
			args:          []string{"image:tag"},
		},
	}
	for _, tc := range testCases {
		cli := test.NewFakeCli(&fakeClient{
			createContainerFunc: func(config *container.Config,
				hostConfig *container.HostConfig,
				networkingConfig *network.NetworkingConfig,
				containerName string,
			) (container.ContainerCreateCreatedBody, error) {
				return container.ContainerCreateCreatedBody{}, fmt.Errorf("shouldn't try to pull image")
			},
		}, test.EnableContentTrust)
		cli.SetNotaryClient(tc.notaryFunc)
		cmd := NewCreateCommand(cli)
		cmd.SetOutput(ioutil.Discard)
		cmd.SetArgs(tc.args)
		err := cmd.Execute()
		assert.ErrorContains(t, err, tc.expectedError)
	}
}

func TestNewCreateCommandWithWarnings(t *testing.T) {
	testCases := []struct {
		name    string
		args    []string
		warning bool
	}{
		{
			name: "container-create-without-oom-kill-disable",
			args: []string{"image:tag"},
		},
		{
			name: "container-create-oom-kill-disable-false",
			args: []string{"--oom-kill-disable=false", "image:tag"},
		},
		{
			name:    "container-create-oom-kill-without-memory-limit",
			args:    []string{"--oom-kill-disable", "image:tag"},
			warning: true,
		},
		{
			name:    "container-create-oom-kill-true-without-memory-limit",
			args:    []string{"--oom-kill-disable=true", "image:tag"},
			warning: true,
		},
		{
			name: "container-create-oom-kill-true-with-memory-limit",
			args: []string{"--oom-kill-disable=true", "--memory=100M", "image:tag"},
		},
		{
			name:    "container-create-localhost-dns",
			args:    []string{"--dns=127.0.0.11", "image:tag"},
			warning: true,
		},
		{
			name:    "container-create-localhost-dns-ipv6",
			args:    []string{"--dns=::1", "image:tag"},
			warning: true,
		},
	}
	for _, tc := range testCases {
		t.Run(tc.name, func(t *testing.T) {
			cli := test.NewFakeCli(&fakeClient{
				createContainerFunc: func(config *container.Config,
					hostConfig *container.HostConfig,
					networkingConfig *network.NetworkingConfig,
					containerName string,
				) (container.ContainerCreateCreatedBody, error) {
					return container.ContainerCreateCreatedBody{}, nil
				},
			})
			cmd := NewCreateCommand(cli)
			cmd.SetOutput(ioutil.Discard)
			cmd.SetArgs(tc.args)
			err := cmd.Execute()
			assert.NilError(t, err)
			if tc.warning {
				golden.Assert(t, cli.ErrBuffer().String(), tc.name+".golden")
			} else {
				assert.Equal(t, cli.ErrBuffer().String(), "")
			}
		})
	}
}

func TestCreateContainerWithProxyConfig(t *testing.T) {
	expected := []string{
		"HTTP_PROXY=httpProxy",
		"http_proxy=httpProxy",
		"HTTPS_PROXY=httpsProxy",
		"https_proxy=httpsProxy",
		"NO_PROXY=noProxy",
		"no_proxy=noProxy",
		"FTP_PROXY=ftpProxy",
		"ftp_proxy=ftpProxy",
	}
	sort.Strings(expected)

	cli := test.NewFakeCli(&fakeClient{
		createContainerFunc: func(config *container.Config,
			hostConfig *container.HostConfig,
			networkingConfig *network.NetworkingConfig,
			containerName string,
		) (container.ContainerCreateCreatedBody, error) {
			sort.Strings(config.Env)
			assert.DeepEqual(t, config.Env, expected)
			return container.ContainerCreateCreatedBody{}, nil
		},
	})
	cli.SetConfigFile(&configfile.ConfigFile{
		Proxies: map[string]configfile.ProxyConfig{
			"default": {
				HTTPProxy:  "httpProxy",
				HTTPSProxy: "httpsProxy",
				NoProxy:    "noProxy",
				FTPProxy:   "ftpProxy",
			},
		},
	})
	cmd := NewCreateCommand(cli)
	cmd.SetOutput(ioutil.Discard)
	cmd.SetArgs([]string{"image:tag"})
	err := cmd.Execute()
	assert.NilError(t, err)
}

type fakeNotFound struct{}

func (f fakeNotFound) NotFound() bool { return true }
func (f fakeNotFound) Error() string  { return "error fake not found" }
<|endoftext|>


<commit_before>package torrent

import (
	"time"

	"github.com/cenkalti/rain/internal/piecewriter"
	"github.com/cenkalti/rain/internal/urldownloader"
	"github.com/cenkalti/rain/internal/webseedsource"
)

func (t *torrent) handleWebseedPieceResult(msg *urldownloader.PieceResult) {
	if msg.Error != nil {
		t.log.Errorln("webseed download error:", msg.Error)
		// Possible causes:
		// * Client.Do error
		// * Unexpected status code
		// * Response.Body.Read error
		t.disableSource(msg.Downloader.URL, msg.Error, true)
		return
	}

	piece := &t.pieces[msg.Index]

	t.counters.Incr(counterBytesDownloaded, int64(len(msg.Buffer.Data)))
	t.downloadSpeed.Update(int64(len(msg.Buffer.Data)))
	for _, src := range t.webseedSources {
		if src.URL != msg.Downloader.URL {
			continue
		}
		src.UpdateSpeed(len(msg.Buffer.Data))
		break
	}

	if piece.Writing {
		panic("piece is already writing")
	}
	piece.Writing = true

	// Prevent receiving piece messages to avoid more than 1 write per torrent.
	t.pieceMessagesC.Suspend()
	t.webseedPieceResultC.Suspend()

	pw := piecewriter.New(piece, msg.Downloader, msg.Buffer)
	go pw.Run(t.pieceWriterResultC, t.doneC)

	if msg.Done {
		for _, src := range t.webseedSources {
			if src.URL != msg.Downloader.URL {
				continue
			}
			t.closeWebseedDownloader(src)
			t.startPieceDownloaderForWebseed(src)
			break
		}
	}
}

func (t *torrent) disableSource(srcurl string, err error, retry bool) {
	for _, src := range t.webseedSources {
		if src.URL != srcurl {
			continue
		}
		src.Disabled = true
		src.DisabledAt = time.Now()
		src.LastError = err
		t.closeWebseedDownloader(src)
		if retry {
			go t.notifyWebseedRetry(src)
		}
		break
	}
}

func (t *torrent) notifyWebseedRetry(src *webseedsource.WebseedSource) {
	select {
	case <-time.After(time.Minute):
		select {
		case t.webseedRetryC <- src:
		case <-t.closeC:
		}
	case <-t.closeC:
	}
}
<commit_msg>webseed download errors are debug level logs<commit_after>package torrent

import (
	"time"

	"github.com/cenkalti/rain/internal/piecewriter"
	"github.com/cenkalti/rain/internal/urldownloader"
	"github.com/cenkalti/rain/internal/webseedsource"
)

func (t *torrent) handleWebseedPieceResult(msg *urldownloader.PieceResult) {
	if msg.Error != nil {
		t.log.Debugln("webseed download error:", msg.Error)
		// Possible causes:
		// * Client.Do error
		// * Unexpected status code
		// * Response.Body.Read error
		t.disableSource(msg.Downloader.URL, msg.Error, true)
		return
	}

	piece := &t.pieces[msg.Index]

	t.counters.Incr(counterBytesDownloaded, int64(len(msg.Buffer.Data)))
	t.downloadSpeed.Update(int64(len(msg.Buffer.Data)))
	for _, src := range t.webseedSources {
		if src.URL != msg.Downloader.URL {
			continue
		}
		src.UpdateSpeed(len(msg.Buffer.Data))
		break
	}

	if piece.Writing {
		panic("piece is already writing")
	}
	piece.Writing = true

	// Prevent receiving piece messages to avoid more than 1 write per torrent.
	t.pieceMessagesC.Suspend()
	t.webseedPieceResultC.Suspend()

	pw := piecewriter.New(piece, msg.Downloader, msg.Buffer)
	go pw.Run(t.pieceWriterResultC, t.doneC)

	if msg.Done {
		for _, src := range t.webseedSources {
			if src.URL != msg.Downloader.URL {
				continue
			}
			t.closeWebseedDownloader(src)
			t.startPieceDownloaderForWebseed(src)
			break
		}
	}
}

func (t *torrent) disableSource(srcurl string, err error, retry bool) {
	for _, src := range t.webseedSources {
		if src.URL != srcurl {
			continue
		}
		src.Disabled = true
		src.DisabledAt = time.Now()
		src.LastError = err
		t.closeWebseedDownloader(src)
		if retry {
			go t.notifyWebseedRetry(src)
		}
		break
	}
}

func (t *torrent) notifyWebseedRetry(src *webseedsource.WebseedSource) {
	select {
	case <-time.After(time.Minute):
		select {
		case t.webseedRetryC <- src:
		case <-t.closeC:
		}
	case <-t.closeC:
	}
}
<|endoftext|>


<commit_before>package server

import (
	"encoding/json"
	"fmt"

	"github.com/robfig/cron"
	log "github.com/sirupsen/logrus"
	"github.com/vkuznet/transfer2go/core"
	"github.com/vkuznet/transfer2go/utils"
)

type Router struct {
	CronInterval string `json:"interval"` // Helps to set hourly based cron job
}

// AgentRouter helps to call router's methods
var AgentRouter Router

// newRouter returns new instance of Router type
func newRouter(interval string) *cron.Cron {
	timeConfig := "@every " + interval // It works on this format - http://golang.org/pkg/time/#ParseDuration
	AgentRouter = Router{CronInterval: interval}
	c := cron.New()
	c.AddFunc(timeConfig, train)
	c.Start()
	return c
}

// Function to train the agent
func train() {
	var dataPoints []core.TransferData
	for _, source := range _agents {
		data, err := getHistory(source)
		if err == nil {
			dataPoints = append(dataPoints, data...)
		}
	}
}

// function to get historical data of agent
func getHistory(source string) ([]core.TransferData, error) {
	url := fmt.Sprintf("%s/history?duration=%s", source, AgentRouter.CronInterval)
	resp := utils.FetchResponse(url, []byte{})
	if resp.Error != nil {
		return nil, resp.Error
	}
	var transferRecords []core.TransferData
	err := json.Unmarshal(resp.Data, &transferRecords)
	if err != nil {
		return nil, err
	}
	return transferRecords, nil
}

// TODO: Function to iterate through all the connected agents

// TODO: Function to get the historical data

// TODO: Function to train the model

// TODO: Function to do the cron job
<commit_msg>Remove log package<commit_after>package server

import (
	"encoding/json"
	"fmt"

	"github.com/robfig/cron"
	"github.com/vkuznet/transfer2go/core"
	"github.com/vkuznet/transfer2go/utils"
)

type Router struct {
	CronInterval string `json:"interval"` // Helps to set hourly based cron job
}

// AgentRouter helps to call router's methods
var AgentRouter Router

// newRouter returns new instance of Router type
func newRouter(interval string) *cron.Cron {
	timeConfig := "@every " + interval // It works on this format - http://golang.org/pkg/time/#ParseDuration
	AgentRouter = Router{CronInterval: interval}
	c := cron.New()
	c.AddFunc(timeConfig, train)
	c.Start()
	return c
}

// Function to train the agent
func train() {
	var dataPoints []core.TransferData
	for _, source := range _agents {
		data, err := getHistory(source)
		if err == nil {
			dataPoints = append(dataPoints, data...)
		}
	}
}

// function to get historical data of agent
func getHistory(source string) ([]core.TransferData, error) {
	url := fmt.Sprintf("%s/history?duration=%s", source, AgentRouter.CronInterval)
	resp := utils.FetchResponse(url, []byte{})
	if resp.Error != nil {
		return nil, resp.Error
	}
	var transferRecords []core.TransferData
	err := json.Unmarshal(resp.Data, &transferRecords)
	if err != nil {
		return nil, err
	}
	return transferRecords, nil
}

// TODO: Function to iterate through all the connected agents

// TODO: Function to get the historical data

// TODO: Function to train the model

// TODO: Function to do the cron job
<|endoftext|>


<commit_before>package dilbert

import (
	"fmt"
	"path/filepath"
	"strings"
	"time"

	"github.com/PuerkitoBio/goquery"
	log "github.com/Sirupsen/logrus"
	"github.com/blacktop/scifgif/elasticsearch"
	"github.com/jpillora/backoff"
)

var attempt int

// MaxAttempts max number of download attempts
const MaxAttempts = 20

// Comic is the dilbert comic strip meta data
type Comic struct {
	Title      string
	Tags       []string
	ImageURL   string
	Transcript string
}

// GetComicMetaData gets all the comic strips meta data
func GetComicMetaData(url, date string, b *backoff.Backoff) Comic {
	comic := Comic{}

	if attempt > MaxAttempts {
		return comic
	}

	doc, err := goquery.NewDocument(url)
	if err != nil {
		log.Error(err)
		// backoff and try again
		backoff := b.Duration()
		log.WithFields(log.Fields{"backoff": backoff}).Info("waiting to try to again")
		time.Sleep(backoff)
		// retry url meta data parse
		attempt++
		GetComicMetaData(url, date, b)
	}
	// GET TITLE
	comic.Title = doc.Find(".comic-title-name").Text()
	// GET IMAGE URL
	doc.Find(".img-comic-container").Each(func(i int, s *goquery.Selection) {
		comic.ImageURL, _ = s.Find("img").Attr("src")
	})
	// GET TAGS
	doc.Find(".comic-tags").Each(func(i int, s *goquery.Selection) {
		s.Find("a").Each(func(i int, a *goquery.Selection) {
			comic.Tags = append(comic.Tags, strings.TrimPrefix(a.Text(), "#"))
		})
	})
	// GET TRANSCRIPT
	id := "js-toggle-transcript-" + date
	transcripts := doc.Find("div#" + id).Text()
	// clean up string
	transcripts = strings.TrimSpace(transcripts)
	transcripts = strings.TrimPrefix(transcripts, "Transcript")
	transcripts = strings.TrimSpace(transcripts)
	comic.Transcript = transcripts

	return comic
}

// GetAllDilbert havest all teh comics strips
func GetAllDilbert(folder string, date string) error {
	count := 0
	attempt = 0
	b := &backoff.Backoff{
		//These are the defaults
		Min:    100 * time.Millisecond,
		Max:    600 * time.Second,
		Factor: 2,
		Jitter: true,
	}

	if len(date) < 1 {
		// date = "1989-04-17"
		date = "2017-09-08"
	}
	start, _ := time.Parse("2006-01-02", date)

	for d := start; time.Now().After(d); d = d.AddDate(0, 0, 1) {
		date := fmt.Sprintf("%04d-%02d-%02d", d.Year(), d.Month(), d.Day())
		url := "http://dilbert.com/strip/" + date
		comic := GetComicMetaData(url, date, b)
		// download image
		log.WithFields(log.Fields{
			"id":    date,
			"title": comic.Title,
		}).Debug("downloading file")

		filepath := filepath.Join(folder, date+".jpg")
		go elasticsearch.DownloadImage(comic.ImageURL, filepath)

		// index into elasticsearch
		elasticsearch.WriteImageToDatabase(elasticsearch.ImageMetaData{
			Name:   comic.Title,
			ID:     date,
			Source: "dilbert",
			Title:  strings.Join(comic.Tags, " "),
			Text:   comic.Transcript,
			Path:   filepath,
		}, "dilbert")
		// incr count, reset attempts and reset backoff
		count++
		attempt = 0
		b.Reset()
	}
	log.WithFields(log.Fields{"count": count}).Info("dilbert comic complete")
	return nil
}

// // DownloadImage downloads image to filepath
// func downloadImageWithBackOff(url, filepath string, b *backoff.Backoff) error {
// 	// Create the file
// 	out, err := os.Create(filepath)
// 	if err != nil {
// 		log.Error(err)
// 	}
// 	defer out.Close()

// 	resp, err := http.Get(url)
// 	if err != nil {
// 		log.Error(err)
// 		log.Info("waiting to try to download again")
// 		time.Sleep(b.Duration())
// 		// retry image download
// 		downloadImageWithBackOff(url, filepath, b)
// 	}
// 	defer resp.Body.Close()

// 	// resert backoff
// 	b.Reset()

// 	log.WithFields(log.Fields{
// 		"status":   resp.Status,
// 		"size":     resp.ContentLength,
// 		"filepath": filepath,
// 	}).Debug("downloading file")

// 	// Writer the body to file
// 	_, err = io.Copy(out, resp.Body)
// 	if err != nil {
// 		return err
// 	}
// 	return nil
// }
<commit_msg>tweak backoff<commit_after>package dilbert

import (
	"fmt"
	"path/filepath"
	"strings"
	"time"

	"github.com/PuerkitoBio/goquery"
	log "github.com/Sirupsen/logrus"
	"github.com/blacktop/scifgif/elasticsearch"
	"github.com/jpillora/backoff"
)

var attempt int

// MaxAttempts max number of download attempts
const MaxAttempts = 20

// Comic is the dilbert comic strip meta data
type Comic struct {
	Title      string
	Tags       []string
	ImageURL   string
	Transcript string
}

// GetComicMetaData gets all the comic strips meta data
func GetComicMetaData(url, date string, b *backoff.Backoff) Comic {
	comic := Comic{}

	if attempt > MaxAttempts {
		return comic
	}

	doc, err := goquery.NewDocument(url)
	if err != nil {
		log.Error(err)
		// backoff and try again
		backoff := b.Duration()
		log.WithFields(log.Fields{
			"backoff": backoff,
			"attempt": attempt,
		}).Info("waiting to try to again")
		time.Sleep(backoff)
		// retry url meta data parse
		attempt++
		GetComicMetaData(url, date, b)
	}
	// GET TITLE
	comic.Title = doc.Find(".comic-title-name").Text()
	// GET IMAGE URL
	doc.Find(".img-comic-container").Each(func(i int, s *goquery.Selection) {
		comic.ImageURL, _ = s.Find("img").Attr("src")
	})
	// GET TAGS
	doc.Find(".comic-tags").Each(func(i int, s *goquery.Selection) {
		s.Find("a").Each(func(i int, a *goquery.Selection) {
			comic.Tags = append(comic.Tags, strings.TrimPrefix(a.Text(), "#"))
		})
	})
	// GET TRANSCRIPT
	id := "js-toggle-transcript-" + date
	transcripts := doc.Find("div#" + id).Text()
	// clean up string
	transcripts = strings.TrimSpace(transcripts)
	transcripts = strings.TrimPrefix(transcripts, "Transcript")
	transcripts = strings.TrimSpace(transcripts)
	comic.Transcript = transcripts

	return comic
}

// GetAllDilbert havest all teh comics strips
func GetAllDilbert(folder string, date string) error {
	count := 0
	attempt = 0
	b := &backoff.Backoff{
		//These are the defaults
		Min:    100 * time.Millisecond,
		Max:    1200 * time.Second,
		Factor: 3,
		Jitter: true,
	}

	if len(date) < 1 {
		// date = "1989-04-17"
		date = "2017-09-08"
	}
	start, _ := time.Parse("2006-01-02", date)

	for d := start; time.Now().After(d); d = d.AddDate(0, 0, 1) {
		date := fmt.Sprintf("%04d-%02d-%02d", d.Year(), d.Month(), d.Day())
		url := "http://dilbert.com/strip/" + date
		comic := GetComicMetaData(url, date, b)
		// download image
		log.WithFields(log.Fields{
			"id":    date,
			"title": comic.Title,
		}).Debug("downloading file")

		filepath := filepath.Join(folder, date+".jpg")
		go elasticsearch.DownloadImage(comic.ImageURL, filepath)

		// index into elasticsearch
		elasticsearch.WriteImageToDatabase(elasticsearch.ImageMetaData{
			Name:   comic.Title,
			ID:     date,
			Source: "dilbert",
			Title:  strings.Join(comic.Tags, " "),
			Text:   comic.Transcript,
			Path:   filepath,
		}, "dilbert")
		// incr count, reset attempts and reset backoff
		count++
		attempt = 0
		b.Reset()
	}
	log.WithFields(log.Fields{"count": count}).Info("dilbert comic complete")
	return nil
}

// // DownloadImage downloads image to filepath
// func downloadImageWithBackOff(url, filepath string, b *backoff.Backoff) error {
// 	// Create the file
// 	out, err := os.Create(filepath)
// 	if err != nil {
// 		log.Error(err)
// 	}
// 	defer out.Close()

// 	resp, err := http.Get(url)
// 	if err != nil {
// 		log.Error(err)
// 		log.Info("waiting to try to download again")
// 		time.Sleep(b.Duration())
// 		// retry image download
// 		downloadImageWithBackOff(url, filepath, b)
// 	}
// 	defer resp.Body.Close()

// 	// resert backoff
// 	b.Reset()

// 	log.WithFields(log.Fields{
// 		"status":   resp.Status,
// 		"size":     resp.ContentLength,
// 		"filepath": filepath,
// 	}).Debug("downloading file")

// 	// Writer the body to file
// 	_, err = io.Copy(out, resp.Body)
// 	if err != nil {
// 		return err
// 	}
// 	return nil
// }
<|endoftext|>


<commit_before>package dnsimple

import (
	"encoding/base64"
	"fmt"
)

const (
	httpHeaderDomainToken   = "X-DNSimple-Domain-Token"
	httpHeaderApiToken      = "X-DNSimple-Token"
	httpHeaderAuthorization = "Authorization"
)

// Provides credentials that can be used for authenticating with DNSimple.
//
// See https://developer.dnsimple.com/v2/#authentication
type Credentials interface {
	// Get the HTTP header key and value to use for authentication.
	HttpHeader() (string, string)
}

// Domain token authentication

type domainTokenCredentials struct {
	domainToken string
}

// Construct Credentials using the DNSimple Domain Token method
func NewDomainTokenCredentials(domainToken string) Credentials {
	return &domainTokenCredentials{domainToken: domainToken}
}

func (c *domainTokenCredentials) HttpHeader() (string, string) {
	return httpHeaderDomainToken, c.domainToken
}

// HTTP basic authentication

type httpBasicCredentials struct {
	email    string
	password string
}

// Construct Credentials using HTTP Basic Auth
func NewHttpBasicCredentials(email, password string) Credentials {
	return &httpBasicCredentials{email, password}
}

func (c *httpBasicCredentials) HttpHeader() (string, string) {
	return httpHeaderAuthorization, "Basic " + basicAuth(c.email, c.password)
}

func basicAuth(username, password string) string {
	auth := username + ":" + password
	return base64.StdEncoding.EncodeToString([]byte(auth))
}

// OAuth token authentication

type oauthTokenCredentials struct {
	oauthToken string
}

func NewOauthTokenCredentials(oauthToken string) Credentials {
	return &oauthTokenCredentials{oauthToken: oauthToken}
}

func (c *oauthTokenCredentials) HttpHeader() (string, string) {
	return httpHeaderAuthorization, fmt.Sprintf("Bearer %v", c.oauthToken)
}
<commit_msg>Document the credential structs<commit_after>package dnsimple

import (
	"encoding/base64"
	"fmt"
)

const (
	httpHeaderDomainToken   = "X-DNSimple-Domain-Token"
	httpHeaderApiToken      = "X-DNSimple-Token"
	httpHeaderAuthorization = "Authorization"
)

// Provides credentials that can be used for authenticating with DNSimple.
//
// See https://developer.dnsimple.com/v2/#authentication
type Credentials interface {
	// Get the HTTP header key and value to use for authentication.
	HttpHeader() (string, string)
}

// Domain token authentication

type domainTokenCredentials struct {
	domainToken string
}

// NewDomainTokenCredentials construct Credentials using the DNSimple Domain Token method.
func NewDomainTokenCredentials(domainToken string) Credentials {
	return &domainTokenCredentials{domainToken: domainToken}
}

func (c *domainTokenCredentials) HttpHeader() (string, string) {
	return httpHeaderDomainToken, c.domainToken
}

// HTTP basic authentication

type httpBasicCredentials struct {
	email    string
	password string
}

// NewHttpBasicCredentials construct Credentials using HTTP Basic Auth.
func NewHttpBasicCredentials(email, password string) Credentials {
	return &httpBasicCredentials{email, password}
}

func (c *httpBasicCredentials) HttpHeader() (string, string) {
	return httpHeaderAuthorization, "Basic " + basicAuth(c.email, c.password)
}

func basicAuth(username, password string) string {
	auth := username + ":" + password
	return base64.StdEncoding.EncodeToString([]byte(auth))
}

// OAuth token authentication

type oauthTokenCredentials struct {
	oauthToken string
}

// NewOauthTokenCredentials construct Credentials using the OAuth access token.
func NewOauthTokenCredentials(oauthToken string) Credentials {
	return &oauthTokenCredentials{oauthToken: oauthToken}
}

func (c *oauthTokenCredentials) HttpHeader() (string, string) {
	return httpHeaderAuthorization, fmt.Sprintf("Bearer %v", c.oauthToken)
}
<|endoftext|>


<commit_before>// Copyright 2014 Unknown
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

// Package macaron is a high productive and modular design web framework in Go.
package macaron

import (
	"log"
	"net/http"
	"os"
	"path/filepath"
	"reflect"

	"github.com/julienschmidt/httprouter"

	"github.com/Unknwon/macaron/inject"
)

func Version() string {
	return "0.0.1.0709"
}

// Handler can be any callable function.
// Macaron attempts to inject services into the handler's argument list,
// and panics if an argument could not be fullfilled via dependency injection.
type Handler interface{}

// validateHandler makes sure a handler is a callable function,
// and panics if it is not.
func validateHandler(handler Handler) {
	if reflect.TypeOf(handler).Kind() != reflect.Func {
		panic("mocaron handler must be a callable function")
	}
}

type Context struct {
	inject.Injector
	handlers []Handler
	action   Handler
	rw       ResponseWriter
	index    int

	*Render
	params httprouter.Params
}

// Params return value of given param name.
func (c *Context) Params(name string) string {
	return c.params.ByName(name)
}

func (c *Context) handler() Handler {
	if c.index < len(c.handlers) {
		return c.handlers[c.index]
	}
	if c.index == len(c.handlers) {
		return c.action
	}
	panic("invalid index for context handler")
}

func (c *Context) Next() {
	c.index += 1
	c.run()
}

func (c *Context) Written() bool {
	return c.rw.Written()
}

func (c *Context) run() {
	for c.index <= len(c.handlers) {
		vals, err := c.Invoke(c.handler())
		if err != nil {
			panic(err)
		}
		c.index += 1

		// if the handler returned something, write it to the http response
		if len(vals) > 0 {
			ev := c.GetVal(reflect.TypeOf(ReturnHandler(nil)))
			handleReturn := ev.Interface().(ReturnHandler)
			handleReturn(c, vals)
		}

		if c.Written() {
			return
		}
	}
}

// Macaron represents the top level web application.
// inject.Injector methods can be invoked to map services on a global level.
type Macaron struct {
	inject.Injector
	handlers []Handler
	action   Handler
	*Router
	logger *log.Logger
}

// New creates a bare bones Macaron instance.
// Use this method if you want to have full control over the middleware that is used.
func New() *Macaron {
	m := &Macaron{
		Injector: inject.New(),
		action:   func() {},
		Router: &Router{
			router: httprouter.New(),
		},
		logger: log.New(os.Stdout, "[Macaron] ", 0),
	}
	m.Router.m = m
	m.Map(m.logger)
	m.Map(defaultReturnHandler())
	return m
}

// Classic creates a classic Macaron with some basic default middleware:
// mocaron.Logger, mocaron.Recovery and mocaron.Static.
func Classic() *Macaron {
	m := New()
	m.Use(Logger())
	m.Use(Recovery())
	m.Use(Static("public"))
	return m
}

// Handlers sets the entire middleware stack with the given Handlers.
// This will clear any current middleware handlers,
// and panics if any of the handlers is not a callable function
func (m *Macaron) Handlers(handlers ...Handler) {
	m.handlers = make([]Handler, 0)
	for _, handler := range handlers {
		m.Use(handler)
	}
}

// Action sets the handler that will be called after all the middleware has been invoked.
// This is set to macaron.Router in a macaron.Classic().
func (m *Macaron) Action(handler Handler) {
	validateHandler(handler)
	m.action = handler
}

// Use adds a middleware Handler to the stack,
// and panics if the handler is not a callable func.
// Middleware Handlers are invoked in the order that they are added.
func (m *Macaron) Use(handler Handler) {
	validateHandler(handler)
	m.handlers = append(m.handlers, handler)
}

func (m *Macaron) createContext(res http.ResponseWriter, req *http.Request) *Context {
	c := &Context{
		Injector: inject.New(),
		handlers: m.handlers,
		action:   m.action,
		rw:       NewResponseWriter(res),
		index:    0,
	}
	c.SetParent(m)
	c.Map(c)
	c.MapTo(c.rw, (*http.ResponseWriter)(nil))
	c.Map(req)
	return c
}

// ServeHTTP is the HTTP Entry point for a Macaron instance.
// Useful if you want to control your own HTTP server.
// Be aware that none of middleware will run without registering any router.
func (m *Macaron) ServeHTTP(res http.ResponseWriter, req *http.Request) {
	m.router.ServeHTTP(res, req)
}

// Run the http server. Listening on os.GetEnv("PORT") or 4000 by default.
func (m *Macaron) Run() {
	port := os.Getenv("PORT")
	if port == "" {
		port = "4000"
	}

	host := os.Getenv("HOST")

	logger := m.Injector.GetVal(reflect.TypeOf(m.logger)).Interface().(*log.Logger)

	logger.Printf("listening on %s:%s (%s)\n", host, port, Env)
	logger.Fatalln(http.ListenAndServe(host+":"+port, m))
}

// __________               __
// \______   \ ____  __ ___/  |_  ___________
//  |       _//  _ \|  |  \   __\/ __ \_  __ \
//  |    |   (  <_> )  |  /|  | \  ___/|  | \/
//  |____|_  /\____/|____/ |__|  \___  >__|
//         \/                        \/

func (r *Router) addRoute(method string, pattern string, handlers []Handler) {
	if len(r.groups) > 0 {
		groupPattern := ""
		h := make([]Handler, 0)
		for _, g := range r.groups {
			groupPattern += g.pattern
			h = append(h, g.handlers...)
		}

		pattern = groupPattern + pattern
		h = append(h, handlers...)
		handlers = h
	}

	r.router.Handle(method, pattern, func(resp http.ResponseWriter, req *http.Request, params httprouter.Params) {
		c := r.m.createContext(resp, req)
		c.params = params
		c.handlers = handlers
		c.run()
	})
}

type Router struct {
	m      *Macaron
	router *httprouter.Router
	prefx  string
	groups []group
}

type group struct {
	pattern  string
	handlers []Handler
}

func (r *Router) Group(pattern string, fn func(*Router), h ...Handler) {
	r.groups = append(r.groups, group{pattern, h})
	fn(r)
	r.groups = r.groups[:len(r.groups)-1]
}

func (r *Router) Get(pattern string, h ...Handler) {
	r.addRoute("GET", pattern, h)
}

func (r *Router) Patch(pattern string, h ...Handler) {
	r.addRoute("PATCH", pattern, h)
}

func (r *Router) Post(pattern string, h ...Handler) {
	r.addRoute("POST", pattern, h)
}

func (r *Router) Put(pattern string, h ...Handler) {
	r.addRoute("PUT", pattern, h)
}

func (r *Router) Delete(pattern string, h ...Handler) {
	r.addRoute("DELETE", pattern, h)
}

func (r *Router) Options(pattern string, h ...Handler) {
	r.addRoute("OPTIONS", pattern, h)
}

func (r *Router) Head(pattern string, h ...Handler) {
	r.addRoute("HEAD", pattern, h)
}

func (r *Router) NotFound(handlers ...Handler) {
	r.router.NotFound = func(resp http.ResponseWriter, req *http.Request) {
		c := r.m.createContext(resp, req)
		c.handlers = append(r.m.handlers, handlers...)
		c.run()
	}
}

// \_   _____/ _______  __
//  |    __)_ /    \  \/ /
//  |        \   |  \   /
// /_______  /___|  /\_/
//         \/     \/

const (
	DEV  string = "development"
	PROD string = "production"
	TEST string = "test"
)

// Env is the environment that Macaron is executing in.
// The MACARON_ENV is read on initialization to set this variable.
var Env = DEV
var Root string

func setENV(e string) {
	if len(e) > 0 {
		Env = e
	}
}

func init() {
	setENV(os.Getenv("MACARON_ENV"))
	path, err := filepath.Abs(os.Args[0])
	if err != nil {
		panic(err)
	}
	Root = filepath.Dir(path)
}
<commit_msg>Fix test cases<commit_after>// Copyright 2014 Unknown
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

// Package macaron is a high productive and modular design web framework in Go.
package macaron

import (
	"log"
	"net/http"
	"os"
	"path/filepath"
	"reflect"

	"github.com/julienschmidt/httprouter"

	"github.com/Unknwon/macaron/inject"
)

func Version() string {
	return "0.0.1.0709"
}

// Handler can be any callable function.
// Macaron attempts to inject services into the handler's argument list,
// and panics if an argument could not be fullfilled via dependency injection.
type Handler interface{}

// validateHandler makes sure a handler is a callable function,
// and panics if it is not.
func validateHandler(handler Handler) {
	if reflect.TypeOf(handler).Kind() != reflect.Func {
		panic("mocaron handler must be a callable function")
	}
}

type Context struct {
	inject.Injector
	handlers []Handler
	action   Handler
	rw       ResponseWriter
	index    int

	*Render
	params httprouter.Params
}

// Params return value of given param name.
func (c *Context) Params(name string) string {
	return c.params.ByName(name)
}

func (c *Context) handler() Handler {
	if c.index < len(c.handlers) {
		return c.handlers[c.index]
	}
	if c.index == len(c.handlers) {
		return c.action
	}
	panic("invalid index for context handler")
}

func (c *Context) Next() {
	c.index += 1
	c.run()
}

func (c *Context) Written() bool {
	return c.rw.Written()
}

func (c *Context) run() {
	for c.index <= len(c.handlers) {
		vals, err := c.Invoke(c.handler())
		if err != nil {
			panic(err)
		}
		c.index += 1

		// if the handler returned something, write it to the http response
		if len(vals) > 0 {
			ev := c.GetVal(reflect.TypeOf(ReturnHandler(nil)))
			handleReturn := ev.Interface().(ReturnHandler)
			handleReturn(c, vals)
		}

		if c.Written() {
			return
		}
	}
}

// Macaron represents the top level web application.
// inject.Injector methods can be invoked to map services on a global level.
type Macaron struct {
	inject.Injector
	handlers []Handler
	action   Handler
	*Router
	logger *log.Logger
}

// New creates a bare bones Macaron instance.
// Use this method if you want to have full control over the middleware that is used.
func New() *Macaron {
	m := &Macaron{
		Injector: inject.New(),
		action:   func() {},
		Router: &Router{
			router: httprouter.New(),
		},
		logger: log.New(os.Stdout, "[Macaron] ", 0),
	}
	m.Router.m = m
	m.Map(m.logger)
	m.Map(defaultReturnHandler())
	return m
}

// Classic creates a classic Macaron with some basic default middleware:
// mocaron.Logger, mocaron.Recovery and mocaron.Static.
func Classic() *Macaron {
	m := New()
	m.Use(Logger())
	m.Use(Recovery())
	m.Use(Static("public"))
	return m
}

// Handlers sets the entire middleware stack with the given Handlers.
// This will clear any current middleware handlers,
// and panics if any of the handlers is not a callable function
func (m *Macaron) Handlers(handlers ...Handler) {
	m.handlers = make([]Handler, 0)
	for _, handler := range handlers {
		m.Use(handler)
	}
}

// Action sets the handler that will be called after all the middleware has been invoked.
// This is set to macaron.Router in a macaron.Classic().
func (m *Macaron) Action(handler Handler) {
	validateHandler(handler)
	m.action = handler
}

// Use adds a middleware Handler to the stack,
// and panics if the handler is not a callable func.
// Middleware Handlers are invoked in the order that they are added.
func (m *Macaron) Use(handler Handler) {
	validateHandler(handler)
	m.handlers = append(m.handlers, handler)
}

func (m *Macaron) createContext(res http.ResponseWriter, req *http.Request) *Context {
	c := &Context{
		Injector: inject.New(),
		handlers: m.handlers,
		action:   m.action,
		rw:       NewResponseWriter(res),
		index:    0,
	}
	c.SetParent(m)
	c.Map(c)
	c.MapTo(c.rw, (*http.ResponseWriter)(nil))
	c.Map(req)
	return c
}

// ServeHTTP is the HTTP Entry point for a Macaron instance.
// Useful if you want to control your own HTTP server.
// Be aware that none of middleware will run without registering any router.
func (m *Macaron) ServeHTTP(res http.ResponseWriter, req *http.Request) {
	m.router.ServeHTTP(res, req)
}

// Run the http server. Listening on os.GetEnv("PORT") or 4000 by default.
func (m *Macaron) Run() {
	port := os.Getenv("PORT")
	if port == "" {
		port = "4000"
	}

	host := os.Getenv("HOST")

	logger := m.Injector.GetVal(reflect.TypeOf(m.logger)).Interface().(*log.Logger)

	logger.Printf("listening on %s:%s (%s)\n", host, port, Env)
	logger.Fatalln(http.ListenAndServe(host+":"+port, m))
}

// __________               __
// \______   \ ____  __ ___/  |_  ___________
//  |       _//  _ \|  |  \   __\/ __ \_  __ \
//  |    |   (  <_> )  |  /|  | \  ___/|  | \/
//  |____|_  /\____/|____/ |__|  \___  >__|
//         \/                        \/

func (r *Router) addRoute(method string, pattern string, handlers []Handler) {
	if len(r.groups) > 0 {
		groupPattern := ""
		h := make([]Handler, 0)
		for _, g := range r.groups {
			groupPattern += g.pattern
			h = append(h, g.handlers...)
		}

		pattern = groupPattern + pattern
		h = append(h, handlers...)
		handlers = h
	}

	r.router.Handle(method, pattern, func(resp http.ResponseWriter, req *http.Request, params httprouter.Params) {
		c := r.m.createContext(resp, req)
		c.params = params
		c.handlers = append(r.m.handlers, handlers...)
		c.run()
	})
}

type Router struct {
	m      *Macaron
	router *httprouter.Router
	prefx  string
	groups []group
}

type group struct {
	pattern  string
	handlers []Handler
}

func (r *Router) Group(pattern string, fn func(*Router), h ...Handler) {
	r.groups = append(r.groups, group{pattern, h})
	fn(r)
	r.groups = r.groups[:len(r.groups)-1]
}

func (r *Router) Get(pattern string, h ...Handler) {
	r.addRoute("GET", pattern, h)
}

func (r *Router) Patch(pattern string, h ...Handler) {
	r.addRoute("PATCH", pattern, h)
}

func (r *Router) Post(pattern string, h ...Handler) {
	r.addRoute("POST", pattern, h)
}

func (r *Router) Put(pattern string, h ...Handler) {
	r.addRoute("PUT", pattern, h)
}

func (r *Router) Delete(pattern string, h ...Handler) {
	r.addRoute("DELETE", pattern, h)
}

func (r *Router) Options(pattern string, h ...Handler) {
	r.addRoute("OPTIONS", pattern, h)
}

func (r *Router) Head(pattern string, h ...Handler) {
	r.addRoute("HEAD", pattern, h)
}

func (r *Router) NotFound(handlers ...Handler) {
	r.router.NotFound = func(resp http.ResponseWriter, req *http.Request) {
		c := r.m.createContext(resp, req)
		c.handlers = append(r.m.handlers, handlers...)
		c.run()
	}
}

// \_   _____/ _______  __
//  |    __)_ /    \  \/ /
//  |        \   |  \   /
// /_______  /___|  /\_/
//         \/     \/

const (
	DEV  string = "development"
	PROD string = "production"
	TEST string = "test"
)

// Env is the environment that Macaron is executing in.
// The MACARON_ENV is read on initialization to set this variable.
var Env = DEV
var Root string

func setENV(e string) {
	if len(e) > 0 {
		Env = e
	}
}

func init() {
	setENV(os.Getenv("MACARON_ENV"))
	path, err := filepath.Abs(os.Args[0])
	if err != nil {
		panic(err)
	}
	Root = filepath.Dir(path)
}
<|endoftext|>


<commit_before>// Copyright (C) 2017 Jennal(jennalcn@gmail.com). All rights reserved.
//
// Licensed under the MIT License (the "License"); you may not use this file except
// in compliance with the License. You may obtain a copy of the License at
//
// http://opensource.org/licenses/MIT
//
// Unless required by applicable law or agreed to in writing, software distributed
// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
// CONDITIONS OF ANY KIND, either express or implied. See the License for the
// specific language governing permissions and limitations under the License.

package router

import (
	"reflect"

	"github.com/jennal/goplay/aop"
	"github.com/jennal/goplay/encode"
	"github.com/jennal/goplay/helpers"
	"github.com/jennal/goplay/log"
	"github.com/jennal/goplay/pkg"
	"github.com/jennal/goplay/session"
)

type Method struct {
	caller interface{}
	method reflect.Method
}

func NewMethod(caller interface{}, method reflect.Method) *Method {
	return &Method{
		caller: caller,
		method: method,
	}
}

func (m *Method) Call(sess *session.Session, header *pkg.Header, data []byte) (result []interface{}, err error) {
	result = nil
	err = log.NewErrorf("Method.Call can't come to here, must be something wrong, m.NumIn() = %v", m.NumIn())

	if m.NumIn() == 2 {
		aop.Recover(func() {
			result = m.CallArgs(sess)
			err = nil
		}, func(e interface{}) {
			if e, ok := err.(error); ok {
				log.Error(e)
				err = e
			} else {
				log.Errorf("%#v", err)
			}
		})
	} else if m.NumIn() == 3 {
		val := m.NewArg(2)
		// fmt.Printf("Service.callRouteFunc: %#v => %v\n", val, reflect.TypeOf(val))
		decoder := encode.GetEncodeDecoder(header.Encoding)
		err = decoder.Unmarshal(data, val)
		if err != nil {
			return nil, log.NewErrorf("Service.callRouteFunc decoder.Unmarshal failed: %v", err)
		}
		// fmt.Printf("Service.callRouteFunc: %#v => %v\n", val, reflect.TypeOf(val))

		var arg2 interface{} = val
		if m.method.Type.In(2).Kind() != reflect.Ptr {
			arg2 = helpers.GetValueFromPtr(val)
		}

		aop.Recover(func() {
			result = m.CallArgs(sess, arg2)
			err = nil
		}, func(e interface{}) {
			if e, ok := err.(error); ok {
				log.Error(e)
				err = e
			} else {
				log.Errorf("%#v", err)
			}
		})
	} else if m.NumIn() == 4 {
		aop.Recover(func() {
			result = m.CallArgs(sess, header, data)
			err = nil
		}, func(e interface{}) {
			if e, ok := err.(error); ok {
				log.Error(e)
				err = e
			} else {
				log.Errorf("%#v", err)
			}
		})
	}

	return result, err
}

func (m *Method) CallArgs(args ...interface{}) []interface{} {
	vals := []reflect.Value{
		reflect.ValueOf(m.caller),
	}

	for _, v := range args {
		vals = append(vals, reflect.ValueOf(v))
	}

	result := []interface{}{}
	res := m.method.Func.Call(vals)

	for _, r := range res {
		result = append(result, r.Interface())
	}

	return result
}

func (m *Method) NewArg(i int) interface{} {
	if m.method.Type.NumIn() <= i {
		return nil
	}

	argType := m.method.Type.In(i)
	if argType.Kind() == reflect.Ptr {
		argType = argType.Elem()
	}
	return reflect.New(argType).Interface()
}

func (m *Method) NumIn() int {
	return m.method.Type.NumIn()
}
<commit_msg>fix error display on router.method<commit_after>// Copyright (C) 2017 Jennal(jennalcn@gmail.com). All rights reserved.
//
// Licensed under the MIT License (the "License"); you may not use this file except
// in compliance with the License. You may obtain a copy of the License at
//
// http://opensource.org/licenses/MIT
//
// Unless required by applicable law or agreed to in writing, software distributed
// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
// CONDITIONS OF ANY KIND, either express or implied. See the License for the
// specific language governing permissions and limitations under the License.

package router

import (
	"reflect"

	"github.com/jennal/goplay/aop"
	"github.com/jennal/goplay/encode"
	"github.com/jennal/goplay/helpers"
	"github.com/jennal/goplay/log"
	"github.com/jennal/goplay/pkg"
	"github.com/jennal/goplay/session"
)

type Method struct {
	caller interface{}
	method reflect.Method
}

func NewMethod(caller interface{}, method reflect.Method) *Method {
	return &Method{
		caller: caller,
		method: method,
	}
}

func (m *Method) Call(sess *session.Session, header *pkg.Header, data []byte) (result []interface{}, err error) {
	result = nil
	err = nil

	if m.NumIn() == 2 {
		aop.Recover(func() {
			result = m.CallArgs(sess)
		}, func(e interface{}) {
			if e, ok := err.(error); ok {
				log.Error(e)
				err = e
			} else {
				log.Errorf("%#v", err)
			}
		})
	} else if m.NumIn() == 3 {
		val := m.NewArg(2)
		// fmt.Printf("Service.callRouteFunc: %#v => %v\n", val, reflect.TypeOf(val))
		decoder := encode.GetEncodeDecoder(header.Encoding)
		err = decoder.Unmarshal(data, val)
		if err != nil {
			return nil, log.NewErrorf("Service.callRouteFunc decoder.Unmarshal failed: %v", err)
		}
		// fmt.Printf("Service.callRouteFunc: %#v => %v\n", val, reflect.TypeOf(val))

		var arg2 interface{} = val
		if m.method.Type.In(2).Kind() != reflect.Ptr {
			arg2 = helpers.GetValueFromPtr(val)
		}

		aop.Recover(func() {
			result = m.CallArgs(sess, arg2)
		}, func(e interface{}) {
			if e, ok := err.(error); ok {
				log.Error(e)
				err = e
			} else {
				log.Errorf("%#v", err)
			}
		})
	} else if m.NumIn() == 4 {
		aop.Recover(func() {
			result = m.CallArgs(sess, header, data)
		}, func(e interface{}) {
			if e, ok := err.(error); ok {
				log.Error(e)
				err = e
			} else {
				log.Errorf("%#v", err)
			}
		})
	}

	if result != nil {
		return
	}

	err = log.NewErrorf("Method.Call can't come to here, must be something wrong, m.NumIn() = %v", m.NumIn())
	return result, err
}

func (m *Method) CallArgs(args ...interface{}) []interface{} {
	vals := []reflect.Value{
		reflect.ValueOf(m.caller),
	}

	for _, v := range args {
		vals = append(vals, reflect.ValueOf(v))
	}

	result := []interface{}{}
	res := m.method.Func.Call(vals)

	for _, r := range res {
		result = append(result, r.Interface())
	}

	return result
}

func (m *Method) NewArg(i int) interface{} {
	if m.method.Type.NumIn() <= i {
		return nil
	}

	argType := m.method.Type.In(i)
	if argType.Kind() == reflect.Ptr {
		argType = argType.Elem()
	}
	return reflect.New(argType).Interface()
}

func (m *Method) NumIn() int {
	return m.method.Type.NumIn()
}
<|endoftext|>