<commit_before>package fs

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"runtime"
	"strconv"
	"strings"
	"syscall"
	"time"
)

type FileEntry struct {
	Name     string      `json:"name"`
	FullPath string      `json:"fullPath"`
	IsDir    bool        `json:"isDir"`
	Exists   bool        `json:"exists"`
	Size     int64       `json:"size"`
	Mode     os.FileMode `json:"mode"`
	Time     time.Time   `json:"time"`
	IsBroken bool        `json:"isBroken"`
	Readable bool        `json:"readable"`
	Writable bool        `json:"writable"`
}

func NewFileEntry(name string, fullPath string) *FileEntry {
	return &FileEntry{
		Name:     name,
		Exists:   true,
		FullPath: fullPath,
	}
}

func readDirectory(p string) ([]*FileEntry, error) {
	files, err := ioutil.ReadDir(p)
	if err != nil {
		return nil, err
	}

	ls := make([]*FileEntry, len(files))
	for i, info := range files {
		ls[i] = makeFileEntry(path.Join(p, info.Name()), info)
	}

	return ls, nil
}

func glob(glob string) ([]string, error) {
	files, err := filepath.Glob(glob)
	if err != nil {
		return nil, err
	}

	return files, nil
}

func readFile(path string) (map[string]interface{}, error) {
	file, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	fi, err := file.Stat()
	if err != nil {
		return nil, err
	}

	if fi.Size() > 10*1024*1024 {
		return nil, fmt.Errorf("File larger than 10MiB.")
	}

	buf := make([]byte, fi.Size())
	if _, err := io.ReadFull(file, buf); err != nil {
		return nil, err
	}

	return map[string]interface{}{"content": buf}, nil
}

func writeFile(filename string, data []byte, doNotOverwrite, Append bool) (int, error) {
	flags := os.O_RDWR | os.O_CREATE
	if doNotOverwrite {
		flags |= os.O_EXCL
	}

	if !Append {
		flags |= os.O_TRUNC
	} else {
		flags |= os.O_APPEND
	}

	file, err := os.OpenFile(filename, flags, 0666)
	if err != nil {
		return 0, err
	}

	defer file.Close()

	return file.Write(data)
}

var suffixRegexp = regexp.MustCompile(`.((_\d+)?)(\.\w*)?$`)

func uniquePath(name string) (string, error) {
	index := 1
	for {
		_, err := os.Stat(name)
		if err != nil {
			if os.IsNotExist(err) {
				break
			}
			return "", err
		}

		loc := suffixRegexp.FindStringSubmatchIndex(name)
		name = name[:loc[2]] + "_" + strconv.Itoa(index) + name[loc[3]:]
		index++
	}

	return name, nil
}

func getInfo(path string) (*FileEntry, error) {
	fi, err := os.Stat(path)
	if err != nil {
		if os.IsNotExist(err) {
			// The file doesn't exists, let the client side let this know
			// instead of returning error
			return &FileEntry{
				Name:   path,
				Exists: false,
			}, nil
		}

		return nil, err
	}

	return makeFileEntry(path, fi), nil
}

func makeFileEntry(fullPath string, fi os.FileInfo) *FileEntry {
	var fileUid uint32
	var fileGid uint32
	if runtime.GOOS != "windows" && fi.Sys() != nil {
		if f, ok := fi.Sys().(*syscall.Stat_t); ok {
			fileUid = f.Uid
			fileGid = f.Gid
		}
	}

	// check only if the files owner or group are the same, otherwise they
	// don't have any permission by default
	readable, writable := false, false
	if fileUid == uint32(os.Getuid()) {
		readable = fi.Mode()&0400 != 0
		writable = fi.Mode()&0200 != 0
	} else if fileGid == uint32(os.Getgid()) {
		readable = fi.Mode()&0040 != 0
		writable = fi.Mode()&0020 != 0
	} else {
		// all users
		readable = fi.Mode()&0004 != 0
		writable = fi.Mode()&0002 != 0
	}

	entry := &FileEntry{
		Name:     fi.Name(),
		Exists:   true,
		FullPath: fullPath,
		IsDir:    fi.IsDir(),
		Size:     fi.Size(),
		Mode:     fi.Mode(),
		Time:     fi.ModTime(),
		Readable: readable,
		Writable: writable,
	}

	if fi.Mode()&os.ModeSymlink != 0 {
		symlinkInfo, err := os.Stat(path.Dir(fullPath) + "/" + fi.Name())
		if err != nil {
			entry.IsBroken = true
			return entry
		}
		entry.IsDir = symlinkInfo.IsDir()
		entry.Size = symlinkInfo.Size()
		entry.Mode = symlinkInfo.Mode()
		entry.Time = symlinkInfo.ModTime()
	}

	return entry
}

func setPermissions(name string, mode os.FileMode, recursive bool) error {
	var doChange func(name string) error

	doChange = func(name string) error {
		if err := os.Chmod(name, mode); err != nil {
			return err
		}

		if !recursive {
			return nil
		}

		fi, err := os.Stat(name)
		if err != nil {
			return err
		}

		if !fi.IsDir() {
			return nil
		}

		dir, err := os.Open(name)
		if err != nil {
			return err
		}
		defer dir.Close()

		entries, err := dir.Readdirnames(0)
		if err != nil {
			return err
		}
		var firstErr error
		for _, entry := range entries {
			err := doChange(name + "/" + entry)
			if err != nil && firstErr == nil {
				firstErr = err
			}
		}
		return firstErr
	}

	return doChange(name)
}

func remove(path string, recursive bool) error {
	if recursive {
		return os.RemoveAll(path)
	}

	return os.Remove(path)
}

func rename(oldname, newname string) error {
	return os.Rename(oldname, newname)
}

func createDirectory(name string, recursive bool) error {
	if recursive {
		return os.MkdirAll(name, 0755)
	}

	return os.Mkdir(name, 0755)
}

type info struct {
	exists bool
	isDir  bool
}

// TODO: merge with FileEntry
func newInfo(file string) *info {
	fi, err := os.Stat(file)
	if err == nil {
		return &info{
			isDir:  fi.IsDir(),
			exists: true,
		}
	}

	if os.IsNotExist(err) {
		return &info{
			isDir:  false, // don't care
			exists: false,
		}
	}

	return nil
}

func cp(src, dst string) error {
	srcInfo, dstInfo := newInfo(src), newInfo(dst)

	// if the given path doesn't exist, there is nothing to be copied.
	if !srcInfo.exists {
		return fmt.Errorf("%s: no such file or directory.", src)
	}

	if !filepath.IsAbs(dst) || !filepath.IsAbs(src) {
		return errors.New("paths must be absolute.")
	}

	// cleanup paths before we continue. That means the followings will be equal:
	// "/home/arslan/" and "/home/arslan"
	src, dst = filepath.Clean(src), filepath.Clean(dst)

	// deny these cases:
	// "/home/arslan/Web" to "/home/arslan"
	// "/home/arslan"    to "/home/arslan"
	if src == dst || filepath.Dir(src) == dst {
		return fmt.Errorf("%s and %s are identical (not copied).", src, dst)
	}

	if srcInfo.isDir && dstInfo.exists {
		// deny this case:
		// "/home/arslan/Web" to "/home/arslan/server.go"
		if !dstInfo.isDir {
			return errors.New("can't copy a folder to a file")
		}

		// deny this case:
		// "/home/arslan" to "/home/arslan/Web"
		if strings.HasPrefix(dst, src) {
			return errors.New("cycle detected")
		}
	}

	srcBase, _ := filepath.Split(src)
	walks := 0

	// dstPath returns the rewritten destination path for the given source path
	dstPath := func(srcPath string) string {
		srcPath = strings.TrimPrefix(srcPath, srcBase)

		// foo/example/hello.txt -> bar/example/hello.txt
		if walks != 0 {
			return filepath.Join(dst, srcPath)
		}

		// hello.txt -> example/hello.txt
		if dstInfo.exists && dstInfo.isDir {
			return filepath.Join(dst, filepath.Base(srcPath))
		}

		// hello.txt -> test.txt
		return dst
	}

	return filepath.Walk(src, func(srcPath string, file os.FileInfo, err error) error {
		defer func() { walks++ }()

		if file.IsDir() {
			err := os.MkdirAll(dstPath(srcPath), 0755)
			if err != nil {
				fmt.Println("error 3", err)
				return errors.New("copy error [3]")
			}
		} else {
			err = copyFile(srcPath, dstPath(srcPath))
			if err != nil {
				fmt.Println("error 4", err)
				return errors.New("copy error [4]")
			}
		}

		return nil
	})
}

func copyFile(src, dst string) error {
	sf, err := os.Open(src)
	if err != nil {
		return err
	}
	defer sf.Close()

	fi, err := sf.Stat()
	if err != nil {
		return err
	}

	if fi.IsDir() {
		return errors.New("src is a directory, please provide a file")
	}

	df, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, fi.Mode())
	if err != nil {
		return err
	}
	defer df.Close()

	if _, err := io.Copy(df, sf); err != nil {
		return err
	}

	return nil
}
<commit_msg>fs: Use os.OpenFile to determine Readability/Writability<commit_after>package fs

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"time"
)

type FileEntry struct {
	Name     string      `json:"name"`
	FullPath string      `json:"fullPath"`
	IsDir    bool        `json:"isDir"`
	Exists   bool        `json:"exists"`
	Size     int64       `json:"size"`
	Mode     os.FileMode `json:"mode"`
	Time     time.Time   `json:"time"`
	IsBroken bool        `json:"isBroken"`
	Readable bool        `json:"readable"`
	Writable bool        `json:"writable"`
}

func NewFileEntry(name string, fullPath string) *FileEntry {
	return &FileEntry{
		Name:     name,
		Exists:   true,
		FullPath: fullPath,
	}
}

func readDirectory(p string) ([]*FileEntry, error) {
	files, err := ioutil.ReadDir(p)
	if err != nil {
		return nil, err
	}

	ls := make([]*FileEntry, len(files))
	for i, info := range files {
		ls[i] = makeFileEntry(path.Join(p, info.Name()), info)
	}

	return ls, nil
}

func glob(glob string) ([]string, error) {
	files, err := filepath.Glob(glob)
	if err != nil {
		return nil, err
	}

	return files, nil
}

func readFile(path string) (map[string]interface{}, error) {
	file, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	fi, err := file.Stat()
	if err != nil {
		return nil, err
	}

	if fi.Size() > 10*1024*1024 {
		return nil, fmt.Errorf("File larger than 10MiB.")
	}

	buf := make([]byte, fi.Size())
	if _, err := io.ReadFull(file, buf); err != nil {
		return nil, err
	}

	return map[string]interface{}{"content": buf}, nil
}

func writeFile(filename string, data []byte, doNotOverwrite, Append bool) (int, error) {
	flags := os.O_RDWR | os.O_CREATE
	if doNotOverwrite {
		flags |= os.O_EXCL
	}

	if !Append {
		flags |= os.O_TRUNC
	} else {
		flags |= os.O_APPEND
	}

	file, err := os.OpenFile(filename, flags, 0666)
	if err != nil {
		return 0, err
	}

	defer file.Close()

	return file.Write(data)
}

var suffixRegexp = regexp.MustCompile(`.((_\d+)?)(\.\w*)?$`)

func uniquePath(name string) (string, error) {
	index := 1
	for {
		_, err := os.Stat(name)
		if err != nil {
			if os.IsNotExist(err) {
				break
			}
			return "", err
		}

		loc := suffixRegexp.FindStringSubmatchIndex(name)
		name = name[:loc[2]] + "_" + strconv.Itoa(index) + name[loc[3]:]
		index++
	}

	return name, nil
}

func getInfo(path string) (*FileEntry, error) {
	fi, err := os.Stat(path)
	if err != nil {
		if os.IsNotExist(err) {
			// The file doesn't exists, let the client side let this know
			// instead of returning error
			return &FileEntry{
				Name:   path,
				Exists: false,
			}, nil
		}

		return nil, err
	}

	return makeFileEntry(path, fi), nil
}

func makeFileEntry(fullPath string, fi os.FileInfo) *FileEntry {
	var (
		readable bool
		writable bool
	)

	f, err := os.OpenFile(fullPath, os.O_RDONLY, 0)
	if f != nil {
		f.Close()
	}

	// If there is no error in attempting to open the file for Reading,
	// it is readable.
	if err == nil {
		readable = true
	}

	f, err = os.OpenFile(fullPath, os.O_WRONLY, 0)
	if f != nil {
		f.Close()
	}

	// If there are no error in attempting to open the file for Writing,
	// it is writable.
	if err == nil {
		writable = true
	}

	entry := &FileEntry{
		Name:     fi.Name(),
		Exists:   true,
		FullPath: fullPath,
		IsDir:    fi.IsDir(),
		Size:     fi.Size(),
		Mode:     fi.Mode(),
		Time:     fi.ModTime(),
		Readable: readable,
		Writable: writable,
	}

	if fi.Mode()&os.ModeSymlink != 0 {
		symlinkInfo, err := os.Stat(path.Dir(fullPath) + "/" + fi.Name())
		if err != nil {
			entry.IsBroken = true
			return entry
		}
		entry.IsDir = symlinkInfo.IsDir()
		entry.Size = symlinkInfo.Size()
		entry.Mode = symlinkInfo.Mode()
		entry.Time = symlinkInfo.ModTime()
	}

	return entry
}

func setPermissions(name string, mode os.FileMode, recursive bool) error {
	var doChange func(name string) error

	doChange = func(name string) error {
		if err := os.Chmod(name, mode); err != nil {
			return err
		}

		if !recursive {
			return nil
		}

		fi, err := os.Stat(name)
		if err != nil {
			return err
		}

		if !fi.IsDir() {
			return nil
		}

		dir, err := os.Open(name)
		if err != nil {
			return err
		}
		defer dir.Close()

		entries, err := dir.Readdirnames(0)
		if err != nil {
			return err
		}
		var firstErr error
		for _, entry := range entries {
			err := doChange(name + "/" + entry)
			if err != nil && firstErr == nil {
				firstErr = err
			}
		}
		return firstErr
	}

	return doChange(name)
}

func remove(path string, recursive bool) error {
	if recursive {
		return os.RemoveAll(path)
	}

	return os.Remove(path)
}

func rename(oldname, newname string) error {
	return os.Rename(oldname, newname)
}

func createDirectory(name string, recursive bool) error {
	if recursive {
		return os.MkdirAll(name, 0755)
	}

	return os.Mkdir(name, 0755)
}

type info struct {
	exists bool
	isDir  bool
}

// TODO: merge with FileEntry
func newInfo(file string) *info {
	fi, err := os.Stat(file)
	if err == nil {
		return &info{
			isDir:  fi.IsDir(),
			exists: true,
		}
	}

	if os.IsNotExist(err) {
		return &info{
			isDir:  false, // don't care
			exists: false,
		}
	}

	return nil
}

func cp(src, dst string) error {
	srcInfo, dstInfo := newInfo(src), newInfo(dst)

	// if the given path doesn't exist, there is nothing to be copied.
	if !srcInfo.exists {
		return fmt.Errorf("%s: no such file or directory.", src)
	}

	if !filepath.IsAbs(dst) || !filepath.IsAbs(src) {
		return errors.New("paths must be absolute.")
	}

	// cleanup paths before we continue. That means the followings will be equal:
	// "/home/arslan/" and "/home/arslan"
	src, dst = filepath.Clean(src), filepath.Clean(dst)

	// deny these cases:
	// "/home/arslan/Web" to "/home/arslan"
	// "/home/arslan"    to "/home/arslan"
	if src == dst || filepath.Dir(src) == dst {
		return fmt.Errorf("%s and %s are identical (not copied).", src, dst)
	}

	if srcInfo.isDir && dstInfo.exists {
		// deny this case:
		// "/home/arslan/Web" to "/home/arslan/server.go"
		if !dstInfo.isDir {
			return errors.New("can't copy a folder to a file")
		}

		// deny this case:
		// "/home/arslan" to "/home/arslan/Web"
		if strings.HasPrefix(dst, src) {
			return errors.New("cycle detected")
		}
	}

	srcBase, _ := filepath.Split(src)
	walks := 0

	// dstPath returns the rewritten destination path for the given source path
	dstPath := func(srcPath string) string {
		srcPath = strings.TrimPrefix(srcPath, srcBase)

		// foo/example/hello.txt -> bar/example/hello.txt
		if walks != 0 {
			return filepath.Join(dst, srcPath)
		}

		// hello.txt -> example/hello.txt
		if dstInfo.exists && dstInfo.isDir {
			return filepath.Join(dst, filepath.Base(srcPath))
		}

		// hello.txt -> test.txt
		return dst
	}

	return filepath.Walk(src, func(srcPath string, file os.FileInfo, err error) error {
		defer func() { walks++ }()

		if file.IsDir() {
			err := os.MkdirAll(dstPath(srcPath), 0755)
			if err != nil {
				fmt.Println("error 3", err)
				return errors.New("copy error [3]")
			}
		} else {
			err = copyFile(srcPath, dstPath(srcPath))
			if err != nil {
				fmt.Println("error 4", err)
				return errors.New("copy error [4]")
			}
		}

		return nil
	})
}

func copyFile(src, dst string) error {
	sf, err := os.Open(src)
	if err != nil {
		return err
	}
	defer sf.Close()

	fi, err := sf.Stat()
	if err != nil {
		return err
	}

	if fi.IsDir() {
		return errors.New("src is a directory, please provide a file")
	}

	df, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, fi.Mode())
	if err != nil {
		return err
	}
	defer df.Close()

	if _, err := io.Copy(df, sf); err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2012 Yuichi Araki. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package yall

import (
	"fmt"
)

var builtinFunctions = map[string]func(*Cell) Expr{

	"car": func(args *Cell) Expr {
		if cell, ok := args.Car().(*Cell); ok && cell != Empty {
			return cell.Car()
		}
		panic(NewRuntimeError("pair required, but got " + args.Car().String()))
	},

	"cdr": func(args *Cell) Expr {
		if cell, ok := args.Car().(*Cell); ok && cell != Empty {
			return cell.Cdr()
		}
		panic(NewRuntimeError("pair required, but got " + args.Car().String()))
	},

	"cons": func(arg *Cell) Expr {
		if cadr, ok := arg.Cadr().(*Cell); ok {
			return NewCell(arg.Car(), cadr)
		}
		panic(NewRuntimeError("Cons requires a cell for the second argument"))
	},

	"+": func(args *Cell) Expr {
		result := 0
		for Empty != args {
			result += args.Car().(*Integer).Value()
			args = args.Cdr()
		}
		return NewInteger(result)
	},

	"-": func(args *Cell) Expr {
		if Empty == args {
			panic(NewRuntimeError("Too few arguments to minus, at least 1 required"))
		}
		i, iok := args.Car().(*Integer)
		if !iok {
			panic(NewRuntimeError("Minus requires integers"))
		}
		result := i.Value()
		if Empty == args.Cdr() {
			return NewInteger(result * -1)
		}
		for cell := args.Cdr(); cell != Empty; cell = cell.Cdr() {
			i, iok := cell.Car().(*Integer)
			if !iok {
				panic(NewRuntimeError("Minus requires integers"))
			}
			result -= i.Value()
		}
		return NewInteger(result)
	},

	"*": func(args *Cell) Expr {
		result := 1
		for Empty != args {
			result *= args.Car().(*Integer).Value()
			args = args.Cdr()
		}
		return NewInteger(result)
	},

	"/": func(args *Cell) Expr {
		if Empty == args {
			panic(NewRuntimeError("Too few arguments to '/', at least 1 required"))
		}
		i, iok := args.Car().(*Integer)
		if !iok {
			panic(NewRuntimeError("'/' requires integers"))
		}
		result := i.Value()
		if Empty == args.Cdr() {
			return NewInteger(1 / result)
		}
		for cell := args.Cdr(); cell != Empty; cell = cell.Cdr() {
			i, iok := cell.Car().(*Integer)
			if !iok {
				panic(NewRuntimeError("'/' requires integers"))
			}
			result /= i.Value()
		}
		return NewInteger(result)
	},

	"type-of": func(args *Cell) Expr {
		if args.Cdr() != Empty {
			panic(NewRuntimeError("Too many arguments to type-of"))
		}
		return typeOf(args.Car())
	},

	"println": func(args *Cell) Expr {
		args.Each(func(expr Expr) {
			if str, ok := expr.(*String); ok {
				fmt.Println(str.value)
			} else {
				fmt.Println(expr.String())
			}
		})
		return True
	},

	"empty?": func(args *Cell) Expr {
		if Empty == args.car {
			return True
		}
		return False
	},

	"list": func(args *Cell) Expr {
		return args
	},
}
<commit_msg>Add '='.<commit_after>// Copyright 2012 Yuichi Araki. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package yall

import (
	"fmt"
)

var builtinFunctions = map[string]func(*Cell) Expr{

	"car": func(args *Cell) Expr {
		if cell, ok := args.Car().(*Cell); ok && cell != Empty {
			return cell.Car()
		}
		panic(NewRuntimeError("pair required, but got " + args.Car().String()))
	},

	"cdr": func(args *Cell) Expr {
		if cell, ok := args.Car().(*Cell); ok && cell != Empty {
			return cell.Cdr()
		}
		panic(NewRuntimeError("pair required, but got " + args.Car().String()))
	},

	"cons": func(arg *Cell) Expr {
		if cadr, ok := arg.Cadr().(*Cell); ok {
			return NewCell(arg.Car(), cadr)
		}
		panic(NewRuntimeError("Cons requires a cell for the second argument"))
	},

	"+": func(args *Cell) Expr {
		result := 0
		for Empty != args {
			result += args.Car().(*Integer).Value()
			args = args.Cdr()
		}
		return NewInteger(result)
	},

	"-": func(args *Cell) Expr {
		if Empty == args {
			panic(NewRuntimeError("Too few arguments to minus, at least 1 required"))
		}
		i, iok := args.Car().(*Integer)
		if !iok {
			panic(NewRuntimeError("Minus requires integers"))
		}
		result := i.Value()
		if Empty == args.Cdr() {
			return NewInteger(result * -1)
		}
		for cell := args.Cdr(); cell != Empty; cell = cell.Cdr() {
			i, iok := cell.Car().(*Integer)
			if !iok {
				panic(NewRuntimeError("Minus requires integers"))
			}
			result -= i.Value()
		}
		return NewInteger(result)
	},

	"*": func(args *Cell) Expr {
		result := 1
		for Empty != args {
			result *= args.Car().(*Integer).Value()
			args = args.Cdr()
		}
		return NewInteger(result)
	},

	"/": func(args *Cell) Expr {
		if Empty == args {
			panic(NewRuntimeError("Too few arguments to '/', at least 1 required"))
		}
		i, iok := args.Car().(*Integer)
		if !iok {
			panic(NewRuntimeError("'/' requires integers"))
		}
		result := i.Value()
		if Empty == args.Cdr() {
			return NewInteger(1 / result)
		}
		for cell := args.Cdr(); cell != Empty; cell = cell.Cdr() {
			i, iok := cell.Car().(*Integer)
			if !iok {
				panic(NewRuntimeError("'/' requires integers"))
			}
			result /= i.Value()
		}
		return NewInteger(result)
	},

	"type-of": func(args *Cell) Expr {
		if args.Cdr() != Empty {
			panic(NewRuntimeError("Too many arguments to type-of"))
		}
		return typeOf(args.Car())
	},

	"println": func(args *Cell) Expr {
		args.Each(func(expr Expr) {
			if str, ok := expr.(*String); ok {
				fmt.Println(str.value)
			} else {
				fmt.Println(expr.String())
			}
		})
		return True
	},

	"empty?": func(args *Cell) Expr {
		if Empty == args.car {
			return True
		}
		return False
	},

	"list": func(args *Cell) Expr {
		return args
	},

	"=": func(args *Cell) Expr {
		if Empty == args {
			panic(NewRuntimeError("Too few arguments to '=', at least 1 required"))
		}
		i, iok := args.Car().(*Integer)
		if !iok {
			panic(NewRuntimeError("'=' requires integers"))
		}
		value := i.Value()
		for cell := args.Cdr(); cell != Empty; cell = cell.Cdr() {
			i, iok := cell.Car().(*Integer)
			if !iok {
				panic(NewRuntimeError("'=' requires integers"))
			}
			if (value != i.Value()) {
				return False
			}
		}
		return True
	},
}
<|endoftext|>


<commit_before>// Package route53 implements a DNS provider for solving the DNS-01 challenge
// using AWS Route 53 DNS.
package route53

import (
	"fmt"
	"math/rand"
	"os"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/client"
	"github.com/aws/aws-sdk-go/aws/request"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/route53"
	"github.com/xenolf/lego/acme"
)

const (
	maxRetries = 5
	route53TTL = 10
)

// DNSProvider implements the acme.ChallengeProvider interface
type DNSProvider struct {
	client       *route53.Route53
	hostedZoneID string
}

// customRetryer implements the client.Retryer interface by composing the
// DefaultRetryer. It controls the logic for retrying recoverable request
// errors (e.g. when rate limits are exceeded).
type customRetryer struct {
	client.DefaultRetryer
}

// RetryRules overwrites the DefaultRetryer's method.
// It uses a basic exponential backoff algorithm that returns an initial
// delay of ~400ms with an upper limit of ~30 seconds which should prevent
// causing a high number of consecutive throttling errors.
// For reference: Route 53 enforces an account-wide(!) 5req/s query limit.
func (d customRetryer) RetryRules(r *request.Request) time.Duration {
	retryCount := r.RetryCount
	if retryCount > 7 {
		retryCount = 7
	}

	delay := (1 << uint(retryCount)) * (rand.Intn(50) + 200)
	return time.Duration(delay) * time.Millisecond
}

// NewDNSProvider returns a DNSProvider instance configured for the AWS
// Route 53 service.
//
// AWS Credentials are automatically detected in the following locations
// and prioritized in the following order:
// 1. Environment variables: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY,
//    AWS_REGION, [AWS_SESSION_TOKEN]
// 2. Shared credentials file (defaults to ~/.aws/credentials)
// 3. Amazon EC2 IAM role
//
// If AWS_HOSTED_ZONE_ID is not set, Lego tries to determine the correct
// public hosted zone via the FQDN.
//
// See also: https://github.com/aws/aws-sdk-go/wiki/configuring-sdk
func NewDNSProvider() (*DNSProvider, error) {
	hostedZoneID := os.Getenv("AWS_HOSTED_ZONE_ID")

	r := customRetryer{}
	r.NumMaxRetries = maxRetries
	config := request.WithRetryer(aws.NewConfig(), r)
	client := route53.New(session.New(config))

	return &DNSProvider{
		client:       client,
		hostedZoneID: hostedZoneID,
	}, nil
}

// Present creates a TXT record using the specified parameters
func (r *DNSProvider) Present(domain, token, keyAuth string) error {
	fqdn, value, _ := acme.DNS01Record(domain, keyAuth)
	value = `"` + value + `"`
	return r.changeRecord("UPSERT", fqdn, value, route53TTL)
}

// CleanUp removes the TXT record matching the specified parameters
func (r *DNSProvider) CleanUp(domain, token, keyAuth string) error {
	fqdn, value, _ := acme.DNS01Record(domain, keyAuth)
	value = `"` + value + `"`
	return r.changeRecord("DELETE", fqdn, value, route53TTL)
}

func (r *DNSProvider) changeRecord(action, fqdn, value string, ttl int) error {
	hostedZoneID, err := r.getHostedZoneID(fqdn)
	if err != nil {
		return fmt.Errorf("Failed to determine Route 53 hosted zone ID: %v", err)
	}

	recordSet := newTXTRecordSet(fqdn, value, ttl)
	reqParams := &route53.ChangeResourceRecordSetsInput{
		HostedZoneId: aws.String(hostedZoneID),
		ChangeBatch: &route53.ChangeBatch{
			Comment: aws.String("Managed by Lego"),
			Changes: []*route53.Change{
				{
					Action:            aws.String(action),
					ResourceRecordSet: recordSet,
				},
			},
		},
	}

	resp, err := r.client.ChangeResourceRecordSets(reqParams)
	if err != nil {
		return fmt.Errorf("Failed to change Route 53 record set: %v", err)
	}

	statusID := resp.ChangeInfo.Id

	return acme.WaitFor(120*time.Second, 4*time.Second, func() (bool, error) {
		reqParams := &route53.GetChangeInput{
			Id: statusID,
		}
		resp, err := r.client.GetChange(reqParams)
		if err != nil {
			return false, fmt.Errorf("Failed to query Route 53 change status: %v", err)
		}
		if *resp.ChangeInfo.Status == route53.ChangeStatusInsync {
			return true, nil
		}
		return false, nil
	})
}

func (r *DNSProvider) getHostedZoneID(fqdn string) (string, error) {
	if r.hostedZoneID != "" {
		return r.hostedZoneID, nil
	}

	authZone, err := acme.FindZoneByFqdn(fqdn, acme.RecursiveNameservers)
	if err != nil {
		return "", err
	}

	// .DNSName should not have a trailing dot
	reqParams := &route53.ListHostedZonesByNameInput{
		DNSName: aws.String(acme.UnFqdn(authZone)),
	}
	resp, err := r.client.ListHostedZonesByName(reqParams)
	if err != nil {
		return "", err
	}

	var hostedZoneID string
	for _, hostedZone := range resp.HostedZones {
		// .Name has a trailing dot
		if !*hostedZone.Config.PrivateZone && *hostedZone.Name == authZone {
			hostedZoneID = *hostedZone.Id
			break
		}
	}

	if len(hostedZoneID) == 0 {
		return "", fmt.Errorf("Zone %s not found in Route 53 for domain %s", authZone, fqdn)
	}

	if strings.HasPrefix(hostedZoneID, "/hostedzone/") {
		hostedZoneID = strings.TrimPrefix(hostedZoneID, "/hostedzone/")
	}

	return hostedZoneID, nil
}

func newTXTRecordSet(fqdn, value string, ttl int) *route53.ResourceRecordSet {
	return &route53.ResourceRecordSet{
		Name: aws.String(fqdn),
		Type: aws.String("TXT"),
		TTL:  aws.Int64(int64(ttl)),
		ResourceRecords: []*route53.ResourceRecord{
			{Value: aws.String(value)},
		},
	}
}
<commit_msg>route53: Use NewSessionWithOptions instead of deprecated New. Fixes #458. (#528)<commit_after>// Package route53 implements a DNS provider for solving the DNS-01 challenge
// using AWS Route 53 DNS.
package route53

import (
	"fmt"
	"math/rand"
	"os"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/client"
	"github.com/aws/aws-sdk-go/aws/request"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/route53"
	"github.com/xenolf/lego/acme"
)

const (
	maxRetries = 5
	route53TTL = 10
)

// DNSProvider implements the acme.ChallengeProvider interface
type DNSProvider struct {
	client       *route53.Route53
	hostedZoneID string
}

// customRetryer implements the client.Retryer interface by composing the
// DefaultRetryer. It controls the logic for retrying recoverable request
// errors (e.g. when rate limits are exceeded).
type customRetryer struct {
	client.DefaultRetryer
}

// RetryRules overwrites the DefaultRetryer's method.
// It uses a basic exponential backoff algorithm that returns an initial
// delay of ~400ms with an upper limit of ~30 seconds which should prevent
// causing a high number of consecutive throttling errors.
// For reference: Route 53 enforces an account-wide(!) 5req/s query limit.
func (d customRetryer) RetryRules(r *request.Request) time.Duration {
	retryCount := r.RetryCount
	if retryCount > 7 {
		retryCount = 7
	}

	delay := (1 << uint(retryCount)) * (rand.Intn(50) + 200)
	return time.Duration(delay) * time.Millisecond
}

// NewDNSProvider returns a DNSProvider instance configured for the AWS
// Route 53 service.
//
// AWS Credentials are automatically detected in the following locations
// and prioritized in the following order:
// 1. Environment variables: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY,
//    AWS_REGION, [AWS_SESSION_TOKEN]
// 2. Shared credentials file (defaults to ~/.aws/credentials)
// 3. Amazon EC2 IAM role
//
// If AWS_HOSTED_ZONE_ID is not set, Lego tries to determine the correct
// public hosted zone via the FQDN.
//
// See also: https://github.com/aws/aws-sdk-go/wiki/configuring-sdk
func NewDNSProvider() (*DNSProvider, error) {
	hostedZoneID := os.Getenv("AWS_HOSTED_ZONE_ID")

	r := customRetryer{}
	r.NumMaxRetries = maxRetries
	config := request.WithRetryer(aws.NewConfig(), r)
	session, err := session.NewSessionWithOptions(session.Options{Config: *config})
	if err != nil {
		return nil, err
	}
	client := route53.New(session)

	return &DNSProvider{
		client:       client,
		hostedZoneID: hostedZoneID,
	}, nil
}

// Present creates a TXT record using the specified parameters
func (r *DNSProvider) Present(domain, token, keyAuth string) error {
	fqdn, value, _ := acme.DNS01Record(domain, keyAuth)
	value = `"` + value + `"`
	return r.changeRecord("UPSERT", fqdn, value, route53TTL)
}

// CleanUp removes the TXT record matching the specified parameters
func (r *DNSProvider) CleanUp(domain, token, keyAuth string) error {
	fqdn, value, _ := acme.DNS01Record(domain, keyAuth)
	value = `"` + value + `"`
	return r.changeRecord("DELETE", fqdn, value, route53TTL)
}

func (r *DNSProvider) changeRecord(action, fqdn, value string, ttl int) error {
	hostedZoneID, err := r.getHostedZoneID(fqdn)
	if err != nil {
		return fmt.Errorf("Failed to determine Route 53 hosted zone ID: %v", err)
	}

	recordSet := newTXTRecordSet(fqdn, value, ttl)
	reqParams := &route53.ChangeResourceRecordSetsInput{
		HostedZoneId: aws.String(hostedZoneID),
		ChangeBatch: &route53.ChangeBatch{
			Comment: aws.String("Managed by Lego"),
			Changes: []*route53.Change{
				{
					Action:            aws.String(action),
					ResourceRecordSet: recordSet,
				},
			},
		},
	}

	resp, err := r.client.ChangeResourceRecordSets(reqParams)
	if err != nil {
		return fmt.Errorf("Failed to change Route 53 record set: %v", err)
	}

	statusID := resp.ChangeInfo.Id

	return acme.WaitFor(120*time.Second, 4*time.Second, func() (bool, error) {
		reqParams := &route53.GetChangeInput{
			Id: statusID,
		}
		resp, err := r.client.GetChange(reqParams)
		if err != nil {
			return false, fmt.Errorf("Failed to query Route 53 change status: %v", err)
		}
		if *resp.ChangeInfo.Status == route53.ChangeStatusInsync {
			return true, nil
		}
		return false, nil
	})
}

func (r *DNSProvider) getHostedZoneID(fqdn string) (string, error) {
	if r.hostedZoneID != "" {
		return r.hostedZoneID, nil
	}

	authZone, err := acme.FindZoneByFqdn(fqdn, acme.RecursiveNameservers)
	if err != nil {
		return "", err
	}

	// .DNSName should not have a trailing dot
	reqParams := &route53.ListHostedZonesByNameInput{
		DNSName: aws.String(acme.UnFqdn(authZone)),
	}
	resp, err := r.client.ListHostedZonesByName(reqParams)
	if err != nil {
		return "", err
	}

	var hostedZoneID string
	for _, hostedZone := range resp.HostedZones {
		// .Name has a trailing dot
		if !*hostedZone.Config.PrivateZone && *hostedZone.Name == authZone {
			hostedZoneID = *hostedZone.Id
			break
		}
	}

	if len(hostedZoneID) == 0 {
		return "", fmt.Errorf("Zone %s not found in Route 53 for domain %s", authZone, fqdn)
	}

	if strings.HasPrefix(hostedZoneID, "/hostedzone/") {
		hostedZoneID = strings.TrimPrefix(hostedZoneID, "/hostedzone/")
	}

	return hostedZoneID, nil
}

func newTXTRecordSet(fqdn, value string, ttl int) *route53.ResourceRecordSet {
	return &route53.ResourceRecordSet{
		Name: aws.String(fqdn),
		Type: aws.String("TXT"),
		TTL:  aws.Int64(int64(ttl)),
		ResourceRecords: []*route53.ResourceRecord{
			{Value: aws.String(value)},
		},
	}
}
<|endoftext|>


<commit_before>package gameLoop

import (
	"runtime"
	"time"
)

type GameLoop struct {
	onUpdate  func(float64)
	tickRate  time.Duration
	canUpdate bool
}

// Create new game loop
func New(tickRate time.Duration, onUpdate func(float64)) *GameLoop {
	return &GameLoop{
		onUpdate:  onUpdate,
		tickRate:  tickRate,
		canUpdate: false,
	}
}

func (gl *GameLoop) startLoop() {
	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	tickInterval := time.Second / gl.tickRate
	timeStart := time.Now().UnixNano()

	tick := time.Tick(tickInterval)

	for {
		if !gl.canUpdate {
			break
		}

		select {
		case <-tick:
			now := time.Now().UnixNano()
			// DT in seconds
			delta := float64(now-timeStart) / 1000000000
			timeStart = now
			gl.onUpdate(delta)
		}
	}
}

// Start game loop
func (gl *GameLoop) Start() {
	gl.canUpdate = true
	go gl.startLoop()
}

// Stop game loop
func (gl *GameLoop) Stop() {
	gl.canUpdate = false
}
<commit_msg>SetTickRate func<commit_after>package gameLoop

import (
	"runtime"
	"time"
)

type GameLoop struct {
	onUpdate  func(float64)
	tickRate  time.Duration
	canUpdate bool
}

// Create new game loop
func New(tickRate time.Duration, onUpdate func(float64)) *GameLoop {
	return &GameLoop{
		onUpdate:  onUpdate,
		tickRate:  tickRate,
		canUpdate: false,
	}
}

func (gl *GameLoop) startLoop() {
	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	tickInterval := time.Second / gl.tickRate
	timeStart := time.Now().UnixNano()

	tick := time.Tick(tickInterval)

	for {
		if !gl.canUpdate {
			break
		}

		select {
		case <-tick:
			now := time.Now().UnixNano()
			// DT in seconds
			delta := float64(now-timeStart) / 1000000000
			timeStart = now
			gl.onUpdate(delta)
		}
	}
}

func (gl *GameLoop) GetTickRate () time.Duration {
	return gl.tickRate
}

// Set tickRate and restart game loop
func (gl *GameLoop) SetTickRate (tickRate time.Duration) {
	gl.tickRate = tickRate
	gl.Restart()
}

// Start game loop
func (gl *GameLoop) Start() {
	gl.canUpdate = true
	go gl.startLoop()
}

// Stop game loop
func (gl *GameLoop) Stop() {
	gl.canUpdate = false
}

// Restart game loop
func (gl *GameLoop) Restart () {
	gl.Stop()
	gl.Start()
}
<|endoftext|>


<commit_before>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package os_test

import (
	"fmt"
	"log"
	"os"
	"time"
)

func ExampleOpenFile() {
	f, err := os.OpenFile("notes.txt", os.O_RDWR|os.O_CREATE, 0755)
	if err != nil {
		log.Fatal(err)
	}
	if err := f.Close(); err != nil {
		log.Fatal(err)
	}
}

func ExampleChmod() {
	if err := os.Chmod("some-filename", 0644); err != nil {
		log.Fatal(err)
	}
}

func ExampleChtimes() {
	mtime := time.Date(2006, time.February, 1, 3, 4, 5, 0, time.UTC)
	atime := time.Date(2007, time.March, 2, 4, 5, 6, 0, time.UTC)
	if err := os.Chtimes("some-filename", atime, mtime); err != nil {
		log.Fatal(err)
	}
}

func ExampleFileMode() {
	fi, err := os.Stat("some-filename")
	if err != nil {
		log.Fatal(err)
	}

	switch mode := fi.Mode(); {
	case mode.IsRegular():
		fmt.Println("regular file")
	case mode.IsDir():
		fmt.Println("directory")
	case mode&os.ModeSymlink != 0:
		fmt.Println("symbolic link")
	case mode&os.ModeNamedPipe != 0:
		fmt.Println("named pipe")
	}
}
<commit_msg>os: add example for IsNotExist<commit_after>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package os_test

import (
	"fmt"
	"log"
	"os"
	"time"
)

func ExampleOpenFile() {
	f, err := os.OpenFile("notes.txt", os.O_RDWR|os.O_CREATE, 0755)
	if err != nil {
		log.Fatal(err)
	}
	if err := f.Close(); err != nil {
		log.Fatal(err)
	}
}

func ExampleChmod() {
	if err := os.Chmod("some-filename", 0644); err != nil {
		log.Fatal(err)
	}
}

func ExampleChtimes() {
	mtime := time.Date(2006, time.February, 1, 3, 4, 5, 0, time.UTC)
	atime := time.Date(2007, time.March, 2, 4, 5, 6, 0, time.UTC)
	if err := os.Chtimes("some-filename", atime, mtime); err != nil {
		log.Fatal(err)
	}
}

func ExampleFileMode() {
	fi, err := os.Stat("some-filename")
	if err != nil {
		log.Fatal(err)
	}

	switch mode := fi.Mode(); {
	case mode.IsRegular():
		fmt.Println("regular file")
	case mode.IsDir():
		fmt.Println("directory")
	case mode&os.ModeSymlink != 0:
		fmt.Println("symbolic link")
	case mode&os.ModeNamedPipe != 0:
		fmt.Println("named pipe")
	}
}

func ExampleIsNotExist() {
	filename := "a-nonexistent-file"
	if _, err := os.Stat(filename); os.IsNotExist(err) {
		fmt.Printf("file does not exist")
	}
	// Output:
	// file does not exist
}
<|endoftext|>


<commit_before>package overlord

import (
	"net/http"
	"crypto/sha1"
	"encoding/hex"
	"time"
	"strconv"
	"fmt"
	"io"
)

type Client struct {
	Host  string
	User  string
	Key   string
}

func (self *Client) AuthHeader(method, uri string) string {
	// Authorization: user ts sha1(user + key + ts + method + uri)
	tsStr := strconv.Itoa(time.Now().Unix())
	strToHash := self.User + self.Key + tsStr + method + uri
	sha1Sum := sha1.Sum([]byte(strToHash))
	hexHash := hex.EncodeToString(sha1Sum[:])
	return fmt.Sprintf("%s %s %s", self.User, tsStr, hexHash)
}

func (self *Client) HttpClient(method, uri string, body io.Reader) {
	ret, err:= http.NewRequest(method, uri, body)
	if err != nil {
		return
	}
	ret.Header.Set("Authorization", self.AuthHeader(method, uri))
}
<commit_msg>remove client code<commit_after><|endoftext|>


<commit_before>package switching

import (
	"github.com/kandoo/beehive-netctrl/nom"
	bh "github.com/kandoo/beehive"
)

type Hub struct{}

func (h *Hub) Rcv(msg bh.Msg, ctx bh.RcvContext) error {
	in := msg.Data().(nom.PacketIn)
	out := nom.PacketOut{
		Node:     in.Node,
		InPort:   in.InPort,
		BufferID: in.BufferID,
		Packet:   in.Packet,
		Actions:  []nom.Action{nom.ActionFlood{}},
	}
	ctx.ReplyTo(msg, out)
	return nil
}

func (h *Hub) Map(msg bh.Msg, ctx bh.MapContext) bh.MappedCells {
	return bh.MappedCells{{"N", bh.Key(msg.Data().(nom.PacketIn).Node)}}
}
<commit_msg>Fix hub according to the new beehive API<commit_after>package switching

import (
	bh "github.com/kandoo/beehive"
	"github.com/kandoo/beehive-netctrl/nom"
)

type Hub struct{}

func (h Hub) Rcv(msg bh.Msg, ctx bh.RcvContext) error {
	in := msg.Data().(nom.PacketIn)
	out := nom.PacketOut{
		Node:     in.Node,
		InPort:   in.InPort,
		BufferID: in.BufferID,
		Packet:   in.Packet,
		Actions:  []nom.Action{nom.ActionFlood{}},
	}
	ctx.ReplyTo(msg, out)
	return nil
}

func (h Hub) Map(msg bh.Msg, ctx bh.MapContext) bh.MappedCells {
	return bh.MappedCells{{"N", string(msg.Data().(nom.PacketIn).Node)}}
}
<|endoftext|>


<commit_before>// Copyright 2017-2021 Jeff Foley. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package systems

import (
	"errors"
	"fmt"
	"log"
	"net"
	"os"
	"runtime"
	"sort"
	"sync"
	"time"

	"github.com/OWASP/Amass/v3/config"
	"github.com/OWASP/Amass/v3/limits"
	amassnet "github.com/OWASP/Amass/v3/net"
	"github.com/OWASP/Amass/v3/requests"
	"github.com/caffix/netmap"
	"github.com/caffix/resolve"
	"github.com/caffix/service"
)

// LocalSystem implements a System to be executed within a single process.
type LocalSystem struct {
	Cfg               *config.Config
	pool              resolve.Resolver
	graphs            []*netmap.Graph
	cache             *requests.ASNCache
	done              chan struct{}
	doneAlreadyClosed bool
	addSource         chan service.Service
	allSources        chan chan []service.Service
}

// NewLocalSystem returns an initialized LocalSystem object.
func NewLocalSystem(c *config.Config) (*LocalSystem, error) {
	if err := c.CheckSettings(); err != nil {
		return nil, err
	}

	max := int(float64(limits.GetFileLimit()) * 0.7)

	var pool resolve.Resolver
	if len(c.Resolvers) == 0 {
		pool = publicResolverSetup(c, max)
	} else {
		pool = customResolverSetup(c, max)
	}
	if pool == nil {
		return nil, errors.New("the system was unable to build the pool of resolvers")
	}

	sys := &LocalSystem{
		Cfg:        c,
		pool:       pool,
		cache:      requests.NewASNCache(),
		done:       make(chan struct{}, 2),
		addSource:  make(chan service.Service),
		allSources: make(chan chan []service.Service, 10),
	}

	// Load the ASN information into the cache
	if err := sys.loadCacheData(); err != nil {
		_ = sys.Shutdown()
		return nil, err
	}
	// Make sure that the output directory is setup for this local system
	if err := sys.setupOutputDirectory(); err != nil {
		_ = sys.Shutdown()
		return nil, err
	}
	// Setup the correct graph database handler
	if err := sys.setupGraphDBs(); err != nil {
		_ = sys.Shutdown()
		return nil, err
	}

	go sys.manageDataSources()
	return sys, nil
}

// Config implements the System interface.
func (l *LocalSystem) Config() *config.Config {
	return l.Cfg
}

// Pool implements the System interface.
func (l *LocalSystem) Pool() resolve.Resolver {
	return l.pool
}

// Cache implements the System interface.
func (l *LocalSystem) Cache() *requests.ASNCache {
	return l.cache
}

// AddSource implements the System interface.
func (l *LocalSystem) AddSource(src service.Service) error {
	l.addSource <- src
	return nil
}

// AddAndStart implements the System interface.
func (l *LocalSystem) AddAndStart(srv service.Service) error {
	err := srv.Start()

	if err == nil {
		return l.AddSource(srv)
	}
	return err
}

// DataSources implements the System interface.
func (l *LocalSystem) DataSources() []service.Service {
	ch := make(chan []service.Service, 2)

	l.allSources <- ch
	return <-ch
}

// SetDataSources assigns the data sources that will be used by the system.
func (l *LocalSystem) SetDataSources(sources []service.Service) {
	f := func(src service.Service, ch chan error) { ch <- l.AddAndStart(src) }

	ch := make(chan error, len(sources))
	// Add all the data sources that successfully start to the list
	for _, src := range sources {
		go f(src, ch)
	}

	t := time.NewTimer(5 * time.Second)
	defer t.Stop()
loop:
	for i := 0; i < len(sources); i++ {
		select {
		case <-t.C:
			break loop
		case <-ch:
		}
	}
}

// GraphDatabases implements the System interface.
func (l *LocalSystem) GraphDatabases() []*netmap.Graph {
	return l.graphs
}

// Shutdown implements the System interface.
func (l *LocalSystem) Shutdown() error {
	if l.doneAlreadyClosed {
		return nil
	}
	l.doneAlreadyClosed = true

	var wg sync.WaitGroup
	for _, src := range l.DataSources() {
		wg.Add(1)

		go func(s service.Service, w *sync.WaitGroup) {
			defer w.Done()
			_ = s.Stop()
		}(src, &wg)
	}

	wg.Wait()
	close(l.done)

	for _, g := range l.GraphDatabases() {
		g.Close()
	}

	l.pool.Stop()
	l.cache = nil
	return nil
}

func (l *LocalSystem) setupOutputDirectory() error {
	path := config.OutputDirectory(l.Cfg.Dir)
	if path == "" {
		return nil
	}

	var err error
	// If the directory does not yet exist, create it
	if err = os.MkdirAll(path, 0755); err != nil {
		return nil
	}

	return nil
}

// Select the graph that will store the System findings.
func (l *LocalSystem) setupGraphDBs() error {
	cfg := l.Config()

	var dbs []*config.Database
	if db := cfg.LocalDatabaseSettings(cfg.GraphDBs); db != nil {
		dbs = append(dbs, db)
	}
	dbs = append(dbs, cfg.GraphDBs...)

	for _, db := range dbs {
		cayley := netmap.NewCayleyGraph(db.System, db.URL, db.Options)
		if cayley == nil {
			return fmt.Errorf("System: Failed to create the %s graph", db.System)
		}

		g := netmap.NewGraph(cayley)
		if g == nil {
			return fmt.Errorf("System: Failed to create the %s graph", g.String())
		}

		l.graphs = append(l.graphs, g)
	}

	return nil
}

// GetMemoryUsage returns the number bytes allocated to heap objects on this system.
func (l *LocalSystem) GetMemoryUsage() uint64 {
	var m runtime.MemStats

	runtime.ReadMemStats(&m)
	return m.Alloc
}

func (l *LocalSystem) manageDataSources() {
	var dataSources []service.Service

	for {
		select {
		case <-l.done:
			return
		case add := <-l.addSource:
			dataSources = append(dataSources, add)
			sort.Slice(dataSources, func(i, j int) bool {
				return dataSources[i].String() < dataSources[j].String()
			})
		case all := <-l.allSources:
			all <- dataSources
		}
	}
}

func (l *LocalSystem) loadCacheData() error {
	ranges, err := config.GetIP2ASNData()
	if err != nil {
		return err
	}

	for _, r := range ranges {
		cidr := amassnet.Range2CIDR(r.FirstIP, r.LastIP)
		if cidr == nil {
			continue
		}
		if ones, _ := cidr.Mask.Size(); ones == 0 {
			continue
		}

		l.cache.Update(&requests.ASNRequest{
			Address:     r.FirstIP.String(),
			ASN:         r.ASN,
			CC:          r.CC,
			Prefix:      cidr.String(),
			Description: r.Description,
		})
	}

	return nil
}

func customResolverSetup(cfg *config.Config, max int) resolve.Resolver {
	num := len(cfg.Resolvers)
	if num > max {
		num = max
	}

	if cfg.MaxDNSQueries == 0 {
		cfg.MaxDNSQueries = num * config.DefaultQueriesPerBaselineResolver
	} else if cfg.MaxDNSQueries < num {
		cfg.MaxDNSQueries = num
	}

	rate := cfg.MaxDNSQueries / num
	var trusted []resolve.Resolver
	for _, addr := range cfg.Resolvers {
		if r := resolve.NewBaseResolver(addr, rate, cfg.Log); r != nil {
			trusted = append(trusted, r)
		}
	}

	return resolve.NewResolverPool(trusted, 2*time.Second, nil, 1, cfg.Log)
}

func publicResolverSetup(cfg *config.Config, max int) resolve.Resolver {
	baselines := len(config.DefaultBaselineResolvers)

	num := len(config.PublicResolvers)
	if num > max {
		num = max - baselines
	}

	if cfg.MaxDNSQueries == 0 {
		cfg.MaxDNSQueries = num * config.DefaultQueriesPerPublicResolver
	} else if cfg.MaxDNSQueries < num {
		cfg.MaxDNSQueries = num
	}

	trusted := setupResolvers(config.DefaultBaselineResolvers, baselines, config.DefaultQueriesPerBaselineResolver, cfg.Log)
	if len(trusted) == 0 {
		return nil
	}

	wcd := resolve.NewBaseResolver("8.8.8.8", 50, cfg.Log)
	baseline := resolve.NewResolverPool(trusted, time.Second, wcd, 1, cfg.Log)

	r := setupResolvers(config.PublicResolvers, max, config.DefaultQueriesPerPublicResolver, cfg.Log)
	return resolve.NewResolverPool(r, 2*time.Second, baseline, 2, cfg.Log)
}

func setupResolvers(addrs []string, max, rate int, log *log.Logger) []resolve.Resolver {
	if len(addrs) <= 0 {
		return nil
	}

	addrs = checkAddresses(addrs)
	addrs = runSubnetChecks(addrs)

	finished := make(chan resolve.Resolver, 10)

	for _, addr := range addrs {
		go func(ip string, ch chan resolve.Resolver) {
			if n := resolve.NewBaseResolver(ip, rate, log); n != nil {
				ch <- n
				return
			}
			ch <- nil
		}(addr, finished)
	}

	var count int
	l := len(addrs)
	var resolvers []resolve.Resolver
	for i := 0; i < l; i++ {
		if r := <-finished; r != nil {
			if count > max {
				r.Stop()
				continue
			}
			resolvers = append(resolvers, r)
			count++
		}
	}

	if len(resolvers) == 0 {
		return nil
	}
	return resolvers
}

func checkAddresses(addrs []string) []string {
	ips := []string{}

	for _, addr := range addrs {
		ip, port, err := net.SplitHostPort(addr)
		if err != nil {
			if net.ParseIP(addr) == nil {
				continue
			}
			// Add the default port number to the IP address
			addr = net.JoinHostPort(addr, "53")
		} else {
			if net.ParseIP(ip) == nil {
				continue
			}
			addr = net.JoinHostPort(ip, port)
		}
		ips = append(ips, addr)
	}

	return ips
}

func runSubnetChecks(addrs []string) []string {
	finished := make(chan string, 10)

	for _, addr := range addrs {
		go func(ip string, ch chan string) {
			if err := resolve.ClientSubnetCheck(ip); err == nil {
				ch <- ip
				return
			}
			ch <- ""
		}(addr, finished)
	}

	l := len(addrs)
	var ips []string
	for i := 0; i < l; i++ {
		if ip := <-finished; ip != "" {
			ips = append(ips, ip)
		}
	}

	if len(ips) == 0 {
		return addrs
	}
	return ips
}
<commit_msg>a better implementation not requiring the use of else<commit_after>// Copyright 2017-2021 Jeff Foley. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package systems

import (
	"errors"
	"fmt"
	"log"
	"net"
	"os"
	"runtime"
	"sort"
	"sync"
	"time"

	"github.com/OWASP/Amass/v3/config"
	"github.com/OWASP/Amass/v3/limits"
	amassnet "github.com/OWASP/Amass/v3/net"
	"github.com/OWASP/Amass/v3/requests"
	"github.com/caffix/netmap"
	"github.com/caffix/resolve"
	"github.com/caffix/service"
)

// LocalSystem implements a System to be executed within a single process.
type LocalSystem struct {
	Cfg               *config.Config
	pool              resolve.Resolver
	graphs            []*netmap.Graph
	cache             *requests.ASNCache
	done              chan struct{}
	doneAlreadyClosed bool
	addSource         chan service.Service
	allSources        chan chan []service.Service
}

// NewLocalSystem returns an initialized LocalSystem object.
func NewLocalSystem(c *config.Config) (*LocalSystem, error) {
	if err := c.CheckSettings(); err != nil {
		return nil, err
	}

	max := int(float64(limits.GetFileLimit()) * 0.7)

	var pool resolve.Resolver
	if len(c.Resolvers) == 0 {
		pool = publicResolverSetup(c, max)
	} else {
		pool = customResolverSetup(c, max)
	}
	if pool == nil {
		return nil, errors.New("the system was unable to build the pool of resolvers")
	}

	sys := &LocalSystem{
		Cfg:        c,
		pool:       pool,
		cache:      requests.NewASNCache(),
		done:       make(chan struct{}, 2),
		addSource:  make(chan service.Service),
		allSources: make(chan chan []service.Service, 10),
	}

	// Load the ASN information into the cache
	if err := sys.loadCacheData(); err != nil {
		_ = sys.Shutdown()
		return nil, err
	}
	// Make sure that the output directory is setup for this local system
	if err := sys.setupOutputDirectory(); err != nil {
		_ = sys.Shutdown()
		return nil, err
	}
	// Setup the correct graph database handler
	if err := sys.setupGraphDBs(); err != nil {
		_ = sys.Shutdown()
		return nil, err
	}

	go sys.manageDataSources()
	return sys, nil
}

// Config implements the System interface.
func (l *LocalSystem) Config() *config.Config {
	return l.Cfg
}

// Pool implements the System interface.
func (l *LocalSystem) Pool() resolve.Resolver {
	return l.pool
}

// Cache implements the System interface.
func (l *LocalSystem) Cache() *requests.ASNCache {
	return l.cache
}

// AddSource implements the System interface.
func (l *LocalSystem) AddSource(src service.Service) error {
	l.addSource <- src
	return nil
}

// AddAndStart implements the System interface.
func (l *LocalSystem) AddAndStart(srv service.Service) error {
	err := srv.Start()

	if err == nil {
		return l.AddSource(srv)
	}
	return err
}

// DataSources implements the System interface.
func (l *LocalSystem) DataSources() []service.Service {
	ch := make(chan []service.Service, 2)

	l.allSources <- ch
	return <-ch
}

// SetDataSources assigns the data sources that will be used by the system.
func (l *LocalSystem) SetDataSources(sources []service.Service) {
	f := func(src service.Service, ch chan error) { ch <- l.AddAndStart(src) }

	ch := make(chan error, len(sources))
	// Add all the data sources that successfully start to the list
	for _, src := range sources {
		go f(src, ch)
	}

	t := time.NewTimer(5 * time.Second)
	defer t.Stop()
loop:
	for i := 0; i < len(sources); i++ {
		select {
		case <-t.C:
			break loop
		case <-ch:
		}
	}
}

// GraphDatabases implements the System interface.
func (l *LocalSystem) GraphDatabases() []*netmap.Graph {
	return l.graphs
}

// Shutdown implements the System interface.
func (l *LocalSystem) Shutdown() error {
	if l.doneAlreadyClosed {
		return nil
	}
	l.doneAlreadyClosed = true

	var wg sync.WaitGroup
	for _, src := range l.DataSources() {
		wg.Add(1)

		go func(s service.Service, w *sync.WaitGroup) {
			defer w.Done()
			_ = s.Stop()
		}(src, &wg)
	}

	wg.Wait()
	close(l.done)

	for _, g := range l.GraphDatabases() {
		g.Close()
	}

	l.pool.Stop()
	l.cache = nil
	return nil
}

func (l *LocalSystem) setupOutputDirectory() error {
	path := config.OutputDirectory(l.Cfg.Dir)
	if path == "" {
		return nil
	}

	var err error
	// If the directory does not yet exist, create it
	if err = os.MkdirAll(path, 0755); err != nil {
		return nil
	}

	return nil
}

// Select the graph that will store the System findings.
func (l *LocalSystem) setupGraphDBs() error {
	cfg := l.Config()

	var dbs []*config.Database
	if db := cfg.LocalDatabaseSettings(cfg.GraphDBs); db != nil {
		dbs = append(dbs, db)
	}
	dbs = append(dbs, cfg.GraphDBs...)

	for _, db := range dbs {
		cayley := netmap.NewCayleyGraph(db.System, db.URL, db.Options)
		if cayley == nil {
			return fmt.Errorf("System: Failed to create the %s graph", db.System)
		}

		g := netmap.NewGraph(cayley)
		if g == nil {
			return fmt.Errorf("System: Failed to create the %s graph", g.String())
		}

		l.graphs = append(l.graphs, g)
	}

	return nil
}

// GetMemoryUsage returns the number bytes allocated to heap objects on this system.
func (l *LocalSystem) GetMemoryUsage() uint64 {
	var m runtime.MemStats

	runtime.ReadMemStats(&m)
	return m.Alloc
}

func (l *LocalSystem) manageDataSources() {
	var dataSources []service.Service

	for {
		select {
		case <-l.done:
			return
		case add := <-l.addSource:
			dataSources = append(dataSources, add)
			sort.Slice(dataSources, func(i, j int) bool {
				return dataSources[i].String() < dataSources[j].String()
			})
		case all := <-l.allSources:
			all <- dataSources
		}
	}
}

func (l *LocalSystem) loadCacheData() error {
	ranges, err := config.GetIP2ASNData()
	if err != nil {
		return err
	}

	for _, r := range ranges {
		cidr := amassnet.Range2CIDR(r.FirstIP, r.LastIP)
		if cidr == nil {
			continue
		}
		if ones, _ := cidr.Mask.Size(); ones == 0 {
			continue
		}

		l.cache.Update(&requests.ASNRequest{
			Address:     r.FirstIP.String(),
			ASN:         r.ASN,
			CC:          r.CC,
			Prefix:      cidr.String(),
			Description: r.Description,
		})
	}

	return nil
}

func customResolverSetup(cfg *config.Config, max int) resolve.Resolver {
	num := len(cfg.Resolvers)
	if num > max {
		num = max
	}

	if cfg.MaxDNSQueries == 0 {
		cfg.MaxDNSQueries = num * config.DefaultQueriesPerBaselineResolver
	} else if cfg.MaxDNSQueries < num {
		cfg.MaxDNSQueries = num
	}

	rate := cfg.MaxDNSQueries / num
	var trusted []resolve.Resolver
	for _, addr := range cfg.Resolvers {
		if r := resolve.NewBaseResolver(addr, rate, cfg.Log); r != nil {
			trusted = append(trusted, r)
		}
	}

	return resolve.NewResolverPool(trusted, 2*time.Second, nil, 1, cfg.Log)
}

func publicResolverSetup(cfg *config.Config, max int) resolve.Resolver {
	baselines := len(config.DefaultBaselineResolvers)

	num := len(config.PublicResolvers)
	if num > max {
		num = max - baselines
	}

	if cfg.MaxDNSQueries == 0 {
		cfg.MaxDNSQueries = num * config.DefaultQueriesPerPublicResolver
	} else if cfg.MaxDNSQueries < num {
		cfg.MaxDNSQueries = num
	}

	trusted := setupResolvers(config.DefaultBaselineResolvers, baselines, config.DefaultQueriesPerBaselineResolver, cfg.Log)
	if len(trusted) == 0 {
		return nil
	}

	wcd := resolve.NewBaseResolver("8.8.8.8", 50, cfg.Log)
	baseline := resolve.NewResolverPool(trusted, time.Second, wcd, 1, cfg.Log)

	r := setupResolvers(config.PublicResolvers, max, config.DefaultQueriesPerPublicResolver, cfg.Log)
	return resolve.NewResolverPool(r, 2*time.Second, baseline, 2, cfg.Log)
}

func setupResolvers(addrs []string, max, rate int, log *log.Logger) []resolve.Resolver {
	if len(addrs) <= 0 {
		return nil
	}

	addrs = checkAddresses(addrs)
	addrs = runSubnetChecks(addrs)

	finished := make(chan resolve.Resolver, 10)

	for _, addr := range addrs {
		go func(ip string, ch chan resolve.Resolver) {
			if n := resolve.NewBaseResolver(ip, rate, log); n != nil {
				ch <- n
				return
			}
			ch <- nil
		}(addr, finished)
	}

	var count int
	l := len(addrs)
	var resolvers []resolve.Resolver
	for i := 0; i < l; i++ {
		if r := <-finished; r != nil {
			if count > max {
				r.Stop()
				continue
			}
			resolvers = append(resolvers, r)
			count++
		}
	}

	if len(resolvers) == 0 {
		return nil
	}
	return resolvers
}

func checkAddresses(addrs []string) []string {
	ips := []string{}

	for _, addr := range addrs {
		ip, port, err := net.SplitHostPort(addr)
		if err != nil {
			ip = addr
			port = "53"
		}
		if net.ParseIP(ip) == nil {
			continue
		}
		addr = net.JoinHostPort(ip, port)
		ips = append(ips, addr)
	}

	return ips
}

func runSubnetChecks(addrs []string) []string {
	finished := make(chan string, 10)

	for _, addr := range addrs {
		go func(ip string, ch chan string) {
			if err := resolve.ClientSubnetCheck(ip); err == nil {
				ch <- ip
				return
			}
			ch <- ""
		}(addr, finished)
	}

	l := len(addrs)
	var ips []string
	for i := 0; i < l; i++ {
		if ip := <-finished; ip != "" {
			ips = append(ips, ip)
		}
	}

	if len(ips) == 0 {
		return addrs
	}
	return ips
}
<|endoftext|>


<commit_before>package action

import (
	"reflect"

	boshlog "github.com/cloudfoundry/bosh-utils/logger"

	bslcstem "github.com/maximilien/bosh-softlayer-cpi/softlayer/stemcell"
)

const (
	deleteStemcellLogTag = "DeleteStemcell"
)

type DeleteStemcell struct {
	stemcellFinder bslcstem.Finder
	logger         boshlog.Logger
}

func NewDeleteStemcell(stemcellFinder bslcstem.Finder, logger boshlog.Logger) DeleteStemcell {
	return DeleteStemcell{stemcellFinder: stemcellFinder, logger: logger}
}

func (a DeleteStemcell) Run(stemcellCID StemcellCID) (interface{}, error) {
   if isInteger(stemcellCID) {
	   _, found, err := a.stemcellFinder.FindById(int(stemcellCID))
	   if err != nil {
		   a.logger.Info(deleteStemcellLogTag, "Error trying to find stemcell '%s': %s", stemcellCID, err)
	   } else if !found {
		   a.logger.Info(deleteStemcellLogTag, "Stemcell '%s' not found", stemcellCID)
	   }
   }
	return nil, nil
}

func isInteger(a interface{}) bool {
	kind := reflect.TypeOf(a).Kind()
	return reflect.Int <= kind && kind <= reflect.Int64
}
<commit_msg>Fixes delete stemcell issue<commit_after>package action

import (
	"reflect"

	boshlog "github.com/cloudfoundry/bosh-utils/logger"

	bslcstem "github.com/maximilien/bosh-softlayer-cpi/softlayer/stemcell"
)

const (
	deleteStemcellLogTag = "DeleteStemcell"
)

type DeleteStemcell struct {
	stemcellFinder bslcstem.Finder
	logger         boshlog.Logger
}

func NewDeleteStemcell(stemcellFinder bslcstem.Finder, logger boshlog.Logger) DeleteStemcell {
	return DeleteStemcell{stemcellFinder: stemcellFinder, logger: logger}
}

func (a DeleteStemcell) Run(stemcellCID StemcellCID) (interface{}, error) {
   if isInteger(stemcellCID) {
	   _, found, err := a.stemcellFinder.FindById(int(stemcellCID))
	   if err != nil {
		   a.logger.Info(deleteStemcellLogTag, "Error trying to find stemcell '%s': %s", stemcellCID, err)
	   } else if !found {
		   a.logger.Info(deleteStemcellLogTag, "Stemcell '%s' not found", stemcellCID)
	   }
   }

   return nil, nil
}

func isInteger(a interface{}) bool {
	kind := reflect.TypeOf(a).Kind()
	return reflect.Int <= kind && kind <= reflect.Int64
}
<|endoftext|>


<commit_before>package floatgeom

//Tri3 is a triangle of Point3s
type Tri3 [3]Point3

//Centroid finds the centroid of a triangle
// Credit goes to github.com/yellingintothefan for their work in gel
func (t Tri3) Centroid(x, y float64) Point3 {
	p := Point3{x, y, 0.0}
	v0 := t[1].Sub(t[0])
	v1 := t[2].Sub(t[0])
	v2 := p.Sub(t[0])
	d00 := v0.Dot(v0)
	d01 := v0.Dot(v1)
	d11 := v1.Dot(v1)
	d20 := v2.Dot(v0)
	d21 := v2.Dot(v1)
	v := (d11*d20 - d01*d21) / (d00*d11 - d01*d01)
	w := (d00*d21 - d01*d20) / (d00*d11 - d01*d01)
	u := 1.0 - v - w
	return Point3{v, w, u}
}
<commit_msg>Rename Centroid -> Barycentric<commit_after>package floatgeom

//Tri3 is a triangle of Point3s
type Tri3 [3]Point3

// Barycentric finds the barycentric coordinates of the given x,y cartesian
// coordinates within this triangle. If the point (x,y) is outside of the
// triangle, one of the output values will be negative.
// Credit goes to github.com/yellingintothefan for their work in gel
func (t Tri3) Barycentric(x, y float64) Point3 {
	p := Point3{x, y, 0.0}
	v0 := t[1].Sub(t[0])
	v1 := t[2].Sub(t[0])
	v2 := p.Sub(t[0])
	d00 := v0.Dot(v0)
	d01 := v0.Dot(v1)
	d11 := v1.Dot(v1)
	d20 := v2.Dot(v0)
	d21 := v2.Dot(v1)
	v := (d11*d20 - d01*d21) / (d00*d11 - d01*d01)
	w := (d00*d21 - d01*d20) / (d00*d11 - d01*d01)
	u := 1.0 - v - w
	return Point3{v, w, u}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"

	"errors"
	"golang.org/x/net/context"
	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
	"google.golang.org/api/gmail/v1"
	"gopkg.in/cfchou/go-gentle.v1/gentle"
	log15 "gopkg.in/inconshreveable/log15.v2"
	"os"
	"sync"
	"time"
)

const (
	app_secret_file = "app_secret.json"
)

var logHandler = log15.MultiHandler(log15.StdoutHandler,
	log15.Must.FileHandler("./test.log", log15.LogfmtFormat()))
var log = log15.New("mixin", "main")
var ErrEOF = errors.New("EOF")

// getTokenFromWeb uses Config to request a Token.
// It returns the retrieved Token.
func getTokenFromWeb(config *oauth2.Config) *oauth2.Token {
	authURL := config.AuthCodeURL("state-token", oauth2.AccessTypeOffline)
	fmt.Printf("Go to the following link in your browser then type the "+
		"authorization code: \n%v\n", authURL)

	var code string
	if _, err := fmt.Scan(&code); err != nil {
		log.Error("Unable to read authorization code", "err", err)
		os.Exit(1)
	}

	tok, err := config.Exchange(context.TODO(), code)
	if err != nil {
		log.Error("Unable to retrieve token from web", "err", err)
		os.Exit(1)
	}
	return tok
}

func getAppSecret(file string) *oauth2.Config {
	bs, err := ioutil.ReadFile(file)
	if err != nil {
		log.Error("ReadFile err", "err", err)
		os.Exit(1)
	}

	config, err := google.ConfigFromJSON(bs, gmail.GmailReadonlyScope)
	if err != nil {
		log.Error("ConfigFromJson err", "err", err)
		os.Exit(1)
	}
	return config
}

type gmailMessage struct {
	msg *gmail.Message
}

func (m *gmailMessage) Id() string {
	return m.msg.Id
}

type gmailListStream struct {
	Log           log15.Logger
	service       *gmail.Service
	listCall      *gmail.UsersMessagesListCall
	lock          sync.Mutex
	messages      []*gmail.Message
	nextPageToken string
	page_num      int
	page_last     bool
	terminate     chan *struct{}
}

func NewGmailListStream(appConfig *oauth2.Config, userTok *oauth2.Token, max_results int64) *gmailListStream {
	client := appConfig.Client(context.Background(), userTok)
	// Timeout for a request
	client.Timeout = time.Second * 30

	service, err := gmail.New(client)
	if err != nil {
		log.Error("gmail.New err", "err", err)
		os.Exit(1)
	}

	listCall := service.Users.Messages.List("me")
	listCall.MaxResults(max_results)
	return &gmailListStream{
		service:   service,
		listCall:  listCall,
		lock:      sync.Mutex{},
		Log:       log.New("mixin", "list"),
		page_last: false,
		terminate: make(chan *struct{}),
	}
}

func (s *gmailListStream) nextMessage() (*gmailMessage, error) {
	// assert s.lock is Locked
	if s.messages == nil || len(s.messages) == 0 {
		s.Log.Error("Invalid state")
		os.Exit(1)
	}
	msg := &gmailMessage{msg: s.messages[0]}
	s.messages = s.messages[1:]
	return msg, nil
}

func (s *gmailListStream) shutdown() {
	s.terminate <- &struct{}{}
}

func (s *gmailListStream) Get() (gentle.Message, error) {
	s.lock.Lock()
	defer s.lock.Unlock()
	if s.messages != nil && len(s.messages) > 0 {
		return s.nextMessage()
	}
	// Messages on this page are consumed, fetch next page
	if s.page_last {
		s.Log.Info("EOF, no more messages and pages")
		select {
		case <-s.terminate:
			return nil, ErrEOF
		}
	}
	if s.nextPageToken != "" {
		s.listCall.PageToken(s.nextPageToken)
	}
	resp, err := s.listCall.Do()
	if err != nil {
		s.Log.Error("List() err", "err", err)
		return nil, err
	}

	if resp.NextPageToken == "" {
		s.Log.Info("No more pages")
		s.page_last = true
	}

	s.messages = resp.Messages
	s.nextPageToken = resp.NextPageToken
	s.page_num++
	s.Log.Info("Read a page", "page", s.page_num,
		"len_msgs", len(s.messages), "nextPageToken", s.nextPageToken)
	if len(s.messages) == 0 {
		s.Log.Info("EOF, no more messages")
		select {
		case <-s.terminate:
			return nil, ErrEOF
		}
	}
	return s.nextMessage()
}

type gmailMessageHandler struct {
	service *gmail.Service
	Log     log15.Logger
}

func NewGmailMessageHandler(appConfig *oauth2.Config, userTok *oauth2.Token) *gmailMessageHandler {
	client := appConfig.Client(context.Background(), userTok)
	//client.Timeout = time.Second * 30
	service, err := gmail.New(client)
	if err != nil {
		log.Error("gmail.New err", "err", err)
		os.Exit(1)
	}
	return &gmailMessageHandler{
		service: service,
		Log:     log.New("mixin", "download"),
	}
}

func (h *gmailMessageHandler) Handle(msg gentle.Message) (gentle.Message, error) {
	getCall := h.service.Users.Messages.Get("me", msg.Id())
	gmsg, err := getCall.Do()
	if err != nil {
		h.Log.Error("Messages.Get() err", "err", err)
		return nil, err
	}
	h.Log.Debug("Messages.Get() ok", "size", gmsg.SizeEstimate)
	return &gmailMessage{msg: gmsg}, nil
}

func example_list_only(appConfig *oauth2.Config, userTok *oauth2.Token) gentle.Stream {
	lstream := NewGmailListStream(appConfig, userTok, 500)
	return lstream
}

func example_hit_ratelimit(appConfig *oauth2.Config, userTok *oauth2.Token) gentle.Stream {

	lstream := NewGmailListStream(appConfig, userTok, 500)
	lstream.Log.SetHandler(log15.LvlFilterHandler(log15.LvlInfo, logHandler))

	mstream := gentle.NewMappedStream("gmail", lstream,
		NewGmailMessageHandler(appConfig, userTok))

	// This is likely to hit error:
	// googleapi: Error 429: Too many concurrent requests for user, rateLimitExceeded
	// googleapi: Error 429: User-rate limit exceeded.  Retry after 2017-03-13T19:26:54.011Z, rateLimitExceeded
	return gentle.NewConcurrentFetchStream("gmail", mstream, 300)
}

func example_ratelimited(appConfig *oauth2.Config, userTok *oauth2.Token) gentle.Stream {

	lstream := NewGmailListStream(appConfig, userTok, 500)
	lstream.Log.SetHandler(log15.LvlFilterHandler(log15.LvlDebug, logHandler))

	handler := NewGmailMessageHandler(appConfig, userTok)

	rhandler := gentle.NewRateLimitedHandler("gmail", handler,
		// (1000/request_interval) messages/sec, but it's an upper
		// bound, the real speed is likely much lower.
		gentle.NewTokenBucketRateLimit(1, 1))

	mstream := gentle.NewMappedStream("gmail", lstream, rhandler)
	mstream.Log.SetHandler(logHandler)

	return gentle.NewConcurrentFetchStream("gmail", mstream, 300)
}

func example_ratelimited_retry(appConfig *oauth2.Config, userTok *oauth2.Token) gentle.Stream {

	lstream := NewGmailListStream(appConfig, userTok, 500)
	lstream.Log.SetHandler(log15.LvlFilterHandler(log15.LvlDebug, logHandler))

	handler := NewGmailMessageHandler(appConfig, userTok)

	rhandler := gentle.NewRateLimitedHandler("gmail", handler,
		// (1000/request_interval) messages/sec, but it's an upper
		// bound, the real speed is likely much lower.
		gentle.NewTokenBucketRateLimit(1, 1))

	rthandler := gentle.NewRetryHandler("gmail", rhandler, []time.Duration{
		20 * time.Millisecond, 40 * time.Millisecond, 80 * time.Millisecond})
	rthandler.Log.SetHandler(logHandler)

	mstream := gentle.NewMappedStream("gmail", lstream, rthandler)
	mstream.Log.SetHandler(logHandler)

	return gentle.NewConcurrentFetchStream("gmail", mstream, 300)
}

func main() {
	h := log15.LvlFilterHandler(log15.LvlDebug, logHandler)
	log.SetHandler(h)
	config := getAppSecret(app_secret_file)
	tok := getTokenFromWeb(config)

	count := 2000
	// total should be, if gmail Messages.List() doesn't return error, the
	// total of all gmailListStream emits pluses 1(ErrEOF).
	total := 0
	// success_total should be, the number of mails have been successfully
	// downloaded.
	success_total := 0
	var totalSize int64
	//stream := example_hit_ratelimit(config, tok)
	//stream := example_ratelimited(config, tok)
	stream := example_ratelimited_retry(config, tok)

	total_begin := time.Now()
	var total_time_success time.Duration
	mails := make(map[string]bool)
	for i := 0; i < count; i++ {
		total++
		begin := time.Now()
		msg, err := GetWithTimeout(stream, 10*time.Second)
		dura := time.Now().Sub(begin)
		if err != nil {
			if err == ErrEOF {
				log.Error("Got() EOF")
				break
			} else {
				log.Error("Got() err", "err", err)
				continue
			}
		}
		gmsg := msg.(*gmailMessage).msg
		log.Debug("Got message", "msg", gmsg.Id,
			"size", gmsg.SizeEstimate, "dura", dura)
		total_time_success += dura
		success_total++
		totalSize += gmsg.SizeEstimate
		// Test duplication
		if _, existed := mails[msg.Id()]; existed {
			log.Error("Duplicated Messagge", "msg", gmsg.Id)
			break
		}
		mails[msg.Id()] = true
	}
	log.Info("Done", "total", total, "success_total", success_total,
		"total_size", totalSize,
		"total_time", time.Now().Sub(total_begin),
		"success_time", total_time_success)
	fmt.Printf("total: %d, success_total: %d, size: %d, "+
		"total_time: %s, success_time: %s\n",
		total, success_total, totalSize,
		time.Now().Sub(total_begin), total_time_success)
}

func GetWithTimeout(stream gentle.Stream, timeout time.Duration) (gentle.Message, error) {
	tm := time.NewTimer(timeout)
	result := make(chan interface{})
	go func() {
		msg, err := stream.Get()
		if err != nil {
			log.Error("stream.Get() err", "err", err)
			result <- err
		} else {
			result <- msg
		}
	}()
	var v interface{}
	select {
	case v = <-result:
	case <-tm.C:
		log.Error("timeout expired")
		return nil, ErrEOF
	}
	if inst, ok := v.(gentle.Message); ok {
		return inst, nil
	} else {
		return nil, v.(error)
	}
}
<commit_msg>tweak log<commit_after>package main

import (
	"fmt"
	"io/ioutil"

	"errors"
	"golang.org/x/net/context"
	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
	"google.golang.org/api/gmail/v1"
	"gopkg.in/cfchou/go-gentle.v1/gentle"
	log15 "gopkg.in/inconshreveable/log15.v2"
	"os"
	"sync"
	"time"
)

const (
	app_secret_file = "app_secret.json"
)

var logHandler = log15.MultiHandler(log15.StdoutHandler,
	log15.Must.FileHandler("./test.log", log15.LogfmtFormat()))
var log = log15.New("mixin", "main")
var ErrEOF = errors.New("EOF")

// getTokenFromWeb uses Config to request a Token.
// It returns the retrieved Token.
func getTokenFromWeb(config *oauth2.Config) *oauth2.Token {
	authURL := config.AuthCodeURL("state-token", oauth2.AccessTypeOffline)
	fmt.Printf("Go to the following link in your browser then type the "+
		"authorization code: \n%v\n", authURL)

	var code string
	if _, err := fmt.Scan(&code); err != nil {
		log.Error("Unable to read authorization code", "err", err)
		os.Exit(1)
	}

	tok, err := config.Exchange(context.TODO(), code)
	if err != nil {
		log.Error("Unable to retrieve token from web", "err", err)
		os.Exit(1)
	}
	return tok
}

func getAppSecret(file string) *oauth2.Config {
	bs, err := ioutil.ReadFile(file)
	if err != nil {
		log.Error("ReadFile err", "err", err)
		os.Exit(1)
	}

	config, err := google.ConfigFromJSON(bs, gmail.GmailReadonlyScope)
	if err != nil {
		log.Error("ConfigFromJson err", "err", err)
		os.Exit(1)
	}
	return config
}

type gmailMessage struct {
	msg *gmail.Message
}

func (m *gmailMessage) Id() string {
	return m.msg.Id
}

type gmailListStream struct {
	Log           log15.Logger
	service       *gmail.Service
	listCall      *gmail.UsersMessagesListCall
	lock          sync.Mutex
	messages      []*gmail.Message
	nextPageToken string
	page_num      int
	page_last     bool
	terminate     chan *struct{}
}

func NewGmailListStream(appConfig *oauth2.Config, userTok *oauth2.Token, max_results int64) *gmailListStream {
	client := appConfig.Client(context.Background(), userTok)
	// Timeout for a request
	client.Timeout = time.Second * 30

	service, err := gmail.New(client)
	if err != nil {
		log.Error("gmail.New err", "err", err)
		os.Exit(1)
	}

	listCall := service.Users.Messages.List("me")
	listCall.MaxResults(max_results)
	return &gmailListStream{
		service:   service,
		listCall:  listCall,
		lock:      sync.Mutex{},
		Log:       log.New("mixin", "list"),
		page_last: false,
		terminate: make(chan *struct{}),
	}
}

func (s *gmailListStream) nextMessage() (*gmailMessage, error) {
	// assert s.lock is already Locked
	if s.messages == nil || len(s.messages) == 0 {
		s.Log.Error("Invalid state")
		os.Exit(1)
	}
	msg := &gmailMessage{msg: s.messages[0]}
	s.messages = s.messages[1:]
	s.Log.Debug("List() nextMessge", "msg", msg.Id(), "page", s.page_num,
		"len_msgs_left", len(s.messages))
	return msg, nil
}

func (s *gmailListStream) Get() (gentle.Message, error) {
	s.Log.Debug("List() ...")
	s.lock.Lock()
	defer s.lock.Unlock()
	if s.messages != nil && len(s.messages) > 0 {
		return s.nextMessage()
	}
	// Messages on this page are consumed, fetch next page
	if s.page_last {
		s.Log.Info("List() EOF, no more messages and pages")
		select {
		case <-s.terminate:
			return nil, ErrEOF
		}
	}
	if s.nextPageToken != "" {
		s.listCall.PageToken(s.nextPageToken)
	}
	resp, err := s.listCall.Do()
	if err != nil {
		s.Log.Error("List() err", "err", err)
		return nil, err
	}

	if resp.NextPageToken == "" {
		s.Log.Info("List() No more pages")
		s.page_last = true
	}

	s.messages = resp.Messages
	s.nextPageToken = resp.NextPageToken
	s.page_num++
	s.Log.Info("List() Read a page", "page", s.page_num,
		"len_msgs", len(s.messages), "nextPageToken", s.nextPageToken)
	if len(s.messages) == 0 {
		s.Log.Info("List() EOF, no more messages")
		select {
		case <-s.terminate:
			return nil, ErrEOF
		}
	}
	return s.nextMessage()
}

type gmailMessageHandler struct {
	service *gmail.Service
	Log     log15.Logger
}

func NewGmailMessageHandler(appConfig *oauth2.Config, userTok *oauth2.Token) *gmailMessageHandler {
	client := appConfig.Client(context.Background(), userTok)
	//client.Timeout = time.Second * 30
	service, err := gmail.New(client)
	if err != nil {
		log.Error("gmail.New err", "err", err)
		os.Exit(1)
	}
	return &gmailMessageHandler{
		service: service,
		Log:     log.New("mixin", "download"),
	}
}

func (h *gmailMessageHandler) Handle(msg gentle.Message) (gentle.Message, error) {
	h.Log.Debug("Message.Get() ...", "msg_in", msg.Id())
	getCall := h.service.Users.Messages.Get("me", msg.Id())
	gmsg, err := getCall.Do()
	if err != nil {
		h.Log.Error("Messages.Get() err", "msg_in", msg.Id(),
			"err", err)
		return nil, err
	}
	h.Log.Debug("Messages.Get() ok","msg_out", gmsg.Id,
		"size", gmsg.SizeEstimate)
	return &gmailMessage{msg: gmsg}, nil
}

func example_list_only(appConfig *oauth2.Config, userTok *oauth2.Token) gentle.Stream {
	lstream := NewGmailListStream(appConfig, userTok, 500)
	return lstream
}

func example_hit_ratelimit(appConfig *oauth2.Config, userTok *oauth2.Token) gentle.Stream {

	lstream := NewGmailListStream(appConfig, userTok, 500)
	lstream.Log.SetHandler(log15.LvlFilterHandler(log15.LvlInfo, logHandler))

	mstream := gentle.NewMappedStream("gmail", lstream,
		NewGmailMessageHandler(appConfig, userTok))

	// This is likely to hit error:
	// googleapi: Error 429: Too many concurrent requests for user, rateLimitExceeded
	// googleapi: Error 429: User-rate limit exceeded.  Retry after 2017-03-13T19:26:54.011Z, rateLimitExceeded
	return gentle.NewConcurrentFetchStream("gmail", mstream, 300)
}

func example_ratelimited(appConfig *oauth2.Config, userTok *oauth2.Token) gentle.Stream {

	lstream := NewGmailListStream(appConfig, userTok, 500)
	lstream.Log.SetHandler(log15.LvlFilterHandler(log15.LvlDebug, logHandler))

	handler := NewGmailMessageHandler(appConfig, userTok)

	rhandler := gentle.NewRateLimitedHandler("gmail", handler,
		// (1000/request_interval) messages/sec, but it's an upper
		// bound, the real speed is likely much lower.
		gentle.NewTokenBucketRateLimit(1, 1))

	mstream := gentle.NewMappedStream("gmail", lstream, rhandler)
	mstream.Log.SetHandler(logHandler)

	return gentle.NewConcurrentFetchStream("gmail", mstream, 300)
}

func example_ratelimited_retry(appConfig *oauth2.Config, userTok *oauth2.Token) gentle.Stream {

	lstream := NewGmailListStream(appConfig, userTok, 500)
	lstream.Log.SetHandler(log15.LvlFilterHandler(log15.LvlDebug, logHandler))

	handler := NewGmailMessageHandler(appConfig, userTok)

	rhandler := gentle.NewRateLimitedHandler("gmail", handler,
		// (1000/request_interval) messages/sec, but it's an upper
		// bound, the real speed is likely much lower.
		gentle.NewTokenBucketRateLimit(1, 1))

	rthandler := gentle.NewRetryHandler("gmail", rhandler, []time.Duration{
		20 * time.Millisecond, 40 * time.Millisecond, 80 * time.Millisecond})
	rthandler.Log.SetHandler(logHandler)

	mstream := gentle.NewMappedStream("gmail", lstream, rthandler)
	mstream.Log.SetHandler(logHandler)

	return gentle.NewConcurrentFetchStream("gmail", mstream, 300)
}

func main() {
	h := log15.LvlFilterHandler(log15.LvlDebug, logHandler)
	log.SetHandler(h)
	config := getAppSecret(app_secret_file)
	tok := getTokenFromWeb(config)

	count := 2000
	// total should be, if gmail Messages.List() doesn't return error, the
	// total of all gmailListStream emits pluses 1(ErrEOF).
	total := 0
	// success_total should be, the number of mails have been successfully
	// downloaded.
	success_total := 0
	var totalSize int64
	//stream := example_hit_ratelimit(config, tok)
	//stream := example_ratelimited(config, tok)
	stream := example_ratelimited_retry(config, tok)

	total_begin := time.Now()
	var total_time_success time.Duration
	mails := make(map[string]bool)
	for i := 0; i < count; i++ {
		total++
		begin := time.Now()
		msg, err := GetWithTimeout(stream, 10*time.Second)
		dura := time.Now().Sub(begin)
		if err != nil {
			if err == ErrEOF {
				log.Error("Got() EOF")
				break
			} else {
				log.Error("Got() err", "err", err)
				continue
			}
		}
		gmsg := msg.(*gmailMessage).msg
		log.Debug("Got message", "msg", gmsg.Id,
			"size", gmsg.SizeEstimate)
		// NOTE: if stream is ConcurrentFetchStream, the travel time of
		// this msg is NOT dura. Because msg is asynchronously processed
		// in parallel. However, the total_time_success is still valid.
		total_time_success += dura
		success_total++
		totalSize += gmsg.SizeEstimate
		// Test duplication
		if _, existed := mails[msg.Id()]; existed {
			log.Error("Duplicated Messagge", "msg", gmsg.Id)
			break
		}
		mails[msg.Id()] = true
	}
	log.Info("Done", "total", total, "success_total", success_total,
		"total_size", totalSize,
		"total_time", time.Now().Sub(total_begin),
		"success_time", total_time_success)
	fmt.Printf("total: %d, success_total: %d, size: %d, "+
		"total_time: %s, success_time: %s\n",
		total, success_total, totalSize,
		time.Now().Sub(total_begin), total_time_success)
}

func GetWithTimeout(stream gentle.Stream, timeout time.Duration) (gentle.Message, error) {
	tm := time.NewTimer(timeout)
	result := make(chan interface{})
	go func() {
		msg, err := stream.Get()
		if err != nil {
			log.Error("stream.Get() err", "err", err)
			result <- err
		} else {
			result <- msg
		}
	}()
	var v interface{}
	select {
	case v = <-result:
	case <-tm.C:
		log.Error("timeout expired")
		return nil, ErrEOF
	}
	if inst, ok := v.(gentle.Message); ok {
		return inst, nil
	} else {
		return nil, v.(error)
	}
}
<|endoftext|>


<commit_before>package pixel

import (
	"fmt"
	"math"
	"math/cmplx"
)

// Vec is a 2d vector type. It is unusually implemented as complex128 for convenience. Since Go does
// not allow operator overloading, implementing vector as a struct leads to a bunch of methods for
// addition, subtraction and multiplication of vectors. With complex128, much of this functionality
// is given through operators.
//
// Create vectors with the V constructor:
//
//   u := pixel.V(1, 2)
//   v := pixel.V(8, -3)
//
// Add and subtract them using the standard + and - operators:
//
//   w := u + v
//   fmt.Println(w)     // Vec(9, -1)
//   fmt.Println(u - v) // Vec(-7, 5)
//
// Additional standard vector operations can be obtained with methods:
//
//   u := pixel.V(2, 3)
//   v := pixel.V(8, 1)
//   if u.X() < 0 {
//       fmt.Println("this won't happen")
//   }
//   x := u.Unit().Dot(v.Unit())
type Vec complex128

// V returns a new 2d vector with the given coordinates.
func V(x, y float64) Vec {
	return Vec(complex(x, y))
}

// String returns the string representation of a vector u.
//
//   u := pixel.V(4.5, -1.3)
//   u.String()     // returns "Vec(4.5, -1.3)"
//   fmt.Println(u) // Vec(4.5, -1.3)
func (u Vec) String() string {
	return fmt.Sprintf("Vec(%v, %v)", u.X(), u.Y())
}

// X returns the x coordinate of a vector u.
func (u Vec) X() float64 {
	return real(u)
}

// Y returns the y coordinate of a vector u.
func (u Vec) Y() float64 {
	return imag(u)
}

// XY returns the components of a vector in two return values.
func (u Vec) XY() (x, y float64) {
	return real(u), imag(u)
}

// Len returns the length of a vector u.
func (u Vec) Len() float64 {
	return cmplx.Abs(complex128(u))
}

// Angle returns the angle between a vector u and the x-axis. The result is in the range [-Pi, Pi].
func (u Vec) Angle() float64 {
	return cmplx.Phase(complex128(u))
}

// Unit returns a vector of length 1 with the same angle as u.
func (u Vec) Unit() Vec {
	return u / V(u.Len(), 0)
}

// Scaled returns a vector u multiplied by c.
func (u Vec) Scaled(c float64) Vec {
	return u * V(c, 0)
}

// Rotated returns a vector u rotated by the given angle in radians.
func (u Vec) Rotated(angle float64) Vec {
	sin, cos := math.Sincos(angle)
	return u * V(cos, sin)
}

// Dot returns the dot product of vectors u and v.
func (u Vec) Dot(v Vec) float64 {
	return u.X()*v.X() + u.Y()*v.Y()
}

// Cross return the cross product of vectors u and v.
func (u Vec) Cross(v Vec) float64 {
	return u.X()*v.Y() - v.X()*u.Y()
}
<commit_msg>add Rect<commit_after>package pixel

import (
	"fmt"
	"math"
	"math/cmplx"
)

// Vec is a 2d vector type. It is unusually implemented as complex128 for convenience. Since Go does
// not allow operator overloading, implementing vector as a struct leads to a bunch of methods for
// addition, subtraction and multiplication of vectors. With complex128, much of this functionality
// is given through operators.
//
// Create vectors with the V constructor:
//
//   u := pixel.V(1, 2)
//   v := pixel.V(8, -3)
//
// Add and subtract them using the standard + and - operators:
//
//   w := u + v
//   fmt.Println(w)     // Vec(9, -1)
//   fmt.Println(u - v) // Vec(-7, 5)
//
// Additional standard vector operations can be obtained with methods:
//
//   u := pixel.V(2, 3)
//   v := pixel.V(8, 1)
//   if u.X() < 0 {
//       fmt.Println("this won't happen")
//   }
//   x := u.Unit().Dot(v.Unit())
type Vec complex128

// V returns a new 2d vector with the given coordinates.
func V(x, y float64) Vec {
	return Vec(complex(x, y))
}

// String returns the string representation of a vector u.
//
//   u := pixel.V(4.5, -1.3)
//   u.String()     // returns "Vec(4.5, -1.3)"
//   fmt.Println(u) // Vec(4.5, -1.3)
func (u Vec) String() string {
	return fmt.Sprintf("Vec(%v, %v)", u.X(), u.Y())
}

// X returns the x coordinate of a vector u.
func (u Vec) X() float64 {
	return real(u)
}

// Y returns the y coordinate of a vector u.
func (u Vec) Y() float64 {
	return imag(u)
}

// XY returns the components of a vector in two return values.
func (u Vec) XY() (x, y float64) {
	return real(u), imag(u)
}

// Len returns the length of a vector u.
func (u Vec) Len() float64 {
	return cmplx.Abs(complex128(u))
}

// Angle returns the angle between a vector u and the x-axis. The result is in the range [-Pi, Pi].
func (u Vec) Angle() float64 {
	return cmplx.Phase(complex128(u))
}

// Unit returns a vector of length 1 with the same angle as u.
func (u Vec) Unit() Vec {
	return u / V(u.Len(), 0)
}

// Scaled returns a vector u multiplied by c.
func (u Vec) Scaled(c float64) Vec {
	return u * V(c, 0)
}

// Rotated returns a vector u rotated by the given angle in radians.
func (u Vec) Rotated(angle float64) Vec {
	sin, cos := math.Sincos(angle)
	return u * V(cos, sin)
}

// Dot returns the dot product of vectors u and v.
func (u Vec) Dot(v Vec) float64 {
	return u.X()*v.X() + u.Y()*v.Y()
}

// Cross return the cross product of vectors u and v.
func (u Vec) Cross(v Vec) float64 {
	return u.X()*v.Y() - v.X()*u.Y()
}

// Rect is a 2d rectangle aligned with the axis of the coordinate system. It has a position and a size.
//
// You can manipulate the position and the size using the usual vector operations.
type Rect struct {
	Pos, Size Vec
}

// R returns a new 2d rectangle with the given position (x, y) and size (w, h).
func R(x, y, w, h float64) Rect {
	return Rect{
		Pos:  V(x, y),
		Size: V(w, h),
	}
}

// String returns the string representation of a rectangle.
//
//   r := pixel.R(100, 50, 200, 300)
//   r.String()     // returns "Rect(100, 50, 200, 300)"
//   fmt.Println(r) // Rect(100, 50, 200, 300)
func (r Rect) String() string {
	return fmt.Sprintf("Rect(%v, %v, %v, %v)", r.X(), r.Y(), r.W(), r.H())
}

// X returns the x coordinate of the position of a rectangle.
func (r Rect) X() float64 {
	return r.Pos.X()
}

// Y returns the y coordinate of the position of a rectangle
func (r Rect) Y() float64 {
	return r.Pos.Y()
}

// W returns the width of a rectangle.
func (r Rect) W() float64 {
	return r.Size.X()
}

// H returns the height of a rectangle.
func (r Rect) H() float64 {
	return r.Size.Y()
}

// XYWH returns all of the four components of a rectangle in four return values.
func (r Rect) XYWH() (x, y, w, h float64) {
	return r.X(), r.Y(), r.W(), r.H()
}

// Center returns the position of the center of a rectangle.
func (r Rect) Center() Vec {
	return r.Pos + r.Size.Scaled(0.5)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/gorilla/handlers"
	"github.com/gorilla/mux"
  "github.com/asaskevich/govalidator"
	"log"
  "io"
  "io/ioutil"
	"net/http"
  "net/url"
	"os"
)

func main() {

  // imgUrl, _ := url.Parse("http://www.jqueryscript.net/images/Simplest-Responsive-jQuery-Image-Lightbox-Plugin-simple-lightbox.jpg")
  // file, err := downloadFile(imgUrl)
  // if err != nil {
  //   panic(err)
  // }
  // fmt.Println(file.Name())

	r := mux.NewRouter()
	r.Methods("GET").Path("/").HandlerFunc(Index)
	r.Methods("GET").Path("/api/num_colors").HandlerFunc(ApiNumColors)
	loggedR := handlers.LoggingHandler(os.Stdout, r)
	fmt.Println("Now listening on port 8080")
	log.Fatal(http.ListenAndServe(":8080", loggedR))
}

func Index(w http.ResponseWriter, r *http.Request) {
	fmt.Println(r.URL.Query())
	fmt.Fprintln(w, "visit /api/num_colors?src=<image url> to get the number of colors in your image")
}

func ApiNumColors(w http.ResponseWriter, r *http.Request) {
	imgPath := r.URL.Query().Get("src")
  if !govalidator.IsURL(imgPath) {
    http.Error(w, "Please use a valid url.", http.StatusBadRequest)
    return
  }
  imgUrl, err := url.Parse(imgPath)
  if err != nil {
    http.Error(w, "Please use a valid url.", http.StatusInternalServerError)
    return
  }

  file, err := downloadFile(imgUrl)
  if err != nil {
    http.Error(w, "We failed to download the iamge file.", http.StatusInternalServerError)
    return
  }
  fmt.Println(file.Name())
  fmt.Println(imgPath)
	fmt.Fprintln(w, "")
}

// func downloadFile(filepath string, url string) (err error) {
func downloadFile(url *url.URL) (f *os.File, err error) {

  // create the local file
  file, err := ioutil.TempFile("", "img")
  if err != nil {
    return nil, err
  }
  defer file.Close()

  resp, err := http.Get(url.String())
  if err != nil {
    return nil, err
  }
  defer resp.Body.Close()

  _, err = io.Copy(file, resp.Body)
  if err != nil {
    return nil, err
  }

  return file, nil
}
<commit_msg>Returning number of colors in image<commit_after>package main

import (
	"fmt"
	"github.com/gorilla/handlers"
	"github.com/gorilla/mux"
  "github.com/asaskevich/govalidator"
	"log"
  "io"
  "io/ioutil"
	"net/http"
  "net/url"
	"os"
  "os/exec"
  "strconv"
)

func main() {
	r := mux.NewRouter()
	r.Methods("GET").Path("/").HandlerFunc(Index)
	r.Methods("GET").Path("/api/num_colors").HandlerFunc(ApiNumColors)
	loggedR := handlers.LoggingHandler(os.Stdout, r)
	fmt.Println("Now listening on port 8080")
	log.Fatal(http.ListenAndServe(":8080", loggedR))
}

func Index(w http.ResponseWriter, r *http.Request) {
	fmt.Println(r.URL.Query())
	fmt.Fprintln(w, "visit /api/num_colors?src=<image url> to get the number of colors in your image")
}

func ApiNumColors(w http.ResponseWriter, r *http.Request) {
	imgPath := r.URL.Query().Get("src")
  if !govalidator.IsURL(imgPath) {
    http.Error(w, "Please use a valid url.", http.StatusBadRequest)
    return
  }
  imgUrl, err := url.Parse(imgPath)
  if err != nil {
    http.Error(w, "Please use a valid url.", http.StatusInternalServerError)
    return
  }

  file, err := downloadFile(imgUrl)
  if err != nil {
    http.Error(w, "We failed to download the image file.", http.StatusInternalServerError)
    return
  }

  colorCount, err := getNumberOfColorsInFile(file)
  if err != nil {
    http.Error(w, "We failed to get the number of colors from the image.", http.StatusInternalServerError)
    return
  }

	fmt.Fprintln(w, "
" + strconv.Itoa(colorCount) + "
")
}

func downloadFile(url *url.URL) (f *os.File, err error) {
  file, err := ioutil.TempFile("", "img")
  if err != nil {
    return nil, err
  }
  defer file.Close()

  resp, err := http.Get(url.String())
  if err != nil {
    return nil, err
  }
  defer resp.Body.Close()

  _, err = io.Copy(file, resp.Body)
  if err != nil {
    return nil, err
  }

  return file, nil
}

func getNumberOfColorsInFile(f *os.File) (n int, err error) {
  program := "identify"
  args := []string{"-format", "%k", f.Name()}
  cmd := exec.Command(program, args...)
  output, err := cmd.CombinedOutput()
  if err != nil {
    return -1, err
  }
  s := string(output)
  i, err := strconv.Atoi(s)
  if err != nil {
    return -1, err
  }
  return i, nil
}
<|endoftext|>