<commit_before>package main

import (
	"fmt"
	"os"
	"os/signal"
	"time"

	"github.com/inconshreveable/mousetrap"
)

//Default global variables
var (
	prefix = "https:"
)

func init() {
	if mousetrap.StartedByExplorer() {
		fmt.Println("Don't double-click ponydownloader")
		fmt.Println("You need to open cmd.exe and run it from the command line!")
		time.Sleep(5 * time.Second)
		os.Exit(1)
	}
}

func main() {
	fmt.Println("Derpibooru.org Downloader version 0.6.1")

	opts, lostArgs := getOptions()

	lInfo("Program start")
	// Checking for extra arguments we got no idea what to do with
	if len(lostArgs) != 0 {
		lErr("Too many arguments, skipping following:", lostArgs)
	}
	//If no arguments after flags and empty/unchanged tag, what we should download? Sane end of line.
	if len(opts.Args.IDs) == 0 && opts.Tag == "" {
		lDone("Nothing to download, bye!")
	}

	if opts.NoHTTPS {
		prefix = "http:" //Horrible kludge that must be removed in favor of url.URL.Scheme
	}

	if opts.UnsafeHTTPS {
		makeHTTPSUnsafe()
	}
	//Creating directory for downloads if it does not yet exist
	err := os.MkdirAll(opts.ImageDir, 0755)

	if err != nil { //Execute bit means different thing for directories that for files. And I was stupid.
		lFatal(err) //We can not create folder for images, end of line.
	}

	//	Creating channels to pass info to downloader and to signal job well done
	imgdat := make(ImageCh, opts.QDepth) //Better leave default queue depth. Experiment shown that depth about 20 provides optimal performance on my system

	if opts.Tag == "" { //Because we can put Image ID with flags. Why not?

		if len(opts.Args.IDs) == 1 {
			lInfo("Processing image №", opts.Args.IDs[0])
		} else {
			lInfo("Processing images №", debracket(opts.Args.IDs))
		}
		go imgdat.ParseImg(opts.Args.IDs, opts.Key) // Sending Image ID to parser. Here validity is our problem

	} else {

		// And here we send tags to getter/parser. Query and JSON validity is mostly server problem
		// Server response validity is ours
		lInfo("Processing tags", opts.Tag)
		go imgdat.ParseTag(opts.TagOpts, opts.Key)
	}

	lInfo("Starting worker") //It would be funny if worker goroutine does not start

	filterInit(opts.FiltOpts)           //Initiating filters based on our given flags
	filtimgdat := FilterChannel(imgdat) //Actual filtration

	sig := make(chan os.Signal, 1)
	signal.Notify(sig, os.Interrupt)

	filtimgdat.dispatch(sig).downloadImages(opts.Config) // Now that we got asynchronous list of images we want to get done, we can get them.

	lDone("Finished")
	//And we are done here! Hooray!
}

func (imgchan ImageCh) dispatch(sig <-chan os.Signal) (outch ImageCh) {
	outch = make(ImageCh)
	go imgchan.dispatcher(sig, outch)
	return outch
}

func (imgchan ImageCh) dispatcher(sig <-chan os.Signal, outch ImageCh) {
	for {
		select {
		case <-sig: //can't test this branch due to lDone killing our test
			close(outch)
			<-sig
			lDone("Download interrupted by user's command")
		default:
			select {
			case img, ok := <-imgchan:
				if !ok {
					close(outch)
					imgchan = nil
					return
				}
				outch <- img
			}
		}
	}
}
<commit_msg>Godoc! Should have done this sooner<commit_after>//Command ponydownloader uses Derpibooru.org API to download pony images
//by ID or by tags, with some client-side filtration ability
package main

import (
	"fmt"
	"os"
	"os/signal"
	"time"

	"github.com/inconshreveable/mousetrap"
)

//Default global variables
var (
	prefix = "https:"
)

func init() {
	if mousetrap.StartedByExplorer() {
		fmt.Println("Don't double-click ponydownloader")
		fmt.Println("You need to open cmd.exe and run it from the command line!")
		time.Sleep(5 * time.Second)
		os.Exit(1)
	}
}

func main() {
	fmt.Println("Derpibooru.org Downloader version 0.6.1")

	opts, lostArgs := getOptions()

	lInfo("Program start")
	// Checking for extra arguments we got no idea what to do with
	if len(lostArgs) != 0 {
		lErr("Too many arguments, skipping following:", lostArgs)
	}
	//If no arguments after flags and empty/unchanged tag, what we should download? Sane end of line.
	if len(opts.Args.IDs) == 0 && opts.Tag == "" {
		lDone("Nothing to download, bye!")
	}

	if opts.NoHTTPS {
		prefix = "http:" //Horrible kludge that must be removed in favor of url.URL.Scheme
	}

	if opts.UnsafeHTTPS {
		makeHTTPSUnsafe()
	}
	//Creating directory for downloads if it does not yet exist
	err := os.MkdirAll(opts.ImageDir, 0755)

	if err != nil { //Execute bit means different thing for directories that for files. And I was stupid.
		lFatal(err) //We can not create folder for images, end of line.
	}

	//	Creating channels to pass info to downloader and to signal job well done
	imgdat := make(ImageCh, opts.QDepth) //Better leave default queue depth. Experiment shown that depth about 20 provides optimal performance on my system

	if opts.Tag == "" { //Because we can put Image ID with flags. Why not?

		if len(opts.Args.IDs) == 1 {
			lInfo("Processing image №", opts.Args.IDs[0])
		} else {
			lInfo("Processing images №", debracket(opts.Args.IDs))
		}
		go imgdat.ParseImg(opts.Args.IDs, opts.Key) // Sending Image ID to parser. Here validity is our problem

	} else {

		// And here we send tags to getter/parser. Query and JSON validity is mostly server problem
		// Server response validity is ours
		lInfo("Processing tags", opts.Tag)
		go imgdat.ParseTag(opts.TagOpts, opts.Key)
	}

	lInfo("Starting worker") //It would be funny if worker goroutine does not start

	filterInit(opts.FiltOpts)           //Initiating filters based on our given flags
	filtimgdat := FilterChannel(imgdat) //Actual filtration

	sig := make(chan os.Signal, 1)
	signal.Notify(sig, os.Interrupt)

	filtimgdat.dispatch(sig).downloadImages(opts.Config) // Now that we got asynchronous list of images we want to get done, we can get them.

	lDone("Finished")
	//And we are done here! Hooray!
}

func (imgchan ImageCh) dispatch(sig <-chan os.Signal) (outch ImageCh) {
	outch = make(ImageCh)
	go imgchan.dispatcher(sig, outch)
	return outch
}

func (imgchan ImageCh) dispatcher(sig <-chan os.Signal, outch ImageCh) {
	for {
		select {
		case <-sig: //can't test this branch due to lDone killing our test
			close(outch)
			<-sig
			lDone("Download interrupted by user's command")
		default:
			select {
			case img, ok := <-imgchan:
				if !ok {
					close(outch)
					imgchan = nil
					return
				}
				outch <- img
			}
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"errors"
	"flag"
	"fmt"
	"go/ast"
	"go/build"
	"go/parser"
	"go/token"
	"go/types"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"
)

var (
	// allImports is a map of already-imported import paths to packages
	allImports map[string]*types.Package

	// ErrCheckErrors is returned by the checkFiles function if any errors were
	// encountered during checking.
	ErrCheckErrors = errors.New("found errors in checked files")
)

// Err prints an error to Stderr
func Err(s string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, "error: "+s+"\n", args...)
}

// Fatal calls Err followed by os.Exit(2)
func Fatal(s string, args ...interface{}) {
	Err(s, args...)
	os.Exit(2)
}

// regexpFlag is a type that can be used with flag.Var for regular expression flags
type regexpFlag struct {
	re *regexp.Regexp
}

func (r regexpFlag) String() string {
	if r.re == nil {
		return ""
	}
	return r.re.String()
}

func (r *regexpFlag) Set(s string) error {
	if s == "" {
		r.re = nil
		return nil
	}

	re, err := regexp.Compile(s)
	if err != nil {
		return err
	}
	r.re = re
	return nil
}

// stringsFlag is a type that can be used with flag.Var for lists that are turned to a set
type stringsFlag struct {
	items map[string]bool
}

func (f stringsFlag) String() string {
	items := make([]string, 0, len(f.items))
	for k := range f.items {
		items = append(items, k)
	}
	return strings.Join(items, ",")
}

func (f *stringsFlag) Set(s string) error {
	if f.items == nil {
		f.items = make(map[string]bool)
	}
	for _, item := range strings.Split(s, ",") {
		f.items[item] = true
	}
	return nil
}

func main() {
	allImports = make(map[string]*types.Package)

	var ignore regexpFlag
	flag.Var(&ignore, "ignore", "regular expression of function names to ignore")
	ignorePkg := &stringsFlag{}
	ignorePkg.Set("fmt")
	flag.Var(ignorePkg, "ignorepkg", "comma-separated list of package paths to ignore")
	flag.Parse()
	pkgName := flag.Arg(0)
	if pkgName == "" {
		flag.Usage()
		Fatal("you must specify a package")
	}

	files, err := getFiles(pkgName)
	if err != nil {
		Fatal("could not import %s: %s", pkgName, err)
	}

	if err := checkFiles(files, ignore.re, ignorePkg.items); err != nil {
		if err == ErrCheckErrors {
			os.Exit(1)
		}
		Fatal("failed to check package: %s", err)
	}
}

// getFiles returns all the Go files found at a package path
func getFiles(path string) ([]string, error) {
	ctx := build.Default
	ctx.CgoEnabled = false
	pkg, err := ctx.Import(path, ".", 0)
	if err != nil {
		return nil, err
	}

	files := make([]string, len(pkg.GoFiles))
	for i, fileName := range pkg.GoFiles {
		files[i] = filepath.Join(pkg.Dir, fileName)
	}
	return files, nil
}

type file struct {
	fset  *token.FileSet
	name  string
	ast   *ast.File
	lines [][]byte
}

func parseFile(fset *token.FileSet, fileName string) (f file, err error) {
	rd, err := os.Open(fileName)
	if err != nil {
		return f, err
	}
	defer rd.Close()

	data, err := ioutil.ReadAll(rd)
	if err != nil {
		return f, err
	}

	astFile, err := parser.ParseFile(fset, fileName, bytes.NewReader(data), parser.ParseComments)
	if err != nil {
		return f, fmt.Errorf("could not parse: %s", err)
	}

	lines := bytes.Split(data, []byte("\n"))
	f = file{fset: fset, name: fileName, ast: astFile, lines: lines}
	return f, nil
}

func typeCheck(fset *token.FileSet, astFiles []*ast.File) (map[*ast.CallExpr]types.Type, map[*ast.Ident]types.Object, error) {
	callTypes := make(map[*ast.CallExpr]types.Type)
	identObjs := make(map[*ast.Ident]types.Object)

	exprFn := func(x ast.Expr, typ types.Type, val interface{}) {
		call, ok := x.(*ast.CallExpr)
		if !ok {
			return
		}
		callTypes[call] = typ
	}
	identFn := func(id *ast.Ident, obj types.Object) {
		identObjs[id] = obj
	}
	context := types.Context{
		Expr:   exprFn,
		Ident:  identFn,
		Import: importer,
	}
	_, err := context.Check(fset, astFiles)
	return callTypes, identObjs, err
}

type checker struct {
	fset      *token.FileSet
	files     map[string]file
	callTypes map[*ast.CallExpr]types.Type
	identObjs map[*ast.Ident]types.Object
	ignore    *regexp.Regexp
	ignorePkg map[string]bool

	errors []error
}

type uncheckedErr struct {
	pos  token.Position
	line []byte
}

func (e uncheckedErr) Error() string {
	return fmt.Sprintf("%s %s", e.pos, e.line)
}

func (c *checker) Visit(node ast.Node) ast.Visitor {
	n, ok := node.(*ast.ExprStmt)
	if !ok {
		return c
	}

	// Check for a call expression
	call, ok := n.X.(*ast.CallExpr)
	if !ok {
		return c
	}

	var id *ast.Ident
	switch exp := call.Fun.(type) {
	case (*ast.Ident):
		id = exp
	case (*ast.SelectorExpr):
		id = exp.Sel
	default:
		fmt.Fprintf(os.Stderr, "unknown call: %T %+v\n", exp, exp)
		return c
	}

	// Ignore if in an ignored package
	if obj := c.identObjs[id]; obj != nil {
		if pkg := obj.GetPkg(); pkg != nil && c.ignorePkg[pkg.Path] {
			return c
		}
	}
	callType := c.callTypes[call]

	// Ignore if a name matches the regexp
	if c.ignore != nil && c.ignore.MatchString(id.Name) {
		return c
	}

	unchecked := false
	switch t := callType.(type) {
	case *types.NamedType:
		// Single return
		if isErrorType(t.Obj) {
			unchecked = true
		}
	case *types.Result:
		// Multiple returns
		for _, v := range t.Values {
			nt, ok := v.Type.(*types.NamedType)
			if !ok {
				continue
			}
			if isErrorType(nt.Obj) {
				unchecked = true
				break
			}
		}
	}

	if unchecked {
		pos := c.fset.Position(id.NamePos)
		c.errors = append(c.errors, uncheckedErr{pos, c.files[pos.Filename].lines[pos.Line-1]})
	}
	return c
}

func checkFiles(fileNames []string, ignore *regexp.Regexp, ignorePkg map[string]bool) error {
	fset := token.NewFileSet()
	astFiles := make([]*ast.File, len(fileNames))
	files := make(map[string]file, len(fileNames))

	for i, fileName := range fileNames {
		f, err := parseFile(fset, fileName)
		if err != nil {
			return fmt.Errorf("could not parse %s: %s", fileName, err)
		}
		files[fileName] = f
		astFiles[i] = f.ast
	}

	callTypes, identObjs, err := typeCheck(fset, astFiles)
	if err != nil {
		return fmt.Errorf("could not type check: %s", err)
	}

	visitor := &checker{fset, files, callTypes, identObjs, ignore, ignorePkg, []error{}}
	for _, astFile := range astFiles {
		ast.Walk(visitor, astFile)
	}

	for _, e := range visitor.errors {
		fmt.Fprintln(os.Stderr, e)
	}

	if len(visitor.errors) > 0 {
		return ErrCheckErrors
	}
	return nil
}

type obj interface {
	GetPkg() *types.Package
	GetName() string
}

func isErrorType(v obj) bool {
	return v.GetPkg() == nil && v.GetName() == "error"
}

func importer(imports map[string]*types.Package, path string) (pkg *types.Package, err error) {
	// types.Importer does not seem to be designed for recursive
	// parsing like we're doing here. Specifically, each nested import
	// will maintain its own imports map. This will lead to duplicate
	// imports and in turn packages, which will lead to funny errors
	// such as "cannot pass argument ip (variable of type net.IP) to
	// variable of type net.IP"
	//
	// To work around this, we keep a global imports map, allImports,
	// to which we add all nested imports, and which we use as the
	// cache, instead of imports.
	//
	// Since all nested imports will also use this importer, there
	// should be no way to end up with duplicate imports.

	// We first try to use GcImport directly. This has the downside of
	// using possibly out-of-date packages, but it has the upside of
	// not having to parse most of the Go standard library.

	buildPkg, buildErr := build.Import(path, ".", 0)

	// If we found no build dir, assume we're dealing with installed
	// but no source. If we found a build dir, only use GcImport if
	// it's in GOROOT. This way we always use up-to-date code for
	// normal packages but avoid parsing the standard library.
	if (buildErr == nil && buildPkg.Goroot) || buildErr != nil {
		pkg, err = types.GcImport(allImports, path)
		if err == nil {
			// We don't use imports, but per API we have to add the package.
			imports[pkg.Path] = pkg
			allImports[pkg.Path] = pkg
			return pkg, nil
		}
	}

	// See if we already imported this package
	if pkg = allImports[path]; pkg != nil && pkg.Complete {
		return pkg, nil
	}

	// allImports failed, try to use go/build
	if buildErr != nil {
		return nil, buildErr
	}

	fileSet := token.NewFileSet()

	isGoFile := func(d os.FileInfo) bool {
		allFiles := make([]string, 0, len(buildPkg.GoFiles)+len(buildPkg.CgoFiles))
		allFiles = append(allFiles, buildPkg.GoFiles...)
		allFiles = append(allFiles, buildPkg.CgoFiles...)

		for _, file := range allFiles {
			if file == d.Name() {
				return true
			}
		}
		return false
	}
	pkgs, err := parser.ParseDir(fileSet, buildPkg.Dir, isGoFile, 0)
	if err != nil {
		return nil, err
	}

	delete(pkgs, "documentation")
	var astPkg *ast.Package
	var name string
	for name, astPkg = range pkgs {
		// Use the first non-main package, or the only package we
		// found.
		//
		// NOTE(dh) I can't think of a reason why there should be
		// multiple packages in a single directory, but ParseDir
		// accommodates for that possibility.
		if len(pkgs) == 1 || name != "main" {
			break
		}
	}

	if astPkg == nil {
		return nil, fmt.Errorf("can't find import: %s", name)
	}

	var ff []*ast.File
	for _, f := range astPkg.Files {
		ff = append(ff, f)
	}

	context := types.Context{
		Import: importer,
	}

	pkg, err = context.Check(fileSet, ff)
	if err != nil {
		return pkg, err
	}

	// We don't use imports, but per API we have to add the package.
	imports[path] = pkg
	allImports[path] = pkg
	pkg.Complete = true
	return pkg, nil
}
<commit_msg>Add fallback of interpreting an import path as a directory path<commit_after>package main

import (
	"bytes"
	"errors"
	"flag"
	"fmt"
	"go/ast"
	"go/build"
	"go/parser"
	"go/token"
	"go/types"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"
)

var (
	// allImports is a map of already-imported import paths to packages
	allImports map[string]*types.Package

	// ErrCheckErrors is returned by the checkFiles function if any errors were
	// encountered during checking.
	ErrCheckErrors = errors.New("found errors in checked files")
)

// Err prints an error to Stderr
func Err(s string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, "error: "+s+"\n", args...)
}

// Fatal calls Err followed by os.Exit(2)
func Fatal(s string, args ...interface{}) {
	Err(s, args...)
	os.Exit(2)
}

// regexpFlag is a type that can be used with flag.Var for regular expression flags
type regexpFlag struct {
	re *regexp.Regexp
}

func (r regexpFlag) String() string {
	if r.re == nil {
		return ""
	}
	return r.re.String()
}

func (r *regexpFlag) Set(s string) error {
	if s == "" {
		r.re = nil
		return nil
	}

	re, err := regexp.Compile(s)
	if err != nil {
		return err
	}
	r.re = re
	return nil
}

// stringsFlag is a type that can be used with flag.Var for lists that are turned to a set
type stringsFlag struct {
	items map[string]bool
}

func (f stringsFlag) String() string {
	items := make([]string, 0, len(f.items))
	for k := range f.items {
		items = append(items, k)
	}
	return strings.Join(items, ",")
}

func (f *stringsFlag) Set(s string) error {
	if f.items == nil {
		f.items = make(map[string]bool)
	}
	for _, item := range strings.Split(s, ",") {
		f.items[item] = true
	}
	return nil
}

func main() {
	allImports = make(map[string]*types.Package)

	var ignore regexpFlag
	flag.Var(&ignore, "ignore", "regular expression of function names to ignore")
	ignorePkg := &stringsFlag{}
	ignorePkg.Set("fmt")
	flag.Var(ignorePkg, "ignorepkg", "comma-separated list of package paths to ignore")
	flag.Parse()

	pkgName := flag.Arg(0)
	if pkgName == "" {
		flag.Usage()
		Fatal("you must specify a package")
	}

	var err1, err2 error

	// First try to treat pkgName as import path...
	pkg, err1 := importPathToPkg(pkgName)
	if err1 != nil {
		// ... then attempt as file path
		pkg, err2 = directoryToPkg(pkgName)
	}

	if err2 != nil {
		// Print both errors so the user can see in what ways the
		// package lookup failed.
		Fatal("could not import %s: %s\n%s", pkgName, err1, err2)
	}

	files := getFiles(pkg)

	if err := checkFiles(files, ignore.re, ignorePkg.items); err != nil {
		if err == ErrCheckErrors {
			os.Exit(1)
		}
		Fatal("failed to check package: %s", err)
	}
}

// directoryToPkg returns a Package found in a directory
func directoryToPkg(directory string) (*build.Package, error) {
	ctx := build.Default
	ctx.CgoEnabled = false
	pkg, err := ctx.ImportDir(directory, 0)
	return pkg, err
}

// importPathToPkg returns a Package found at a package path
func importPathToPkg(path string) (*build.Package, error) {
	ctx := build.Default
	ctx.CgoEnabled = false
	pkg, err := ctx.Import(path, ".", 0)
	return pkg, err
}

// getFiles returns all the Go files found in a package
func getFiles(pkg *build.Package) []string {
	files := make([]string, len(pkg.GoFiles))
	for i, fileName := range pkg.GoFiles {
		files[i] = filepath.Join(pkg.Dir, fileName)
	}
	return files
}

type file struct {
	fset  *token.FileSet
	name  string
	ast   *ast.File
	lines [][]byte
}

func parseFile(fset *token.FileSet, fileName string) (f file, err error) {
	rd, err := os.Open(fileName)
	if err != nil {
		return f, err
	}
	defer rd.Close()

	data, err := ioutil.ReadAll(rd)
	if err != nil {
		return f, err
	}

	astFile, err := parser.ParseFile(fset, fileName, bytes.NewReader(data), parser.ParseComments)
	if err != nil {
		return f, fmt.Errorf("could not parse: %s", err)
	}

	lines := bytes.Split(data, []byte("\n"))
	f = file{fset: fset, name: fileName, ast: astFile, lines: lines}
	return f, nil
}

func typeCheck(fset *token.FileSet, astFiles []*ast.File) (map[*ast.CallExpr]types.Type, map[*ast.Ident]types.Object, error) {
	callTypes := make(map[*ast.CallExpr]types.Type)
	identObjs := make(map[*ast.Ident]types.Object)

	exprFn := func(x ast.Expr, typ types.Type, val interface{}) {
		call, ok := x.(*ast.CallExpr)
		if !ok {
			return
		}
		callTypes[call] = typ
	}
	identFn := func(id *ast.Ident, obj types.Object) {
		identObjs[id] = obj
	}
	context := types.Context{
		Expr:   exprFn,
		Ident:  identFn,
		Import: importer,
	}
	_, err := context.Check(fset, astFiles)
	return callTypes, identObjs, err
}

type checker struct {
	fset      *token.FileSet
	files     map[string]file
	callTypes map[*ast.CallExpr]types.Type
	identObjs map[*ast.Ident]types.Object
	ignore    *regexp.Regexp
	ignorePkg map[string]bool

	errors []error
}

type uncheckedErr struct {
	pos  token.Position
	line []byte
}

func (e uncheckedErr) Error() string {
	return fmt.Sprintf("%s %s", e.pos, e.line)
}

func (c *checker) Visit(node ast.Node) ast.Visitor {
	n, ok := node.(*ast.ExprStmt)
	if !ok {
		return c
	}

	// Check for a call expression
	call, ok := n.X.(*ast.CallExpr)
	if !ok {
		return c
	}

	var id *ast.Ident
	switch exp := call.Fun.(type) {
	case (*ast.Ident):
		id = exp
	case (*ast.SelectorExpr):
		id = exp.Sel
	default:
		fmt.Fprintf(os.Stderr, "unknown call: %T %+v\n", exp, exp)
		return c
	}

	// Ignore if in an ignored package
	if obj := c.identObjs[id]; obj != nil {
		if pkg := obj.GetPkg(); pkg != nil && c.ignorePkg[pkg.Path] {
			return c
		}
	}
	callType := c.callTypes[call]

	// Ignore if a name matches the regexp
	if c.ignore != nil && c.ignore.MatchString(id.Name) {
		return c
	}

	unchecked := false
	switch t := callType.(type) {
	case *types.NamedType:
		// Single return
		if isErrorType(t.Obj) {
			unchecked = true
		}
	case *types.Result:
		// Multiple returns
		for _, v := range t.Values {
			nt, ok := v.Type.(*types.NamedType)
			if !ok {
				continue
			}
			if isErrorType(nt.Obj) {
				unchecked = true
				break
			}
		}
	}

	if unchecked {
		pos := c.fset.Position(id.NamePos)
		c.errors = append(c.errors, uncheckedErr{pos, c.files[pos.Filename].lines[pos.Line-1]})
	}
	return c
}

func checkFiles(fileNames []string, ignore *regexp.Regexp, ignorePkg map[string]bool) error {
	fset := token.NewFileSet()
	astFiles := make([]*ast.File, len(fileNames))
	files := make(map[string]file, len(fileNames))

	for i, fileName := range fileNames {
		f, err := parseFile(fset, fileName)
		if err != nil {
			return fmt.Errorf("could not parse %s: %s", fileName, err)
		}
		files[fileName] = f
		astFiles[i] = f.ast
	}

	callTypes, identObjs, err := typeCheck(fset, astFiles)
	if err != nil {
		return fmt.Errorf("could not type check: %s", err)
	}

	visitor := &checker{fset, files, callTypes, identObjs, ignore, ignorePkg, []error{}}
	for _, astFile := range astFiles {
		ast.Walk(visitor, astFile)
	}

	for _, e := range visitor.errors {
		fmt.Fprintln(os.Stderr, e)
	}

	if len(visitor.errors) > 0 {
		return ErrCheckErrors
	}
	return nil
}

type obj interface {
	GetPkg() *types.Package
	GetName() string
}

func isErrorType(v obj) bool {
	return v.GetPkg() == nil && v.GetName() == "error"
}

func importer(imports map[string]*types.Package, path string) (pkg *types.Package, err error) {
	// types.Importer does not seem to be designed for recursive
	// parsing like we're doing here. Specifically, each nested import
	// will maintain its own imports map. This will lead to duplicate
	// imports and in turn packages, which will lead to funny errors
	// such as "cannot pass argument ip (variable of type net.IP) to
	// variable of type net.IP"
	//
	// To work around this, we keep a global imports map, allImports,
	// to which we add all nested imports, and which we use as the
	// cache, instead of imports.
	//
	// Since all nested imports will also use this importer, there
	// should be no way to end up with duplicate imports.

	// We first try to use GcImport directly. This has the downside of
	// using possibly out-of-date packages, but it has the upside of
	// not having to parse most of the Go standard library.

	buildPkg, buildErr := build.Import(path, ".", 0)

	// If we found no build dir, assume we're dealing with installed
	// but no source. If we found a build dir, only use GcImport if
	// it's in GOROOT. This way we always use up-to-date code for
	// normal packages but avoid parsing the standard library.
	if (buildErr == nil && buildPkg.Goroot) || buildErr != nil {
		pkg, err = types.GcImport(allImports, path)
		if err == nil {
			// We don't use imports, but per API we have to add the package.
			imports[pkg.Path] = pkg
			allImports[pkg.Path] = pkg
			return pkg, nil
		}
	}

	// See if we already imported this package
	if pkg = allImports[path]; pkg != nil && pkg.Complete {
		return pkg, nil
	}

	// allImports failed, try to use go/build
	if buildErr != nil {
		return nil, buildErr
	}

	fileSet := token.NewFileSet()

	isGoFile := func(d os.FileInfo) bool {
		allFiles := make([]string, 0, len(buildPkg.GoFiles)+len(buildPkg.CgoFiles))
		allFiles = append(allFiles, buildPkg.GoFiles...)
		allFiles = append(allFiles, buildPkg.CgoFiles...)

		for _, file := range allFiles {
			if file == d.Name() {
				return true
			}
		}
		return false
	}
	pkgs, err := parser.ParseDir(fileSet, buildPkg.Dir, isGoFile, 0)
	if err != nil {
		return nil, err
	}

	delete(pkgs, "documentation")
	var astPkg *ast.Package
	var name string
	for name, astPkg = range pkgs {
		// Use the first non-main package, or the only package we
		// found.
		//
		// NOTE(dh) I can't think of a reason why there should be
		// multiple packages in a single directory, but ParseDir
		// accommodates for that possibility.
		if len(pkgs) == 1 || name != "main" {
			break
		}
	}

	if astPkg == nil {
		return nil, fmt.Errorf("can't find import: %s", name)
	}

	var ff []*ast.File
	for _, f := range astPkg.Files {
		ff = append(ff, f)
	}

	context := types.Context{
		Import: importer,
	}

	pkg, err = context.Check(fileSet, ff)
	if err != nil {
		return pkg, err
	}

	// We don't use imports, but per API we have to add the package.
	imports[path] = pkg
	allImports[path] = pkg
	pkg.Complete = true
	return pkg, nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"github.com/ChimeraCoder/anaconda"
	"github.com/otiai10/twistream"
	"io/ioutil"
	"regexp"
)

type Config struct {
	ConsumerKey       string `json:"consumerKey"`
	ConsumerSecret    string `json:"consumerSecret"`
	AccessToken       string `json:"accessToken"`
	AccessTokenSecret string `json:"accessTokenSecret"`
}

func main() {
	file, err := ioutil.ReadFile("config.json")
	if err != nil {
		panic(err)
	}
	var config Config
	json.Unmarshal(file, &config)

	consumerKey := config.ConsumerKey
	consumerSecret := config.ConsumerSecret
	accessToken := config.AccessToken
	accessTokenSecret := config.AccessTokenSecret
	timeline, _ := twistream.New(
		"https://userstream.twitter.com/1.1/user.json",
		consumerKey,
		consumerSecret,
		accessToken,
		accessTokenSecret,
	)

	anaconda.SetConsumerKey(consumerKey)
	anaconda.SetConsumerSecret(consumerSecret)
	api := anaconda.NewTwitterApi(accessToken, accessTokenSecret)

	for {
		status := <-timeline.Listen()

		if IsContainSushi(status.Text) && !status.Favorited && !status.Retweeted {
			api.Favorite(status.Id)
		}
		fmt.Println(status.Text)
	}
}

func IsContainSushi(text string) (b bool) {
	if m, _ := regexp.MatchString("寿司|スシ|鮨|寿し|🍣|[sS][uU][sS][hH][iI]", text); !m {
		return false
	}
	return true
}
<commit_msg>Revert "favとRTには反応しないようにした"<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"github.com/ChimeraCoder/anaconda"
	"github.com/otiai10/twistream"
	"io/ioutil"
	"regexp"
)

type Config struct {
	ConsumerKey       string `json:"consumerKey"`
	ConsumerSecret    string `json:"consumerSecret"`
	AccessToken       string `json:"accessToken"`
	AccessTokenSecret string `json:"accessTokenSecret"`
}

func main() {
	file, err := ioutil.ReadFile("config.json")
	if err != nil {
		panic(err)
	}
	var config Config
	json.Unmarshal(file, &config)

	consumerKey := config.ConsumerKey
	consumerSecret := config.ConsumerSecret
	accessToken := config.AccessToken
	accessTokenSecret := config.AccessTokenSecret
	timeline, _ := twistream.New(
		"https://userstream.twitter.com/1.1/user.json",
		consumerKey,
		consumerSecret,
		accessToken,
		accessTokenSecret,
	)

	anaconda.SetConsumerKey(consumerKey)
	anaconda.SetConsumerSecret(consumerSecret)
	api := anaconda.NewTwitterApi(accessToken, accessTokenSecret)

	for {
		status := <-timeline.Listen()

		if IsContainSushi(status.Text) {
			api.Favorite(status.Id)
		}
		fmt.Println(status.Text)
	}
}

func IsContainSushi(text string) (b bool) {
	if m, _ := regexp.MatchString("寿司|スシ|鮨|寿し|🍣|[sS][uU][sS][hH][iI]", text); !m {
		return false
	}
	return true
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/hmac"
	"crypto/md5"
	"crypto/sha1"
	"crypto/subtle"
	"encoding/base64"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"path"
	"strconv"
	"strings"
	"time"

	"github.com/opendoor-labs/gothumb/Godeps/_workspace/src/github.com/DAddYE/vips"
	"github.com/opendoor-labs/gothumb/Godeps/_workspace/src/github.com/julienschmidt/httprouter"
	"github.com/opendoor-labs/gothumb/Godeps/_workspace/src/github.com/rlmcpherson/s3gof3r"
)

var (
	maxAge           int
	securityKey      []byte
	resultBucketName string
	useRRS           bool

	httpClient   *http.Client
	resultBucket *s3gof3r.Bucket
)

func main() {
	log.SetFlags(0) // hide timestamps from Go logs
	securityKey = []byte(mustGetenv("SECURITY_KEY"))
	resultBucketName = mustGetenv("RESULT_STORAGE_BUCKET")

	if maxAgeStr := os.Getenv("MAX_AGE"); maxAgeStr != "" {
		var err error
		if maxAge, err = strconv.Atoi(maxAgeStr); err != nil {
			log.Fatal("invalid MAX_AGE setting")
		}
	}
	if rrs := os.Getenv("USE_RRS"); rrs == "true" || rrs == "1" {
		useRRS = true
	}

	keys, err := s3gof3r.EnvKeys()
	if err != nil {
		log.Fatal(err)
	}
	resultBucket = s3gof3r.New(s3gof3r.DefaultDomain, keys).Bucket(resultBucketName)
	resultBucket.Md5Check = false
	httpClient = resultBucket.Client

	router := httprouter.New()
	router.HEAD("/:signature/:size/*source", handleResize)
	router.GET("/:signature/:size/*source", handleResize)
	port := os.Getenv("PORT")
	if port == "" {
		port = "8888"
	}
	log.Fatal(http.ListenAndServe(":"+port, router))
}

func handleResize(w http.ResponseWriter, req *http.Request, params httprouter.Params) {
	log.Printf(req.Method + " " + req.URL.Path)
	sourceURL, err := url.Parse(strings.TrimPrefix(params.ByName("source"), "/"))
	if err != nil || !(sourceURL.Scheme == "http" || sourceURL.Scheme == "https") {
		http.Error(w, "invalid source URL", 400)
		return
	}

	sig := params.ByName("signature")
	pathToVerify := strings.TrimPrefix(req.URL.Path, "/"+sig+"/")
	if err := validateSignature(sig, pathToVerify); err != nil {
		http.Error(w, "invalid signature", 401)
		return
	}

	width, height, err := parseWidthAndHeight(params.ByName("size"))
	if err != nil {
		http.Error(w, "invalid height requested", 400)
		return
	}

	path := normalizePath(req.URL.Path)

	// try to get stored result
	r, h, err := getStoredResult(req.Method, path)
	if err != nil {
		log.Printf("getting stored result: %s", err)
		generateThumbnail(w, req.Method, path, sourceURL.String(), width, height)
		return
	}
	defer r.Close()

	// return stored result
	length, err := strconv.Atoi(h.Get("Content-Length"))
	if err != nil {
		log.Printf("invalid result content-length: %s", err)
		// TODO: try to generate instead of erroring w/ 500?
		http.Error(w, err.Error(), 500)
		return
	}

	setResultHeaders(w, &result{
		ContentType:   "image/jpeg", // TODO: use stored content type
		ContentLength: length,
		ETag:          strings.Trim(h.Get("Etag"), `"`),
		Path:          path,
	})
	if _, err = io.Copy(w, r); err != nil {
		log.Printf("copying from stored result: %s", err)
		http.Error(w, err.Error(), 500)
		return
	}
	if err = r.Close(); err != nil {
		log.Printf("closing stored result copy: %s", err)
	}
}

type result struct {
	Data          []byte
	ContentType   string
	ContentLength int
	ETag          string
	Path          string
}

func computeHexMD5(data []byte) string {
	h := md5.New()
	h.Write(data)
	return fmt.Sprintf("%x", h.Sum(nil))
}

func copyHeader(dst, src http.Header) {
	for k, vv := range src {
		for _, v := range vv {
			dst.Add(k, v)
		}
	}
}

func generateThumbnail(w http.ResponseWriter, rmethod, rpath string, sourceURL string, width, height uint) {
	log.Printf("generating %s", rpath)
	resp, err := httpClient.Get(sourceURL)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		copyHeader(w.Header(), resp.Header)
		io.Copy(w, resp.Body)
		return
	}

	contentType := resp.Header.Get("Content-Type")
	if !strings.HasPrefix(contentType, "image/") {
		http.Error(w, fmt.Sprintf("invalid content type %q", contentType), 500)
		return
	}

	img, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	buf, err := vips.Resize(img, vips.Options{
		Height:       int(height),
		Width:        int(width),
		Crop:         true,
		Interpolator: vips.BICUBIC,
		Gravity:      vips.CENTRE,
		Quality:      95,
	})
	if err != nil {
		http.Error(w, fmt.Sprintf("resizing image: %s", err.Error()), 500)
		return
	}

	res := &result{
		ContentType:   "image/jpeg",
		ContentLength: len(buf),
		Data:          buf, // TODO: check if I need to copy this
		ETag:          computeHexMD5(buf),
		Path:          rpath,
	}
	setResultHeaders(w, res)
	if rmethod != "HEAD" {
		if _, err = w.Write(buf); err != nil {
			log.Printf("writing buffer to response: %s", err)
		}
	}

	go storeResult(res)
}

// caller is responsible for closing the returned ReadCloser
func getStoredResult(method, path string) (io.ReadCloser, http.Header, error) {
	if method != "HEAD" {
		return resultBucket.GetReader(path, nil)
	}

	s3URL := fmt.Sprintf("https://%s.s3.amazonaws.com%s", resultBucketName, path)
	req, err := http.NewRequest(method, s3URL, nil)
	if err != nil {
		return nil, nil, err
	}

	resultBucket.Sign(req)
	res, err := httpClient.Do(req)
	if err != nil {
		return nil, nil, err
	}
	if res.StatusCode < 200 || res.StatusCode >= 300 {
		// TODO: drain res.Body to ioutil.Discard before closing?
		res.Body.Close()
		return nil, nil, fmt.Errorf("unexpected status code %d", res.StatusCode)
	}
	res.Header.Set("Content-Length", strconv.FormatInt(res.ContentLength, 10))
	return res.Body, res.Header, err
}

func mustGetenv(name string) string {
	value := os.Getenv(name)
	if value == "" {
		log.Fatalf("missing %s env", name)
	}
	return value
}

func normalizePath(p string) string {
	// TODO(bgentry): Support for custom root path? ala RESULT_STORAGE_AWS_STORAGE_ROOT_PATH
	return path.Clean(p)
}

func parseWidthAndHeight(str string) (width, height uint, err error) {
	sizeParts := strings.Split(str, "x")
	if len(sizeParts) != 2 {
		err = fmt.Errorf("invalid size requested")
		return
	}
	width64, err := strconv.ParseUint(sizeParts[0], 10, 64)
	if err != nil {
		err = fmt.Errorf("invalid width requested")
		return
	}
	height64, err := strconv.ParseUint(sizeParts[1], 10, 64)
	if err != nil {
		err = fmt.Errorf("invalid height requested")
		return
	}
	return uint(width64), uint(height64), nil
}

func setCacheHeaders(w http.ResponseWriter) {
	w.Header().Set("Cache-Control", fmt.Sprintf("max-age=%d,public", maxAge))
	w.Header().Set("Expires", time.Now().UTC().Add(time.Duration(maxAge)*time.Second).Format(http.TimeFormat))
}

func setResultHeaders(w http.ResponseWriter, result *result) {
	w.Header().Set("Content-Type", result.ContentType)
	w.Header().Set("Content-Length", strconv.Itoa(result.ContentLength))
	w.Header().Set("ETag", `"`+result.ETag+`"`)
	setCacheHeaders(w)
}

func storeResult(res *result) {
	h := make(http.Header)
	h.Set("Content-Type", res.ContentType)
	if useRRS {
		h.Set("x-amz-storage-class", "REDUCED_REDUNDANCY")
	}
	w, err := resultBucket.PutWriter(res.Path, h, nil)
	if err != nil {
		log.Printf("storing result for %s: %s", res.Path, err)
		return
	}
	defer w.Close()
	if _, err = w.Write(res.Data); err != nil {
		log.Printf("storing result for %s: %s", res.Path, err)
		return
	}
	if err = w.Close(); err != nil {
		log.Printf("storing result for %s: %s", res.Path, err)
	}
}

func validateSignature(sig, pathPart string) error {
	h := hmac.New(sha1.New, securityKey)
	if _, err := h.Write([]byte(pathPart)); err != nil {
		return err
	}
	actualSig := base64.URLEncoding.EncodeToString(h.Sum(nil))
	// constant-time string comparison
	if subtle.ConstantTimeCompare([]byte(sig), []byte(actualSig)) != 1 {
		return fmt.Errorf("signature mismatch")
	}
	return nil
}
<commit_msg>remove signature from result storage path<commit_after>package main

import (
	"crypto/hmac"
	"crypto/md5"
	"crypto/sha1"
	"crypto/subtle"
	"encoding/base64"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"path"
	"strconv"
	"strings"
	"time"

	"github.com/opendoor-labs/gothumb/Godeps/_workspace/src/github.com/DAddYE/vips"
	"github.com/opendoor-labs/gothumb/Godeps/_workspace/src/github.com/julienschmidt/httprouter"
	"github.com/opendoor-labs/gothumb/Godeps/_workspace/src/github.com/rlmcpherson/s3gof3r"
)

var (
	maxAge           int
	securityKey      []byte
	resultBucketName string
	useRRS           bool

	httpClient   *http.Client
	resultBucket *s3gof3r.Bucket
)

func main() {
	log.SetFlags(0) // hide timestamps from Go logs
	securityKey = []byte(mustGetenv("SECURITY_KEY"))
	resultBucketName = mustGetenv("RESULT_STORAGE_BUCKET")

	if maxAgeStr := os.Getenv("MAX_AGE"); maxAgeStr != "" {
		var err error
		if maxAge, err = strconv.Atoi(maxAgeStr); err != nil {
			log.Fatal("invalid MAX_AGE setting")
		}
	}
	if rrs := os.Getenv("USE_RRS"); rrs == "true" || rrs == "1" {
		useRRS = true
	}

	keys, err := s3gof3r.EnvKeys()
	if err != nil {
		log.Fatal(err)
	}
	resultBucket = s3gof3r.New(s3gof3r.DefaultDomain, keys).Bucket(resultBucketName)
	resultBucket.Md5Check = false
	httpClient = resultBucket.Client

	router := httprouter.New()
	router.HEAD("/:signature/:size/*source", handleResize)
	router.GET("/:signature/:size/*source", handleResize)
	port := os.Getenv("PORT")
	if port == "" {
		port = "8888"
	}
	log.Fatal(http.ListenAndServe(":"+port, router))
}

func handleResize(w http.ResponseWriter, req *http.Request, params httprouter.Params) {
	log.Printf(req.Method + " " + req.URL.Path)
	sourceURL, err := url.Parse(strings.TrimPrefix(params.ByName("source"), "/"))
	if err != nil || !(sourceURL.Scheme == "http" || sourceURL.Scheme == "https") {
		http.Error(w, "invalid source URL", 400)
		return
	}

	sig := params.ByName("signature")
	pathToVerify := strings.TrimPrefix(req.URL.Path, "/"+sig+"/")
	if err := validateSignature(sig, pathToVerify); err != nil {
		http.Error(w, "invalid signature", 401)
		return
	}

	width, height, err := parseWidthAndHeight(params.ByName("size"))
	if err != nil {
		http.Error(w, "invalid height requested", 400)
		return
	}

	resultPath := normalizePath(strings.TrimPrefix(req.URL.Path, "/"+sig))

	// try to get stored result
	r, h, err := getStoredResult(req.Method, resultPath)
	if err != nil {
		log.Printf("getting stored result: %s", err)
		generateThumbnail(w, req.Method, resultPath, sourceURL.String(), width, height)
		return
	}
	defer r.Close()

	// return stored result
	length, err := strconv.Atoi(h.Get("Content-Length"))
	if err != nil {
		log.Printf("invalid result content-length: %s", err)
		// TODO: try to generate instead of erroring w/ 500?
		http.Error(w, err.Error(), 500)
		return
	}

	setResultHeaders(w, &result{
		ContentType:   "image/jpeg", // TODO: use stored content type
		ContentLength: length,
		ETag:          strings.Trim(h.Get("Etag"), `"`),
		Path:          resultPath,
	})
	if _, err = io.Copy(w, r); err != nil {
		log.Printf("copying from stored result: %s", err)
		http.Error(w, err.Error(), 500)
		return
	}
	if err = r.Close(); err != nil {
		log.Printf("closing stored result copy: %s", err)
	}
}

type result struct {
	Data          []byte
	ContentType   string
	ContentLength int
	ETag          string
	Path          string
}

func computeHexMD5(data []byte) string {
	h := md5.New()
	h.Write(data)
	return fmt.Sprintf("%x", h.Sum(nil))
}

func copyHeader(dst, src http.Header) {
	for k, vv := range src {
		for _, v := range vv {
			dst.Add(k, v)
		}
	}
}

func generateThumbnail(w http.ResponseWriter, rmethod, rpath string, sourceURL string, width, height uint) {
	log.Printf("generating %s", rpath)
	resp, err := httpClient.Get(sourceURL)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		copyHeader(w.Header(), resp.Header)
		io.Copy(w, resp.Body)
		return
	}

	contentType := resp.Header.Get("Content-Type")
	if !strings.HasPrefix(contentType, "image/") {
		http.Error(w, fmt.Sprintf("invalid content type %q", contentType), 500)
		return
	}

	img, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	buf, err := vips.Resize(img, vips.Options{
		Height:       int(height),
		Width:        int(width),
		Crop:         true,
		Interpolator: vips.BICUBIC,
		Gravity:      vips.CENTRE,
		Quality:      95,
	})
	if err != nil {
		http.Error(w, fmt.Sprintf("resizing image: %s", err.Error()), 500)
		return
	}

	res := &result{
		ContentType:   "image/jpeg", // TODO: use stored content-type
		ContentLength: len(buf),
		Data:          buf, // TODO: check if I need to copy this
		ETag:          computeHexMD5(buf),
		Path:          rpath,
	}
	setResultHeaders(w, res)
	if rmethod != "HEAD" {
		if _, err = w.Write(buf); err != nil {
			log.Printf("writing buffer to response: %s", err)
		}
	}

	go storeResult(res)
}

// caller is responsible for closing the returned ReadCloser
func getStoredResult(method, path string) (io.ReadCloser, http.Header, error) {
	if method != "HEAD" {
		return resultBucket.GetReader(path, nil)
	}

	s3URL := fmt.Sprintf("https://%s.s3.amazonaws.com%s", resultBucketName, path)
	req, err := http.NewRequest(method, s3URL, nil)
	if err != nil {
		return nil, nil, err
	}

	resultBucket.Sign(req)
	res, err := httpClient.Do(req)
	if err != nil {
		return nil, nil, err
	}
	if res.StatusCode < 200 || res.StatusCode >= 300 {
		// TODO: drain res.Body to ioutil.Discard before closing?
		res.Body.Close()
		return nil, nil, fmt.Errorf("unexpected status code %d", res.StatusCode)
	}
	res.Header.Set("Content-Length", strconv.FormatInt(res.ContentLength, 10))
	return res.Body, res.Header, err
}

func mustGetenv(name string) string {
	value := os.Getenv(name)
	if value == "" {
		log.Fatalf("missing %s env", name)
	}
	return value
}

func normalizePath(p string) string {
	// TODO(bgentry): Support for custom root path? ala RESULT_STORAGE_AWS_STORAGE_ROOT_PATH
	return path.Clean(p)
}

func parseWidthAndHeight(str string) (width, height uint, err error) {
	sizeParts := strings.Split(str, "x")
	if len(sizeParts) != 2 {
		err = fmt.Errorf("invalid size requested")
		return
	}
	width64, err := strconv.ParseUint(sizeParts[0], 10, 64)
	if err != nil {
		err = fmt.Errorf("invalid width requested")
		return
	}
	height64, err := strconv.ParseUint(sizeParts[1], 10, 64)
	if err != nil {
		err = fmt.Errorf("invalid height requested")
		return
	}
	return uint(width64), uint(height64), nil
}

func setCacheHeaders(w http.ResponseWriter) {
	w.Header().Set("Cache-Control", fmt.Sprintf("max-age=%d,public", maxAge))
	w.Header().Set("Expires", time.Now().UTC().Add(time.Duration(maxAge)*time.Second).Format(http.TimeFormat))
}

func setResultHeaders(w http.ResponseWriter, result *result) {
	w.Header().Set("Content-Type", result.ContentType)
	w.Header().Set("Content-Length", strconv.Itoa(result.ContentLength))
	w.Header().Set("ETag", `"`+result.ETag+`"`)
	setCacheHeaders(w)
}

func storeResult(res *result) {
	h := make(http.Header)
	h.Set("Content-Type", res.ContentType)
	if useRRS {
		h.Set("x-amz-storage-class", "REDUCED_REDUNDANCY")
	}
	w, err := resultBucket.PutWriter(res.Path, h, nil)
	if err != nil {
		log.Printf("storing result for %s: %s", res.Path, err)
		return
	}
	defer w.Close()
	if _, err = w.Write(res.Data); err != nil {
		log.Printf("storing result for %s: %s", res.Path, err)
		return
	}
	if err = w.Close(); err != nil {
		log.Printf("storing result for %s: %s", res.Path, err)
	}
}

func validateSignature(sig, pathPart string) error {
	h := hmac.New(sha1.New, securityKey)
	if _, err := h.Write([]byte(pathPart)); err != nil {
		return err
	}
	actualSig := base64.URLEncoding.EncodeToString(h.Sum(nil))
	// constant-time string comparison
	if subtle.ConstantTimeCompare([]byte(sig), []byte(actualSig)) != 1 {
		return fmt.Errorf("signature mismatch")
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"log"
	"net"
	"net/http"
	"os"
	"os/signal"
	"runtime"
	"runtime/pprof"
	"strings"
	"syscall"
	"time"

	"github.com/braintree/manners"
	"github.com/codegangsta/cli"
)

func handleSignals(server *manners.GracefulServer) {
	sigChan := make(chan os.Signal, 3)
	go func() {
		for sig := range sigChan {
			if sig == os.Interrupt || sig == os.Kill {
				server.Close()
			}
			if sig == syscall.SIGUSR1 {
				var buf []byte
				var written int
				currLen := 1024
				for written == len(buf) {
					buf = make([]byte, currLen)
					written = runtime.Stack(buf, true)
					currLen *= 2
				}
				log.Print(string(buf[:written]))
			}
			if sig == syscall.SIGUSR2 {
				go func() {
					cpufile, _ := os.OpenFile("./planb_cpu.pprof", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660)
					memfile, _ := os.OpenFile("./planb_mem.pprof", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660)
					log.Println("enabling profile...")
					pprof.WriteHeapProfile(memfile)
					memfile.Close()
					pprof.StartCPUProfile(cpufile)
					time.Sleep(60 * time.Second)
					pprof.StopCPUProfile()
					cpufile.Close()
					log.Println("profiling done")
				}()
			}
		}
	}()
	signal.Notify(sigChan, os.Interrupt, os.Kill, syscall.SIGUSR1, syscall.SIGUSR2)
}

func runServer(c *cli.Context) {
	listener, err := net.Listen("tcp", c.String("listen"))
	if err != nil {
		log.Fatal(err)
	}
	router := Router{
		ReadRedisHost:  c.String("read-redis-host"),
		ReadRedisPort:  c.Int("read-redis-port"),
		WriteRedisHost: c.String("write-redis-host"),
		WriteRedisPort: c.Int("write-redis-port"),
		LogPath:        c.String("access-log"),
		RequestTimeout: time.Duration(c.Int("request-timeout")) * time.Second,
		DialTimeout:    time.Duration(c.Int("dial-timeout")) * time.Second,
		DeadBackendTTL: c.Int("dead-backend-time"),
		FlushInterval:  time.Duration(c.Int("flush-interval")) * time.Millisecond,
	}
	err = router.Init()
	if err != nil {
		log.Fatal(err)
	}
	s := manners.NewWithServer(&http.Server{Handler: &router})
	handleSignals(s)
	log.Printf("Listening on %v...\n", listener.Addr())
	err = s.Serve(listener)
	router.Stop()
	if err != nil {
		log.Fatal(err)
	}
}

func fixUsage(s string) string {
	parts := strings.Split(s, " ")
	currLen := 0
	lastPart := 0
	var lines []string
	for i := range parts {
		if currLen+len(parts[i])+1 > 55 {
			lines = append(lines, strings.Join(parts[lastPart:i], " "))
			currLen = 0
			lastPart = i
		}
		currLen += len(parts[i]) + 1
	}
	lines = append(lines, strings.Join(parts[lastPart:], " "))
	return strings.Join(lines, "\n\t")
}

func main() {
	app := cli.NewApp()
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "listen, l",
			Value: "0.0.0.0:8989",
			Usage: "Address to listen",
		},
		cli.StringFlag{
			Name:  "read-redis-host",
			Value: "127.0.0.1",
		},
		cli.IntFlag{
			Name:  "read-redis-port",
			Value: 6379,
		},
		cli.StringFlag{
			Name:  "write-redis-host",
			Value: "127.0.0.1",
		},
		cli.IntFlag{
			Name:  "write-redis-port",
			Value: 6379,
		},
		cli.StringFlag{
			Name:  "access-log",
			Value: "./access.log",
			Usage: fixUsage("File path where access log will be written. If value equals `syslog` log will be sent to local syslog."),
		},
		cli.IntFlag{
			Name:  "request-timeout",
			Value: 30,
			Usage: "Total backend request timeout in seconds",
		},
		cli.IntFlag{
			Name:  "dial-timeout",
			Value: 10,
			Usage: "Dial backend request timeout in seconds",
		},
		cli.IntFlag{
			Name:  "dead-backend-time",
			Value: 30,
			Usage: fixUsage("Time in seconds a backend will remain disabled after a network failure."),
		},
		cli.IntFlag{
			Name:  "flush-interval",
			Value: 10,
			Usage: fixUsage("Time in milliseconds to flush the proxied request."),
		},
	}
	app.Version = "0.1.2"
	app.Name = "planb"
	app.Usage = "http and websockets reverse proxy"
	app.Action = runServer
	app.Author = "tsuru team"
	app.Email = "https://github.com/tsuru/planb"
	app.Run(os.Args)
}
<commit_msg>main: bump to 0.1.3<commit_after>// Copyright 2015 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"log"
	"net"
	"net/http"
	"os"
	"os/signal"
	"runtime"
	"runtime/pprof"
	"strings"
	"syscall"
	"time"

	"github.com/braintree/manners"
	"github.com/codegangsta/cli"
)

func handleSignals(server *manners.GracefulServer) {
	sigChan := make(chan os.Signal, 3)
	go func() {
		for sig := range sigChan {
			if sig == os.Interrupt || sig == os.Kill {
				server.Close()
			}
			if sig == syscall.SIGUSR1 {
				var buf []byte
				var written int
				currLen := 1024
				for written == len(buf) {
					buf = make([]byte, currLen)
					written = runtime.Stack(buf, true)
					currLen *= 2
				}
				log.Print(string(buf[:written]))
			}
			if sig == syscall.SIGUSR2 {
				go func() {
					cpufile, _ := os.OpenFile("./planb_cpu.pprof", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660)
					memfile, _ := os.OpenFile("./planb_mem.pprof", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660)
					log.Println("enabling profile...")
					pprof.WriteHeapProfile(memfile)
					memfile.Close()
					pprof.StartCPUProfile(cpufile)
					time.Sleep(60 * time.Second)
					pprof.StopCPUProfile()
					cpufile.Close()
					log.Println("profiling done")
				}()
			}
		}
	}()
	signal.Notify(sigChan, os.Interrupt, os.Kill, syscall.SIGUSR1, syscall.SIGUSR2)
}

func runServer(c *cli.Context) {
	listener, err := net.Listen("tcp", c.String("listen"))
	if err != nil {
		log.Fatal(err)
	}
	router := Router{
		ReadRedisHost:  c.String("read-redis-host"),
		ReadRedisPort:  c.Int("read-redis-port"),
		WriteRedisHost: c.String("write-redis-host"),
		WriteRedisPort: c.Int("write-redis-port"),
		LogPath:        c.String("access-log"),
		RequestTimeout: time.Duration(c.Int("request-timeout")) * time.Second,
		DialTimeout:    time.Duration(c.Int("dial-timeout")) * time.Second,
		DeadBackendTTL: c.Int("dead-backend-time"),
		FlushInterval:  time.Duration(c.Int("flush-interval")) * time.Millisecond,
	}
	err = router.Init()
	if err != nil {
		log.Fatal(err)
	}
	s := manners.NewWithServer(&http.Server{Handler: &router})
	handleSignals(s)
	log.Printf("Listening on %v...\n", listener.Addr())
	err = s.Serve(listener)
	router.Stop()
	if err != nil {
		log.Fatal(err)
	}
}

func fixUsage(s string) string {
	parts := strings.Split(s, " ")
	currLen := 0
	lastPart := 0
	var lines []string
	for i := range parts {
		if currLen+len(parts[i])+1 > 55 {
			lines = append(lines, strings.Join(parts[lastPart:i], " "))
			currLen = 0
			lastPart = i
		}
		currLen += len(parts[i]) + 1
	}
	lines = append(lines, strings.Join(parts[lastPart:], " "))
	return strings.Join(lines, "\n\t")
}

func main() {
	app := cli.NewApp()
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "listen, l",
			Value: "0.0.0.0:8989",
			Usage: "Address to listen",
		},
		cli.StringFlag{
			Name:  "read-redis-host",
			Value: "127.0.0.1",
		},
		cli.IntFlag{
			Name:  "read-redis-port",
			Value: 6379,
		},
		cli.StringFlag{
			Name:  "write-redis-host",
			Value: "127.0.0.1",
		},
		cli.IntFlag{
			Name:  "write-redis-port",
			Value: 6379,
		},
		cli.StringFlag{
			Name:  "access-log",
			Value: "./access.log",
			Usage: fixUsage("File path where access log will be written. If value equals `syslog` log will be sent to local syslog."),
		},
		cli.IntFlag{
			Name:  "request-timeout",
			Value: 30,
			Usage: "Total backend request timeout in seconds",
		},
		cli.IntFlag{
			Name:  "dial-timeout",
			Value: 10,
			Usage: "Dial backend request timeout in seconds",
		},
		cli.IntFlag{
			Name:  "dead-backend-time",
			Value: 30,
			Usage: fixUsage("Time in seconds a backend will remain disabled after a network failure."),
		},
		cli.IntFlag{
			Name:  "flush-interval",
			Value: 10,
			Usage: fixUsage("Time in milliseconds to flush the proxied request."),
		},
	}
	app.Version = "0.1.3"
	app.Name = "planb"
	app.Usage = "http and websockets reverse proxy"
	app.Action = runServer
	app.Author = "tsuru team"
	app.Email = "https://github.com/tsuru/planb"
	app.Run(os.Args)
}
<|endoftext|>


<commit_before>package mandrill

import (
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io"
	"mime"
	"net/http"
	"path/filepath"
	"strings"
)

const (
	MANDRILL_LOCATION string = "https://mandrillapp.com/api/1.0/"
	SEND_LOCATION     string = "messages/send.json"
)

func New(apiKey, subAccount, fromEmail, fromName string) *Client {
	return &Client{
		apiKey:     apiKey,
		subAccount: subAccount,
		fromEmail:  fromEmail,
		fromName:   fromName,
	}
}

func (m *Client) SendMessage(html, subject, toEmail, toName string, tags []string) ([]*SendResponse, error) {
	return m.SendMessageWithAttachments(html, subject, toEmail, toName, tags, nil)
}

func (m *Client) SendMessageWithAttachments(html, subject, toEmail, toName string,
	tags []string, attachments []*MessageAttachment) ([]*SendResponse, error) {
	requestData, err := getSendRequestData(m.apiKey, html, subject, m.fromEmail, m.fromName, toEmail,
		toName, m.subAccount, tags, attachments)
	if err != nil {
		return nil, err
	}

	response, err := sendRequest(SEND_LOCATION, requestData)
	if err != nil {
		return nil, err
	}

	return response, nil
}

func (m *Client) SendMessageWithReader(html, subject, toEmail, toName string,
	tags []string, fname string, r io.Reader) ([]*SendResponse, error) {
	att, err := AttachmentFromReader(fname, r)
	if err != nil {
		return nil, err
	}
	return m.SendMessageWithAttachments(html, subject, toEmail, toName, tags, []*MessageAttachment{att})
}

func AttachmentFromReader(fname string, r io.Reader) (*MessageAttachment, error) {
	var (
		buf = getBuffer()
		enc = base64.NewEncoder(base64.RawStdEncoding, buf)
	)
	defer putBuffer(buf)

	if _, err := io.Copy(enc, r); err != nil {
		return nil, err
	}
	enc.Close()

	return &MessageAttachment{
		Type:    mime.TypeByExtension(filepath.Ext(fname)),
		Name:    fname,
		Content: buf.String(),
	}, nil
}

func sendRequest(loc, requestData string) ([]*SendResponse, error) {
	resp, err := http.Post(MANDRILL_LOCATION+loc, "application/json", strings.NewReader(requestData))
	if err != nil {
		return nil, err
	}

	if resp.StatusCode != 200 {
		var r struct {
			Code    int    `json:"code"`
			Message string `json:"message"`
		}
		if err := json.NewDecoder(resp.Body).Decode(&r); err != nil {
			return nil, err
		}
		return nil, fmt.Errorf("error(%d): %s", r.Code, r.Message)

	}

	var s []*SendResponse
	if err := json.NewDecoder(resp.Body).Decode(&s); err != nil {
		return nil, err
	}

	return s, nil
}
<commit_msg>Access<commit_after>package mandrill

import (
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io"
	"mime"
	"net/http"
	"path/filepath"
	"strings"
)

const (
	MANDRILL_LOCATION string = "https://mandrillapp.com/api/1.0/"
	SEND_LOCATION     string = "messages/send.json"
)

func New(apiKey, subAccount, fromEmail, fromName string) *Client {
	return &Client{
		apiKey:     apiKey,
		subAccount: subAccount,
		fromEmail:  fromEmail,
		fromName:   fromName,
	}
}

func (m *Client) APIKey() string {
	return m.apiKey
}

func (m *Client) SubAccount() string {
	return m.subAccount
}

func (m *Client) SendMessage(html, subject, toEmail, toName string, tags []string) ([]*SendResponse, error) {
	return m.SendMessageWithAttachments(html, subject, toEmail, toName, tags, nil)
}

func (m *Client) SendMessageWithAttachments(html, subject, toEmail, toName string,
	tags []string, attachments []*MessageAttachment) ([]*SendResponse, error) {
	requestData, err := getSendRequestData(m.apiKey, html, subject, m.fromEmail, m.fromName, toEmail,
		toName, m.subAccount, tags, attachments)
	if err != nil {
		return nil, err
	}

	response, err := sendRequest(SEND_LOCATION, requestData)
	if err != nil {
		return nil, err
	}

	return response, nil
}

func (m *Client) SendMessageWithReader(html, subject, toEmail, toName string,
	tags []string, fname string, r io.Reader) ([]*SendResponse, error) {
	att, err := AttachmentFromReader(fname, r)
	if err != nil {
		return nil, err
	}
	return m.SendMessageWithAttachments(html, subject, toEmail, toName, tags, []*MessageAttachment{att})
}

func AttachmentFromReader(fname string, r io.Reader) (*MessageAttachment, error) {
	var (
		buf = getBuffer()
		enc = base64.NewEncoder(base64.RawStdEncoding, buf)
	)
	defer putBuffer(buf)

	if _, err := io.Copy(enc, r); err != nil {
		return nil, err
	}
	enc.Close()

	return &MessageAttachment{
		Type:    mime.TypeByExtension(filepath.Ext(fname)),
		Name:    fname,
		Content: buf.String(),
	}, nil
}

func sendRequest(loc, requestData string) ([]*SendResponse, error) {
	resp, err := http.Post(MANDRILL_LOCATION+loc, "application/json", strings.NewReader(requestData))
	if err != nil {
		return nil, err
	}

	if resp.StatusCode != 200 {
		var r struct {
			Code    int    `json:"code"`
			Message string `json:"message"`
		}
		if err := json.NewDecoder(resp.Body).Decode(&r); err != nil {
			return nil, err
		}
		return nil, fmt.Errorf("error(%d): %s", r.Code, r.Message)

	}

	var s []*SendResponse
	if err := json.NewDecoder(resp.Body).Decode(&s); err != nil {
		return nil, err
	}

	return s, nil
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"github.com/spf13/pflag"
	"github.com/spf13/viper"
	"log"
	"os"
	"strings"
	"time"
)

const staticVersion = "v0.5.0+"

var version string

func setupFlag(name string) {
	viper.SetConfigName(name)
	viper.AddConfigPath("/etc/xdg/vv")
	viper.AddConfigPath("$HOME/.config/vv")
	pflag.String("mpd.host", "", "[DEPRECATED] mpd server hostname to connect")
	pflag.String("mpd.port", "", "[DEPRECATED] mpd server TCP port to connect")
	pflag.String("mpd.network", "tcp", "mpd server network to connect")
	pflag.String("mpd.addr", "localhost:6600", "mpd server address to connect")
	pflag.String("mpd.music_directory", "", "set music_directory in mpd.conf value to search album cover image")
	pflag.String("server.port", "", "[DEPRECATED] this app serving TCP port")
	pflag.String("server.addr", ":8080", "this app serving address")
	pflag.Bool("server.keepalive", true, "use HTTP keep-alive")
	pflag.BoolP("debug", "d", false, "use local assets if exists")
	pflag.Parse()
	viper.BindPFlags(pflag.CommandLine)
}

func getMusicDirectory(confpath string) (string, error) {
	f, err := os.Open(confpath)
	if err != nil {
		return "", err
	}
	defer f.Close()
	sc := bufio.NewScanner(f)
	for i := 1; sc.Scan(); i++ {
		if err := sc.Err(); err != nil {
			return "", err
		}
		l := sc.Text()
		if strings.HasPrefix(l, "music_directory") {
			q := strings.TrimSpace(strings.TrimPrefix(l, "music_directory"))
			if strings.HasPrefix(q, "\"") && strings.HasSuffix(q, "\"") {
				return strings.Trim(q, "\""), nil
			}
		}
	}
	return "", nil
}

//go:generate go-bindata assets
func main() {
	setupFlag("config")
	err := viper.ReadInConfig()
	if err != nil {
		if _, notfound := err.(viper.ConfigFileNotFoundError); !notfound {
			log.Println("[error]", "faied to load config file:", err)
			os.Exit(1)
		}
	}
	musicDirectory := viper.GetString("mpd.music_directory")
	if len(musicDirectory) == 0 {
		dir, err := getMusicDirectory("/etc/mpd.conf")
		if err == nil {
			musicDirectory = dir
		}
	}
	network := viper.GetString("mpd.network")
	addr := viper.GetString("mpd.addr")
	if viper.GetString("mpd.host") != "" && viper.GetString("mpd.port") != "" {
		log.Println("[warn]", "mpd.host and mpd.port are deprecated option. use mpd.addr")
		network = "tcp"
		addr = viper.GetString("mpd.host") + ":" + viper.GetString("mpd.port")
	}
	music, err := Dial(network, addr, "", musicDirectory)
	defer music.Close()
	if err != nil {
		log.Println("[error]", "faied to connect/initialize mpd:", err)
		os.Exit(1)
	}
	serverAddr := viper.GetString("server.addr")
	if viper.GetString("server.port") != "" {
		log.Println("[warn]", "server.port is deprecated option. use server.addr")
		serverAddr = ":" + viper.GetString("server.port")
	}
	s := Server{
		Music:          music,
		MusicDirectory: musicDirectory,
		Addr:           serverAddr,
		StartTime:      time.Now().UTC(),
		KeepAlive:      viper.GetBool("server.keepalive"),
		debug:          viper.GetBool("debug"),
	}
	s.Serve()
}
<commit_msg>v0.5.1+<commit_after>package main

import (
	"bufio"
	"github.com/spf13/pflag"
	"github.com/spf13/viper"
	"log"
	"os"
	"strings"
	"time"
)

const staticVersion = "v0.5.1+"

var version string

func setupFlag(name string) {
	viper.SetConfigName(name)
	viper.AddConfigPath("/etc/xdg/vv")
	viper.AddConfigPath("$HOME/.config/vv")
	pflag.String("mpd.host", "", "[DEPRECATED] mpd server hostname to connect")
	pflag.String("mpd.port", "", "[DEPRECATED] mpd server TCP port to connect")
	pflag.String("mpd.network", "tcp", "mpd server network to connect")
	pflag.String("mpd.addr", "localhost:6600", "mpd server address to connect")
	pflag.String("mpd.music_directory", "", "set music_directory in mpd.conf value to search album cover image")
	pflag.String("server.port", "", "[DEPRECATED] this app serving TCP port")
	pflag.String("server.addr", ":8080", "this app serving address")
	pflag.Bool("server.keepalive", true, "use HTTP keep-alive")
	pflag.BoolP("debug", "d", false, "use local assets if exists")
	pflag.Parse()
	viper.BindPFlags(pflag.CommandLine)
}

func getMusicDirectory(confpath string) (string, error) {
	f, err := os.Open(confpath)
	if err != nil {
		return "", err
	}
	defer f.Close()
	sc := bufio.NewScanner(f)
	for i := 1; sc.Scan(); i++ {
		if err := sc.Err(); err != nil {
			return "", err
		}
		l := sc.Text()
		if strings.HasPrefix(l, "music_directory") {
			q := strings.TrimSpace(strings.TrimPrefix(l, "music_directory"))
			if strings.HasPrefix(q, "\"") && strings.HasSuffix(q, "\"") {
				return strings.Trim(q, "\""), nil
			}
		}
	}
	return "", nil
}

//go:generate go-bindata assets
func main() {
	setupFlag("config")
	err := viper.ReadInConfig()
	if err != nil {
		if _, notfound := err.(viper.ConfigFileNotFoundError); !notfound {
			log.Println("[error]", "faied to load config file:", err)
			os.Exit(1)
		}
	}
	musicDirectory := viper.GetString("mpd.music_directory")
	if len(musicDirectory) == 0 {
		dir, err := getMusicDirectory("/etc/mpd.conf")
		if err == nil {
			musicDirectory = dir
		}
	}
	network := viper.GetString("mpd.network")
	addr := viper.GetString("mpd.addr")
	if viper.GetString("mpd.host") != "" && viper.GetString("mpd.port") != "" {
		log.Println("[warn]", "mpd.host and mpd.port are deprecated option. use mpd.addr")
		network = "tcp"
		addr = viper.GetString("mpd.host") + ":" + viper.GetString("mpd.port")
	}
	music, err := Dial(network, addr, "", musicDirectory)
	defer music.Close()
	if err != nil {
		log.Println("[error]", "faied to connect/initialize mpd:", err)
		os.Exit(1)
	}
	serverAddr := viper.GetString("server.addr")
	if viper.GetString("server.port") != "" {
		log.Println("[warn]", "server.port is deprecated option. use server.addr")
		serverAddr = ":" + viper.GetString("server.port")
	}
	s := Server{
		Music:          music,
		MusicDirectory: musicDirectory,
		Addr:           serverAddr,
		StartTime:      time.Now().UTC(),
		KeepAlive:      viper.GetBool("server.keepalive"),
		debug:          viper.GetBool("debug"),
	}
	s.Serve()
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/jessevdk/go-flags"
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
	"log"
	"net/http"
	"regexp"
	"strings"
	"sync"
	"time"
)

const (
	expectingVersion = "1.1.1 DISABLED"                                  // number being expected. must be changed manually (for now).
	changeURLBase    = "https://code.google.com/p/go/source/detail?r=go" // base url to poll the tag
	updateInterval   = 6 * time.Second                                   // Update interval for the expected number
)

var defaultPage = "http://isgo1point5.outyet.org"

var (
	versions     = make(map[string]*version) // map with all versions by number(string)
	versionsLock sync.RWMutex                // map lock
)

var regexpNumber = regexp.MustCompile(`^[1-9](?:\.[0-9]){0,2}$`)

var colVersions *mgo.Collection
var colNV *mgo.Collection

var options struct {
	Listen string `short:"l" long:"listen" default:"141.138.139.6:80" description:"IP:post to listen on"`
}

func main() {
	args, err := flags.Parse(&options)
	if err != nil {
		log.Fatalln(err)
	}
	if len(args) > 0 {
		log.Fatalln("Unexpected arguments.")
	}

	mgoSess, err := mgo.Dial("localhost")
	if err != nil {
		log.Fatalln(err)
	}
	colVersions = mgoSess.DB("outyet").C("versions")
	colVersions.EnsureIndex(mgo.Index{
		Key:      []string{"number"},
		Unique:   true,
		DropDups: true,
	})
	colNV = mgoSess.DB("outyet").C("namevalue")
	colNV.EnsureIndex(mgo.Index{
		Key:      []string{"name"},
		Unique:   true,
		DropDups: true,
	})

	if err := http.ListenAndServe(options.Listen, http.HandlerFunc(rootHandler)); err != nil {
		log.Fatalln(err)
	}
}

func rootHandler(w http.ResponseWriter, r *http.Request) {
	// handler for stats page
	if r.Host == "stats.outyet.org" {
		statsHandler(w, r)
		return
	}

	// redirect for 'old' domain
	if r.Host == "isgo1point2outyet.com" {
		http.Redirect(w, r, "http://isgo1point2.outyet.org", http.StatusTemporaryRedirect)
		return
	}

	// only handle requests on /
	if r.RequestURI != "/" {
		http.NotFound(w, r)
		return
	}

	// check if Host header matches isgo*.outyet.org
	if !strings.HasSuffix(r.Host, ".outyet.org") || !strings.HasPrefix(r.Host, "isgo") {
		log.Printf("Invalid host format detected. %s\n", r.Host)
		http.Redirect(w, r, defaultPage, http.StatusTemporaryRedirect)
		return
	}

	number := strings.Replace(r.Host[4:len(r.Host)-11], "point", ".", -1)
	log.Println(number)

	if !regexpNumber.MatchString(number) {
		http.Error(w, "invalid request format", http.StatusBadRequest)
		return
	}

	if strings.HasSuffix(number, ".0") {
		number = number[:len(number)-2]
		if len(number) > 0 {
			log.Printf(" > redirecting to %s\n", number)
			http.Redirect(w, r, "http://isgo"+strings.Replace(number, ".", "point", -1)+".outyet.org", http.StatusTemporaryRedirect)
			return
		}
		log.Println("invalid number")
		http.Redirect(w, r, defaultPage, http.StatusTemporaryRedirect)
		return
	}

	// get right version in a safe way
	o := getVersion(number)

	// add hitCount's
	colVersions.Upsert(bson.M{"number": o.number}, bson.M{"$inc": bson.M{"hits": 1}})
	colNV.Upsert(bson.M{"name": "counts"}, bson.M{"$inc": bson.M{"hits": 1}})

	// execute template
	data := dataOutyet{
		Outyet: <-o.isOutyetChan, //retrieve outyet directly from channel
		Number: number,
	}
	err := tmplOutyet.Execute(w, data)
	if err != nil {
		log.Print(err)
	}
}

func statsHandler(w http.ResponseWriter, r *http.Request) {
	data := &dataStats{}

	colNV.Find(bson.M{"name": "counts"}).One(data)
	colVersions.Find(nil).Sort("number").All(&data.Versions)

	for _, v := range data.Versions {
		// get outyet for given version number
		v.Outyet = <-getVersion(v.Number).isOutyetChan

		// add hitCount's
		colVersions.Upsert(bson.M{"number": v.Number}, bson.M{"$inc": bson.M{"hits": 1}})
	}

	err := tmplStats.Execute(w, data)
	if err != nil {
		log.Print(err)
	}
}
<commit_msg>Change git URL base. Fixes #2<commit_after>package main

import (
	"log"
	"net/http"
	"regexp"
	"strings"
	"sync"
	"time"

	"github.com/jessevdk/go-flags"
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
)

const (
	expectingVersion = "1.1.1 DISABLED"                      // number being expected. must be changed manually (for now).
	changeURLBase    = "https://go.googlesource.com/go/+/go" // base url to poll the tag
	updateInterval   = 6 * time.Second                       // Update interval for the expected number
)

var defaultPage = "http://isgo1point5.outyet.org"

var (
	versions     = make(map[string]*version) // map with all versions by number(string)
	versionsLock sync.RWMutex                // map lock
)

var regexpNumber = regexp.MustCompile(`^[1-9](?:\.[0-9]){0,2}$`)

var colVersions *mgo.Collection
var colNV *mgo.Collection

var options struct {
	Listen string `short:"l" long:"listen" default:"141.138.139.6:80" description:"IP:post to listen on"`
}

func main() {
	args, err := flags.Parse(&options)
	if err != nil {
		log.Fatalln(err)
	}
	if len(args) > 0 {
		log.Fatalln("Unexpected arguments.")
	}

	mgoSess, err := mgo.Dial("localhost")
	if err != nil {
		log.Fatalln(err)
	}
	colVersions = mgoSess.DB("outyet").C("versions")
	colVersions.EnsureIndex(mgo.Index{
		Key:      []string{"number"},
		Unique:   true,
		DropDups: true,
	})
	colNV = mgoSess.DB("outyet").C("namevalue")
	colNV.EnsureIndex(mgo.Index{
		Key:      []string{"name"},
		Unique:   true,
		DropDups: true,
	})

	if err := http.ListenAndServe(options.Listen, http.HandlerFunc(rootHandler)); err != nil {
		log.Fatalln(err)
	}
}

func rootHandler(w http.ResponseWriter, r *http.Request) {
	// handler for stats page
	if r.Host == "stats.outyet.org" {
		statsHandler(w, r)
		return
	}

	// redirect for 'old' domain
	if r.Host == "isgo1point2outyet.com" {
		http.Redirect(w, r, "http://isgo1point2.outyet.org", http.StatusTemporaryRedirect)
		return
	}

	// only handle requests on /
	if r.RequestURI != "/" {
		http.NotFound(w, r)
		return
	}

	// check if Host header matches isgo*.outyet.org
	if !strings.HasSuffix(r.Host, ".outyet.org") || !strings.HasPrefix(r.Host, "isgo") {
		log.Printf("Invalid host format detected. %s\n", r.Host)
		http.Redirect(w, r, defaultPage, http.StatusTemporaryRedirect)
		return
	}

	number := strings.Replace(r.Host[4:len(r.Host)-11], "point", ".", -1)
	log.Println(number)

	if !regexpNumber.MatchString(number) {
		http.Error(w, "invalid request format", http.StatusBadRequest)
		return
	}

	if strings.HasSuffix(number, ".0") {
		number = number[:len(number)-2]
		if len(number) > 0 {
			log.Printf(" > redirecting to %s\n", number)
			http.Redirect(w, r, "http://isgo"+strings.Replace(number, ".", "point", -1)+".outyet.org", http.StatusTemporaryRedirect)
			return
		}
		log.Println("invalid number")
		http.Redirect(w, r, defaultPage, http.StatusTemporaryRedirect)
		return
	}

	// get right version in a safe way
	o := getVersion(number)

	// add hitCount's
	colVersions.Upsert(bson.M{"number": o.number}, bson.M{"$inc": bson.M{"hits": 1}})
	colNV.Upsert(bson.M{"name": "counts"}, bson.M{"$inc": bson.M{"hits": 1}})

	// execute template
	data := dataOutyet{
		Outyet: <-o.isOutyetChan, //retrieve outyet directly from channel
		Number: number,
	}
	err := tmplOutyet.Execute(w, data)
	if err != nil {
		log.Print(err)
	}
}

func statsHandler(w http.ResponseWriter, r *http.Request) {
	data := &dataStats{}

	colNV.Find(bson.M{"name": "counts"}).One(data)
	colVersions.Find(nil).Sort("number").All(&data.Versions)

	for _, v := range data.Versions {
		// get outyet for given version number
		v.Outyet = <-getVersion(v.Number).isOutyetChan

		// add hitCount's
		colVersions.Upsert(bson.M{"number": v.Number}, bson.M{"$inc": bson.M{"hits": 1}})
	}

	err := tmplStats.Execute(w, data)
	if err != nil {
		log.Print(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"time"

	"github.com/golang/gddo/httputil"
	"github.com/gorilla/sessions"
	"github.com/hanjos/mea-libris/libris"
	"golang.org/x/net/context"
	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
	"google.golang.org/api/books/v1"
	"strings"
)

var (
	sessionName = "sessionName"

	clientID     = os.Getenv("CLIENT_ID")
	clientSecret = os.Getenv("CLIENT_SECRET")
	redirectURL  = os.Getenv("REDIRECT_URL")
	port         = os.Getenv("PORT")

	config = &oauth2.Config{
		ClientID:     clientID,
		ClientSecret: clientSecret,
		Endpoint:     google.Endpoint,
		RedirectURL:  redirectURL,
		Scopes:       []string{books.BooksScope},
	}

	store = sessions.NewCookieStore([]byte(randomString()))
)

func _index(w http.ResponseWriter, r *http.Request) *appError {
	log.Println("Handling /")

	return nil
}

func _google(w http.ResponseWriter, r *http.Request) *appError {
	log.Println("Handling /google")

	session, err := store.Get(r, sessionName)
	if err != nil {
		// ignore session errors
		//return errSessionNotFound(session, err)
	}

	token, ok := session.Values["accessToken"].(string)
	if !ok {
		return errWrap(errAccessTokenNotFound, _status(http.StatusUnauthorized))
	}

	svc, err := newBooksClient(context.Background(), token)
	if err != nil {
		return errWrap(err, _status(http.StatusInternalServerError))
	}

	bs, err := getBooks(svc)
	if err != nil {
		return errWrap(err, _status(http.StatusInternalServerError))
	}

	err = encodeBooks(bs, w, r)
	if err != nil {
		return errWrap(err, _status(http.StatusInternalServerError))
	}

	return nil
}

func _googleConnect(w http.ResponseWriter, r *http.Request) *appError {
	log.Println("Handling /google/connect")

	session, err := store.Get(r, sessionName)
	if err != nil {
		//return errSessionError(session, err)
	}

	_, ok := session.Values["accessToken"].(string)
	if ok {
		log.Println("User authenticated and authorized.")
		fmt.Fprintln(w, "Connected!") // XXX w.WriteHeader(http.StatusOK) is implicit
		return nil
	}

	log.Println("User not authorized; beginning auth exchange")
	log.Println("Generating a new state")
	state := randomString()

	session.Values["state"] = state
	session.Save(r, w)

	url := config.AuthCodeURL(state)

	log.Println("Redirecting to Google's OAuth servers for a code")
	http.Redirect(w, r, url, http.StatusTemporaryRedirect)
	return nil
}

func newBooksClient(ctx context.Context, token string) (*books.Service, error) {
	log.Println("Using the access token to build a Google Books client")

	tok := new(oauth2.Token)
	tok.AccessToken = token

	client := config.Client(ctx, tok)
	svc, err := books.New(client)
	if err != nil {
		return nil, errCantLoadBooksClient(err)
	}

	return svc, nil
}

func getBooks(svc *books.Service) ([]*libris.Book, error) {
	log.Print("Getting the user's books")

	myBooks := []*libris.Book{}
	nextIndex, totalItems := int64(0), int64(0)
	for {
		volumes, err := svc.Volumes.Mybooks.List().
			StartIndex(nextIndex).
			AcquireMethod("FAMILY_SHARED", "PREORDERED", "PUBLIC_DOMAIN", "PURCHASED", "RENTED", "SAMPLE", "UPLOADED").
			ProcessingState("COMPLETED_SUCCESS").
			Do()
		if err != nil {
			return nil, errCantLoadVolumes(err)
		}

		for _, v := range volumes.Items {
			myBooks = append(myBooks, newBook(v))
		}

		nextIndex, totalItems = nextIndex+int64(len(volumes.Items)), volumes.TotalItems
		if nextIndex >= totalItems {
			// XXX since there's no do-while, here we 'go'
			break
		}
	}

	log.Printf("%d books processed (of a total of %d)\n", len(myBooks), totalItems)
	return myBooks, nil
}

func newBook(v *books.Volume) *libris.Book {
	info := v.VolumeInfo

	// resolving the identification
	var id, idType string

	for _, identifier := range info.IndustryIdentifiers {
		if identifier.Identifier == "" {
			continue
		}

		id = identifier.Identifier
		idType = identifier.Type
		break
	}

	// getting the file type
	fileType := "UNKNOWN"
	if v.AccessInfo.Pdf != nil {
		fileType = "PDF"
	} else if v.AccessInfo.Epub != nil {
		fileType = "EPUB"
	}

	// removing the extension from the title if it's there
	title := info.Title
	if strings.HasSuffix(strings.ToLower(title), ".pdf") && fileType == "PDF" {
		title = title[:len(title)-4]
	} else if strings.HasSuffix(strings.ToLower(title), ".epub") && fileType == "EPUB" {
		title = title[:len(title)-5]
	}

	return &libris.Book{
		Title:          title,
		Authors:        info.Authors,
		Identifier:     id,
		IdentifierType: idType,
		AverageRating:  info.AverageRating,
		Publisher:      info.Publisher,
		FileType:       fileType,
	}
}

func encodeBooks(books []*libris.Book, w io.Writer, r *http.Request) error {
	log.Printf("Requested response format: %s\n", r.Header.Get("Accept"))

	contentType := httputil.NegotiateContentType(r,
		[]string{"application/json", "text/csv", "application/csv"},
		"application/json")

	log.Printf("Negotiated content type: %s\n", contentType)
	switch contentType {
	case "application/json":
		return encodeBooksAsJSON(books, w)
	case "application/csv":
		fallthrough
	case "text/csv":
		return encodeBooksAsCSV(books, w)
	default:
		log.Printf("Unexpected content type %s; rendering as application/json", contentType)
		return encodeBooksAsJSON(books, w)
	}
}

func encodeBooksAsJSON(books []*libris.Book, w io.Writer) error {
	log.Println("Encoding books as JSON")

	// XXX setting headers has do be done BEFORE writing the body, or it'll be ignored!
	if rw, ok := w.(http.ResponseWriter); ok {
		rw.Header().Set("Content-Type", "application/json;charset=utf-8")
	}

	booksJSON, err := json.Marshal(books)
	if err != nil {
		return errCantMarshalBooksToJSON(err)
	}

	_, err2 := fmt.Fprintf(w, "%s", booksJSON)
	if err2 != nil {
		return errCantWriteResponse(err2)
	}

	return nil
}

func encodeBooksAsCSV(books []*libris.Book, w io.Writer) error {
	log.Println("Encoding books as CSV")

	// XXX setting headers has do be done BEFORE writing the body, or it'll be ignored!
	if rw, ok := w.(http.ResponseWriter); ok {
		rw.Header().Set("Content-Type", "text/csv;charset=utf-8")
	}

	err := libris.Books(books).EncodeCSV(w)
	if err != nil {
		return err
	}

	return nil
}

func _googleOAuthCallback(w http.ResponseWriter, r *http.Request) *appError {
	log.Println("Handling /google/oauth2callback")
	log.Println("Validating state")

	session, err := store.Get(r, sessionName)
	if err != nil {
		//return errSessionError(session, err)
	}

	sessionState, ok := session.Values["state"].(string)
	if !ok || r.FormValue("state") != sessionState {
		return errWrap(errInvalidState(sessionState, r.FormValue("state")), _status(http.StatusBadRequest))
	}

	log.Println("Extracting the code")
	code := r.FormValue("code")
	if code == "" {
		return errWrap(errCodeNotFound, _status(http.StatusBadRequest))
	}

	defer func() {
		session.Values["state"] = nil // XXX state is a one-time value; we don't need it anymore
	}()

	log.Println("Exchanging the code for an access token")
	token, err := config.Exchange(context.Background(), code)
	if err != nil {
		return errWrap(errTokenExchangeError(err), _status(http.StatusBadRequest))
	}

	session.Values["accessToken"] = token.AccessToken // XXX can't store a *oauth2.Token, so store a string
	session.Save(r, w)

	http.Redirect(w, r, "/google/connect", http.StatusTemporaryRedirect)
	return nil
}

// MAIN
func main() {
	http.Handle("/", appHandler(_index))
	http.Handle("/google", appHandler(_google))
	http.Handle("/google/connect", appHandler(_googleConnect))
	http.Handle("/google/oauth2callback", appHandler(_googleOAuthCallback))

	log.Printf("Starting server on port %s\n", port)
	http.ListenAndServe(":"+port, nil)
}

// session
type authSession struct {
	State string
	Code  string
}

// appHandler
type appHandler func(http.ResponseWriter, *http.Request) *appError

func (fn appHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if err := fn(w, r); err != nil {
		log.Println(err)
		http.Error(w, err.Message, err.Status)
	}
}

// general helpers
func randomString() string {
	return fmt.Sprintf("st%d", time.Now().UnixNano())
}

// appError
type appError struct {
	Message string
	Status  int
}

func (err appError) Error() string {
	return fmt.Sprintf("Error [%d]: %s", err.Status, err.Message)
}

type appErrorField func(appErr *appError)

func _prefix(str string) appErrorField {
	return func(appErr *appError) {
		if appErr == nil {
			return
		}

		if appErr.Message == "" {
			appErr.Message = str
			return
		}

		appErr.Message = str + ": " + appErr.Message
	}
}

func _status(status int) appErrorField {
	return func(appErr *appError) {
		if appErr == nil {
			return
		}

		appErr.Status = status
	}
}

func errWrap(err error, fields ...appErrorField) *appError {
	if err == nil {
		return nil
	}

	if appErr, ok := err.(*appError); ok {
		return appErr
	}

	appErr := &appError{err.Error(), http.StatusInternalServerError}
	for _, field := range fields {
		field(appErr)
	}

	return appErr
}

func errInvalidState(expected, actual string) error {
	return fmt.Errorf("Invalid state parameter: expected %s; got %s", expected, actual)
}

func errSessionError(s *sessions.Session, err error) error {
	return fmt.Errorf("Error on session %v : %v", s, err)
}

var errAccessTokenNotFound = fmt.Errorf("User not authorized! Use the /google/connect endpoint.")

var errCodeNotFound = fmt.Errorf("Code not found!")

func errTokenExchangeError(err error) error {
	return fmt.Errorf("Problem with token exchange: %v", err)
}

func errCantLoadBooksClient(err error) error {
	return fmt.Errorf("Couldn't load Google Books client: %v", err)
}

func errCantLoadVolumes(err error) error {
	return fmt.Errorf("Couldn't load the user's volumes: %v", err)
}

func errCantMarshalBooksToJSON(err error) error {
	return fmt.Errorf("Couldn't marshal the books' info to JSON: %v", err)
}

func errCantWriteResponse(err error) error {
	return fmt.Errorf("Couldn't write response: %v", err)
}
<commit_msg>Writing /google/disconnect.<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"time"

	"github.com/golang/gddo/httputil"
	"github.com/gorilla/sessions"
	"github.com/hanjos/mea-libris/libris"
	"golang.org/x/net/context"
	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
	"google.golang.org/api/books/v1"
	"strings"
)

var (
	sessionName = "sessionName"

	clientID     = os.Getenv("CLIENT_ID")
	clientSecret = os.Getenv("CLIENT_SECRET")
	redirectURL  = os.Getenv("REDIRECT_URL")
	port         = os.Getenv("PORT")

	config = &oauth2.Config{
		ClientID:     clientID,
		ClientSecret: clientSecret,
		Endpoint:     google.Endpoint,
		RedirectURL:  redirectURL,
		Scopes:       []string{books.BooksScope},
	}

	store = sessions.NewCookieStore([]byte(randomString()))
)

func _index(w http.ResponseWriter, r *http.Request) *appError {
	log.Println("Handling /")

	return nil
}

func _google(w http.ResponseWriter, r *http.Request) *appError {
	log.Println("Handling /google")

	session, err := store.Get(r, sessionName)
	if err != nil {
		// ignore session errors
		//return errSessionNotFound(session, err)
	}

	token, ok := session.Values["accessToken"].(string)
	if !ok {
		return errWrap(errAccessTokenNotFound, _status(http.StatusUnauthorized))
	}

	svc, err := newBooksClient(context.Background(), token)
	if err != nil {
		return errWrap(err, _status(http.StatusInternalServerError))
	}

	bs, err := getBooks(svc)
	if err != nil {
		return errWrap(err, _status(http.StatusInternalServerError))
	}

	err = encodeBooks(bs, w, r)
	if err != nil {
		return errWrap(err, _status(http.StatusInternalServerError))
	}

	return nil
}

func _googleConnect(w http.ResponseWriter, r *http.Request) *appError {
	log.Println("Handling /google/connect")

	session, err := store.Get(r, sessionName)
	if err != nil {
		//return errSessionError(session, err)
	}

	_, ok := session.Values["accessToken"].(string)
	if ok {
		log.Println("User authenticated and authorized.")
		fmt.Fprintln(w, "Connected!") // XXX w.WriteHeader(http.StatusOK) is implicit
		return nil
	}

	log.Println("User not authorized; beginning auth exchange")
	log.Println("Generating a new state")
	state := randomString()

	session.Values["state"] = state
	session.Save(r, w)

	url := config.AuthCodeURL(state)

	log.Println("Redirecting to Google's OAuth servers for a code")
	http.Redirect(w, r, url, http.StatusTemporaryRedirect)
	return nil
}

func _googleDisconnect(w http.ResponseWriter, r *http.Request) *appError {
	log.Println("Handling /google/disconnect")

	session, err := store.Get(r, sessionName)
	if err != nil {
		//return errSessionError(session, err)
	}

	token, ok := session.Values["accessToken"].(string)
	if !ok {
		log.Println("User wasn't connected. Nothing was done.")
		fmt.Fprintln(w, "User wasn't connected. Nothing was done.")
		return nil
	}

	log.Println("Disconnecting the current user")
	url := "https://accounts.google.com/o/oauth2/revoke?token=" + token
	resp, err := http.Get(url)
	defer resp.Body.Close()
	if err != nil {
		return errWrap(errCantRevokeToken(err), _status(http.StatusInternalServerError))
	}

	// Reset the user's session
	log.Println("Resetting the session")
	session.Values["state"] = nil
	session.Values["accessToken"] = nil
	session.Save(r, w)

	fmt.Fprintln(w, "User disconnected!")
	return nil
}

func newBooksClient(ctx context.Context, token string) (*books.Service, error) {
	log.Println("Using the access token to build a Google Books client")

	tok := new(oauth2.Token)
	tok.AccessToken = token

	client := config.Client(ctx, tok)
	svc, err := books.New(client)
	if err != nil {
		return nil, errCantLoadBooksClient(err)
	}

	return svc, nil
}

func getBooks(svc *books.Service) ([]*libris.Book, error) {
	log.Print("Getting the user's books")

	myBooks := []*libris.Book{}
	nextIndex, totalItems := int64(0), int64(0)
	for {
		volumes, err := svc.Volumes.Mybooks.List().
			StartIndex(nextIndex).
			AcquireMethod("FAMILY_SHARED", "PREORDERED", "PUBLIC_DOMAIN", "PURCHASED", "RENTED", "SAMPLE", "UPLOADED").
			ProcessingState("COMPLETED_SUCCESS").
			Do()
		if err != nil {
			return nil, errCantLoadVolumes(err)
		}

		for _, v := range volumes.Items {
			myBooks = append(myBooks, newBook(v))
		}

		nextIndex, totalItems = nextIndex+int64(len(volumes.Items)), volumes.TotalItems
		if nextIndex >= totalItems {
			// XXX since there's no do-while, here we 'go'
			break
		}
	}

	log.Printf("%d books processed (of a total of %d)\n", len(myBooks), totalItems)
	return myBooks, nil
}

func newBook(v *books.Volume) *libris.Book {
	info := v.VolumeInfo

	// resolving the identification
	var id, idType string

	for _, identifier := range info.IndustryIdentifiers {
		if identifier.Identifier == "" {
			continue
		}

		id = identifier.Identifier
		idType = identifier.Type
		break
	}

	// getting the file type
	fileType := "UNKNOWN"
	if v.AccessInfo.Pdf != nil {
		fileType = "PDF"
	} else if v.AccessInfo.Epub != nil {
		fileType = "EPUB"
	}

	// removing the extension from the title if it's there
	title := info.Title
	if strings.HasSuffix(strings.ToLower(title), ".pdf") && fileType == "PDF" {
		title = title[:len(title)-4]
	} else if strings.HasSuffix(strings.ToLower(title), ".epub") && fileType == "EPUB" {
		title = title[:len(title)-5]
	}

	return &libris.Book{
		Title:          title,
		Authors:        info.Authors,
		Identifier:     id,
		IdentifierType: idType,
		AverageRating:  info.AverageRating,
		Publisher:      info.Publisher,
		FileType:       fileType,
	}
}

func encodeBooks(books []*libris.Book, w io.Writer, r *http.Request) error {
	log.Printf("Requested response format: %s\n", r.Header.Get("Accept"))

	contentType := httputil.NegotiateContentType(r,
		[]string{"application/json", "text/csv", "application/csv"},
		"application/json")

	log.Printf("Negotiated content type: %s\n", contentType)
	switch contentType {
	case "application/json":
		return encodeBooksAsJSON(books, w)
	case "application/csv":
		fallthrough
	case "text/csv":
		return encodeBooksAsCSV(books, w)
	default:
		log.Printf("Unexpected content type %s; rendering as application/json", contentType)
		return encodeBooksAsJSON(books, w)
	}
}

func encodeBooksAsJSON(books []*libris.Book, w io.Writer) error {
	log.Println("Encoding books as JSON")

	// XXX setting headers has do be done BEFORE writing the body, or it'll be ignored!
	if rw, ok := w.(http.ResponseWriter); ok {
		rw.Header().Set("Content-Type", "application/json;charset=utf-8")
	}

	booksJSON, err := json.Marshal(books)
	if err != nil {
		return errCantMarshalBooksToJSON(err)
	}

	_, err2 := fmt.Fprintf(w, "%s", booksJSON)
	if err2 != nil {
		return errCantWriteResponse(err2)
	}

	return nil
}

func encodeBooksAsCSV(books []*libris.Book, w io.Writer) error {
	log.Println("Encoding books as CSV")

	// XXX setting headers has do be done BEFORE writing the body, or it'll be ignored!
	if rw, ok := w.(http.ResponseWriter); ok {
		rw.Header().Set("Content-Type", "text/csv;charset=utf-8")
	}

	err := libris.Books(books).EncodeCSV(w)
	if err != nil {
		return err
	}

	return nil
}

func _googleOAuthCallback(w http.ResponseWriter, r *http.Request) *appError {
	log.Println("Handling /google/oauth2callback")
	log.Println("Validating state")

	session, err := store.Get(r, sessionName)
	if err != nil {
		//return errSessionError(session, err)
	}

	sessionState, ok := session.Values["state"].(string)
	if !ok || r.FormValue("state") != sessionState {
		return errWrap(errInvalidState(sessionState, r.FormValue("state")), _status(http.StatusBadRequest))
	}

	log.Println("Extracting the code")
	code := r.FormValue("code")
	if code == "" {
		return errWrap(errCodeNotFound, _status(http.StatusBadRequest))
	}

	defer func() {
		session.Values["state"] = nil // XXX state is a one-time value; we don't need it anymore
	}()

	log.Println("Exchanging the code for an access token")
	token, err := config.Exchange(context.Background(), code)
	if err != nil {
		return errWrap(errTokenExchangeError(err), _status(http.StatusBadRequest))
	}

	session.Values["accessToken"] = token.AccessToken // XXX can't store a *oauth2.Token, so store a string
	session.Save(r, w)

	http.Redirect(w, r, "/google/connect", http.StatusTemporaryRedirect)
	return nil
}

// MAIN
func main() {
	http.Handle("/", appHandler(_index))
	http.Handle("/google", appHandler(_google))
	http.Handle("/google/connect", appHandler(_googleConnect))
	http.Handle("/google/disconnect", appHandler(_googleDisconnect))
	http.Handle("/google/oauth2callback", appHandler(_googleOAuthCallback))

	log.Printf("Starting server on port %s\n", port)
	http.ListenAndServe(":"+port, nil)
}

// session
type authSession struct {
	State string
	Code  string
}

// MIDDLEWARES
type appHandler func(http.ResponseWriter, *http.Request) *appError

// ServeHTTP implements the http.Handler interface.
func (fn appHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if err := fn(w, r); err != nil {
		log.Println(err)
		http.Error(w, err.Message, err.Status)
	}
}

// general helpers
func randomString() string {
	return fmt.Sprintf("st%d", time.Now().UnixNano())
}

// APPLICATION ERRORS
type appError struct {
	Message string
	Status  int
}

// Error implements the error interface.
func (err appError) Error() string {
	return fmt.Sprintf("Error [%d]: %s", err.Status, err.Message)
}

type appErrorField func(appErr *appError)

func _prefix(str string) appErrorField {
	return func(appErr *appError) {
		if appErr == nil {
			return
		}

		if appErr.Message == "" {
			appErr.Message = str
			return
		}

		appErr.Message = str + ": " + appErr.Message
	}
}

func _status(status int) appErrorField {
	return func(appErr *appError) {
		if appErr == nil {
			return
		}

		appErr.Status = status
	}
}

func errWrap(err error, fields ...appErrorField) *appError {
	if err == nil {
		return nil
	}

	if appErr, ok := err.(*appError); ok {
		return appErr
	}

	appErr := &appError{err.Error(), http.StatusInternalServerError}
	for _, field := range fields {
		field(appErr)
	}

	return appErr
}

func errInvalidState(expected, actual string) error {
	return fmt.Errorf("Invalid state parameter: expected %s; got %s", expected, actual)
}

func errSessionError(s *sessions.Session, err error) error {
	return fmt.Errorf("Error on session %v : %v", s, err)
}

var errAccessTokenNotFound = fmt.Errorf("User not authorized! Use the /google/connect endpoint.")

var errCodeNotFound = fmt.Errorf("Code not found!")

func errTokenExchangeError(err error) error {
	return fmt.Errorf("Problem with token exchange: %v", err)
}

func errCantLoadBooksClient(err error) error {
	return fmt.Errorf("Couldn't load Google Books client: %v", err)
}

func errCantLoadVolumes(err error) error {
	return fmt.Errorf("Couldn't load the user's volumes: %v", err)
}

func errCantMarshalBooksToJSON(err error) error {
	return fmt.Errorf("Couldn't marshal the books' info to JSON: %v", err)
}

func errCantWriteResponse(err error) error {
	return fmt.Errorf("Couldn't write response: %v", err)
}

func errCantRevokeToken(err error) error {
	return fmt.Errorf("Failed to revoke token for the current user: %v", err)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/codegangsta/cli"
	"github.com/joho/godotenv"
	"log"
	"os"
)

func main() {
	loadDotEnv()

	app := NewApp()
	app.RunAndExitOnError()
}

// NewApp creates a new command line app
func NewApp() *cli.App {
	app := cli.NewApp()
	app.Name = "dbmate"
	app.Usage = "A lightweight, framework-independent database migration tool."
	app.Version = Version

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "migrations-dir, d",
			Value: "./db/migrations",
			Usage: "specify the directory containing migration files",
		},
		cli.StringFlag{
			Name:  "env, e",
			Value: "DATABASE_URL",
			Usage: "specify an environment variable containing the database URL",
		},
	}

	app.Commands = []cli.Command{
		{
			Name:  "new",
			Usage: "Generate a new migration file",
			Action: func(ctx *cli.Context) {
				runCommand(NewCommand, ctx)
			},
		},
		{
			Name:  "up",
			Usage: "Create database (if necessary) and migrate to the latest version",
			Action: func(ctx *cli.Context) {
				runCommand(UpCommand, ctx)
			},
		},
		{
			Name:  "create",
			Usage: "Create database",
			Action: func(ctx *cli.Context) {
				runCommand(CreateCommand, ctx)
			},
		},
		{
			Name:  "drop",
			Usage: "Drop database (if it exists)",
			Action: func(ctx *cli.Context) {
				runCommand(DropCommand, ctx)
			},
		},
		{
			Name:  "migrate",
			Usage: "Migrate to the latest version",
			Action: func(ctx *cli.Context) {
				runCommand(MigrateCommand, ctx)
			},
		},
		{
			Name:  "rollback",
			Usage: "Rollback the most recent migration",
			Action: func(ctx *cli.Context) {
				runCommand(RollbackCommand, ctx)
			},
		},
	}

	return app
}

type command func(*cli.Context) error

func runCommand(cmd command, ctx *cli.Context) {
	err := cmd(ctx)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error: %s\n", err)
		os.Exit(1)
	}
}

func loadDotEnv() {
	if _, err := os.Stat(".env"); err != nil {
		return
	}

	if err := godotenv.Load(); err != nil {
		log.Fatal("Error loading .env file")
	}
}
<commit_msg>Fix deprecation warning<commit_after>package main

import (
	"fmt"
	"github.com/codegangsta/cli"
	"github.com/joho/godotenv"
	"log"
	"os"
)

func main() {
	loadDotEnv()

	app := NewApp()
	err := app.Run(os.Args)
	checkErr(err)
}

// NewApp creates a new command line app
func NewApp() *cli.App {
	app := cli.NewApp()
	app.Name = "dbmate"
	app.Usage = "A lightweight, framework-independent database migration tool."
	app.Version = Version

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "migrations-dir, d",
			Value: "./db/migrations",
			Usage: "specify the directory containing migration files",
		},
		cli.StringFlag{
			Name:  "env, e",
			Value: "DATABASE_URL",
			Usage: "specify an environment variable containing the database URL",
		},
	}

	app.Commands = []cli.Command{
		{
			Name:  "new",
			Usage: "Generate a new migration file",
			Action: func(ctx *cli.Context) {
				runCommand(NewCommand, ctx)
			},
		},
		{
			Name:  "up",
			Usage: "Create database (if necessary) and migrate to the latest version",
			Action: func(ctx *cli.Context) {
				runCommand(UpCommand, ctx)
			},
		},
		{
			Name:  "create",
			Usage: "Create database",
			Action: func(ctx *cli.Context) {
				runCommand(CreateCommand, ctx)
			},
		},
		{
			Name:  "drop",
			Usage: "Drop database (if it exists)",
			Action: func(ctx *cli.Context) {
				runCommand(DropCommand, ctx)
			},
		},
		{
			Name:  "migrate",
			Usage: "Migrate to the latest version",
			Action: func(ctx *cli.Context) {
				runCommand(MigrateCommand, ctx)
			},
		},
		{
			Name:  "rollback",
			Usage: "Rollback the most recent migration",
			Action: func(ctx *cli.Context) {
				runCommand(RollbackCommand, ctx)
			},
		},
	}

	return app
}

type command func(*cli.Context) error

func runCommand(cmd command, ctx *cli.Context) {
	err := cmd(ctx)
	checkErr(err)
}

func loadDotEnv() {
	if _, err := os.Stat(".env"); err != nil {
		return
	}

	if err := godotenv.Load(); err != nil {
		log.Fatal("Error loading .env file")
	}
}

func checkErr(err error) {
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error: %s\n", err)
		os.Exit(1)
	}
}
<|endoftext|>