<commit_before>package ergo

import (
	"net/http"
	"strings"
)

type Ergoer interface {
	GetSchemes() []string
	GetConsumes() []string
	GetProduces() []string
	NotFound(*Response, *Request)
	MethodNotAllowed(*Route, *Response, *Request)
	Err(error, *Response, *Request)
	Panic(*Response, *Request)
}

// Ergo

type Ergo struct {
	root   *Route
	router ExternalRouter

	schemes  []string
	consumes []string
	produces []string

	NotFoundHandler         Handler
	MethodNotAllowedHandler MethodNotAllowedHandler
	ErrHandler              ErrHandler
	PanicHandler            Handler
}

func New() *Ergo {
	e := &Ergo{
		NotFoundHandler:         defaultNotFoundHandler,
		MethodNotAllowedHandler: defaultMethodNotAllowedHandler,
		ErrHandler:              defaultErrHandler,
		PanicHandler:            defaultPanicHandler,
	}
	r := NewRoute("")
	r.ergo = e
	e.root = r
	return e
}

func (e *Ergo) New(path string) *Route {
	return e.root.New(path)
}

func (e *Ergo) Schemes(s ...string) *Ergo {
	schemes(e, s)
	return e
}

func (e *Ergo) Consumes(mimes ...string) *Ergo {
	consumes(e, mimes)
	return e
}

func (e *Ergo) Produces(mimes ...string) *Ergo {
	produces(e, mimes)
	return e
}

func (e *Ergo) Params(params ...*Param) *Ergo {
	addParams(e.root, params...)
	return e
}

func (e *Ergo) ResetParams(params ...*Param) *Ergo {
	e.root.setParamsSlice(params...)
	return e
}

func (e *Ergo) SetParams(params map[string]*Param) *Ergo {
	e.root.setParams(params)
	return e
}

func (e *Ergo) IgnoreParams(params ...string) *Ergo {
	ignoreParams(e.root, params...)
	return e
}

func (e *Ergo) IgnoreParamsBut(params ...string) *Ergo {
	ignoreParamsBut(e.root, params...)
	return e
}

// Router uses a router that implement Router interface
// as the main router.
func (e *Ergo) Router(er ExternalRouter) {

}

// GetSchemes returns the default schemes.
func (e *Ergo) GetSchemes() []string {
	return e.schemes
}

// GetConsumes returns the default consumable content types.
func (e *Ergo) GetConsumes() []string {
	return e.consumes
}

// GetProduces returns the default producible content types.
func (e *Ergo) GetProduces() []string {
	return e.produces
}

func (e *Ergo) setSchemes(schemes []string) {
	e.schemes = schemes
}

func (e *Ergo) setConsumes(consumes []string) {
	e.consumes = consumes
}

func (e *Ergo) setProduces(produces []string) {
	e.produces = produces
}

func (e *Ergo) Prepare() error {
	return nil
}

func (e *Ergo) NotFound(res *Response, req *Request) {
	e.NotFoundHandler.ServeHTTP(res, req)
}

func (e *Ergo) MethodNotAllowed(r *Route, res *Response, req *Request) {
	e.MethodNotAllowedHandler.ServeHTTP(r, res, req)
}

func (e *Ergo) Err(err error, res *Response, req *Request) {
	e.ErrHandler.ServeHTTP(err, res, req)
}

func (e *Ergo) Panic(res *Response, req *Request) {
	e.PanicHandler.ServeHTTP(res, req)
}

func (e *Ergo) Run(address string) error {
	err := e.Prepare()
	if err != nil {
		return err
	}
	return http.ListenAndServe(address, e)
}

func (e *Ergo) RunTLS(addr, certFile, keyFile string) error {
	err := e.Prepare()
	if err != nil {
		return err
	}
	return http.ListenAndServeTLS(addr, certFile, keyFile, e)
}

func (e *Ergo) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	path := preparePath(r.URL.Path)
	if r.URL.Path != "/"+path && r.Method == "GET" {
		r.URL.Path = "/" + path
		http.Redirect(w, r, r.URL.String(), http.StatusMovedPermanently)
		return
	}
	route, rp := e.root.Match(path)
	if route == nil {
		// not found
		return
	}

	req := NewRequest(r)
	if len(rp) > 0 {
		ps := strings.Split(rp[:len(rp)], ";")
		for _, p := range ps {
			ci := strings.Index(p, ":")

			req.pathParams[p[:ci]] = p[ci+1:]
		}
	}
	res := NewResponse(w)
	route.ServeHTTP(res, req)
}
<commit_msg>ExternalRouter interface<commit_after>package ergo

import (
	"net/http"
	"strings"
)

type ExternalRouter interface {
}

type Ergoer interface {
	GetSchemes() []string
	GetConsumes() []string
	GetProduces() []string
	NotFound(*Response, *Request)
	MethodNotAllowed(*Route, *Response, *Request)
	Err(error, *Response, *Request)
	Panic(*Response, *Request)
}

// Ergo

type Ergo struct {
	root   *Route
	router ExternalRouter

	schemes  []string
	consumes []string
	produces []string

	NotFoundHandler         Handler
	MethodNotAllowedHandler MethodNotAllowedHandler
	ErrHandler              ErrHandler
	PanicHandler            Handler
}

func New() *Ergo {
	e := &Ergo{
		NotFoundHandler:         defaultNotFoundHandler,
		MethodNotAllowedHandler: defaultMethodNotAllowedHandler,
		ErrHandler:              defaultErrHandler,
		PanicHandler:            defaultPanicHandler,
	}
	r := NewRoute("")
	r.ergo = e
	e.root = r
	return e
}

func (e *Ergo) New(path string) *Route {
	return e.root.New(path)
}

func (e *Ergo) Schemes(s ...string) *Ergo {
	schemes(e, s)
	return e
}

func (e *Ergo) Consumes(mimes ...string) *Ergo {
	consumes(e, mimes)
	return e
}

func (e *Ergo) Produces(mimes ...string) *Ergo {
	produces(e, mimes)
	return e
}

func (e *Ergo) Params(params ...*Param) *Ergo {
	addParams(e.root, params...)
	return e
}

func (e *Ergo) ResetParams(params ...*Param) *Ergo {
	e.root.setParamsSlice(params...)
	return e
}

func (e *Ergo) SetParams(params map[string]*Param) *Ergo {
	e.root.setParams(params)
	return e
}

func (e *Ergo) IgnoreParams(params ...string) *Ergo {
	ignoreParams(e.root, params...)
	return e
}

func (e *Ergo) IgnoreParamsBut(params ...string) *Ergo {
	ignoreParamsBut(e.root, params...)
	return e
}

// Router uses a router that implement Router interface
// as the main router.
func (e *Ergo) Router(er ExternalRouter) {

}

// GetSchemes returns the default schemes.
func (e *Ergo) GetSchemes() []string {
	return e.schemes
}

// GetConsumes returns the default consumable content types.
func (e *Ergo) GetConsumes() []string {
	return e.consumes
}

// GetProduces returns the default producible content types.
func (e *Ergo) GetProduces() []string {
	return e.produces
}

func (e *Ergo) setSchemes(schemes []string) {
	e.schemes = schemes
}

func (e *Ergo) setConsumes(consumes []string) {
	e.consumes = consumes
}

func (e *Ergo) setProduces(produces []string) {
	e.produces = produces
}

func (e *Ergo) Prepare() error {
	return nil
}

func (e *Ergo) NotFound(res *Response, req *Request) {
	e.NotFoundHandler.ServeHTTP(res, req)
}

func (e *Ergo) MethodNotAllowed(r *Route, res *Response, req *Request) {
	e.MethodNotAllowedHandler.ServeHTTP(r, res, req)
}

func (e *Ergo) Err(err error, res *Response, req *Request) {
	e.ErrHandler.ServeHTTP(err, res, req)
}

func (e *Ergo) Panic(res *Response, req *Request) {
	e.PanicHandler.ServeHTTP(res, req)
}

func (e *Ergo) Run(address string) error {
	err := e.Prepare()
	if err != nil {
		return err
	}
	return http.ListenAndServe(address, e)
}

func (e *Ergo) RunTLS(addr, certFile, keyFile string) error {
	err := e.Prepare()
	if err != nil {
		return err
	}
	return http.ListenAndServeTLS(addr, certFile, keyFile, e)
}

func (e *Ergo) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	path := preparePath(r.URL.Path)
	if r.URL.Path != "/"+path && r.Method == "GET" {
		r.URL.Path = "/" + path
		http.Redirect(w, r, r.URL.String(), http.StatusMovedPermanently)
		return
	}
	route, rp := e.root.Match(path)
	if route == nil {
		// not found
		return
	}

	req := NewRequest(r)
	if len(rp) > 0 {
		ps := strings.Split(rp[:len(rp)], ";")
		for _, p := range ps {
			ci := strings.Index(p, ":")

			req.pathParams[p[:ci]] = p[ci+1:]
		}
	}
	res := NewResponse(w)
	route.ServeHTTP(res, req)
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package drive

import (
	"fmt"
	"io"
	"os"

	prettywords "github.com/odeke-em/pretty-words"
)

const (
	AboutKey      = "about"
	AllKey        = "all"
	CopyKey       = "copy"
	DeleteKey     = "delete"
	DiffKey       = "diff"
	EmptyTrashKey = "emptytrash"
	FeaturesKey   = "features"
	HelpKey       = "help"
	InitKey       = "init"
	DeInitKey     = "deinit"
	LinkKey       = "Link"
	ListKey       = "list"
	MoveKey       = "move"
	OSLinuxKey    = "linux"
	PullKey       = "pull"
	PushKey       = "push"
	PubKey        = "pub"
	RenameKey     = "rename"
	QuotaKey      = "quota"
	ShareKey      = "share"
	StatKey       = "stat"
	TouchKey      = "touch"
	TrashKey      = "trash"
	UnshareKey    = "unshare"
	UntrashKey    = "untrash"
	UnpubKey      = "unpub"
	VersionKey    = "version"
	Md5sumKey     = "md5sum"
	NewKey        = "new"
	IndexKey      = "index"
	PruneKey      = "prune"

	CoercedMimeKeyKey     = "coerced-mime"
	DepthKey              = "depth"
	EmailsKey             = "emails"
	EmailMessageKey       = "emailMessage"
	ForceKey              = "force"
	QuietKey              = "quiet"
	QuitShortKey          = "q"
	YesShortKey           = "Y"
	QuitLongKey           = "quit"
	MatchesKey            = "matches"
	HiddenKey             = "hidden"
	Md5Key                = "md5"
	NoPromptKey           = "no-prompt"
	SizeKey               = "size"
	NameKey               = "name"
	OpenKey               = "open"
	OriginalNameKey       = "oname"
	ModTimeKey            = "modt"
	LastViewedByMeTimeKey = "lvt"
	RoleKey               = "role"
	TypeKey               = "type"
	TrashedKey            = "trashed"
	SkipMimeKeyKey        = "skip-mime"
	MatchMimeKeyKey       = "exact-mime"
	ExactTitleKey         = "exact-title"
	MatchOwnerKey         = "match-owner"
	ExactOwnerKey         = "exact-owner"
	NotOwnerKey           = "skip-owner"
	SortKey               = "sort"
	FolderKey             = "folder"
	MimeKey               = "mime-key"
	DriveRepoRelPath      = "github.com/odeke-em/drive"
	UrlKey                = "url"
)

const (
	DescAbout                 = "print out information about your Google drive"
	DescAll                   = "print out the entire help section"
	DescCopy                  = "copy remote paths to a destination"
	DescDelete                = "deletes the items permanently. This operation is irreversible"
	DescDiff                  = "compares local files with their remote equivalent"
	DescEmptyTrash            = "permanently cleans out your trash"
	DescExcludeOps            = "exclude operations"
	DescFeatures              = "returns information about the features of your drive"
	DescIndex                 = "fetch indices from remote"
	DescHelp                  = "Get help for a topic"
	DescInit                  = "initializes a directory and authenticates user"
	DescDeInit                = "removes the user's credentials and initialized files"
	DescList                  = "lists the contents of remote path"
	DescMove                  = "move files/folders"
	DescQuota                 = "prints out information related to your quota space"
	DescPublish               = "publishes a file and prints its publicly available url"
	DescRename                = "renames a file/folder"
	DescPull                  = "pulls remote changes from Google Drive"
	DescPruneIndices          = "remove stale indices"
	DescPush                  = "push local changes to Google Drive"
	DescShare                 = "share files with specific emails giving the specified users specifies roles and permissions"
	DescStat                  = "display information about a file"
	DescTouch                 = "updates a remote file's modification time to that currently on the server"
	DescTrash                 = "moves files to trash"
	DescUnshare               = "revoke a user's access to a file"
	DescUntrash               = "restores files from trash to their original locations"
	DescUnpublish             = "revokes public access to a file"
	DescVersion               = "prints the version"
	DescMd5sum                = "prints a list compatible with md5sum(1)"
	DescAccountTypes          = "\n\t* anyone.\n\t* user.\n\t* domain.\n\t* group"
	DescRoles                 = "\n\t* owner.\n\t* reader.\n\t* writer.\n\t* commenter."
	DescExplicitylPullExports = "explicitly pull exports"
	DescIgnoreChecksum        = "avoids computation of checksums as a final check." +
		"\nUse cases may include:\n\t* when you are low on bandwidth e.g SSHFS." +
		"\n\t* Are on a low power device"
	DescIgnoreConflict     = "turns off the conflict resolution safety"
	DescIgnoreNameClashes  = "ignore name clashes"
	DescSort               = "sort items in the order\n\t* md5.\n\t* name.\n\t* size.\n\t* type.\n\t* version"
	DescSkipMime           = "skip elements with mimeTypes derived from these extensison"
	DescMatchMime          = "get elements with the exact mimeTypes derived from extensisons"
	DescMatchTitle         = "elements with matching titles"
	DescExactTitle         = "get elements with the exact titles"
	DescMatchOwner         = "elements with matching owners"
	DescExactOwner         = "elements with the exact owner"
	DescNotOwner           = "ignore elements owned by these users"
	DescNew                = "create a new file/folder"
	DescAllIndexOperations = "perform all the index related operations"
	DescOpen               = "open a file in the appropriate filemanager or default browser"
	DescUrl                = "returns the url of each file"
	DescVerbose            = "show step by step information verbosely"
)

const (
	CLIOptionExplicitlyExport   = "explicitly-export"
	CLIOptionIgnoreChecksum     = "ignore-checksum"
	CLIOptionIgnoreConflict     = "ignore-conflict"
	CLIOptionIgnoreNameClashes  = "ignore-name-clashes"
	CLIOptionExcludeOperations  = "exclude-ops"
	CLIOptionId                 = "id"
	CLIOptionNoClobber          = "no-clobber"
	CLIOptionNotify             = "notify"
	CLIOptionSkipMime           = "skip-mime"
	CLIOptionMatchMime          = "exact-mime"
	CLIOptionExactTitle         = "exact-title"
	CLIOptionMatchTitle         = "match-mime"
	CLIOptionExactOwner         = "exact-owner"
	CLIOptionMatchOwner         = "match-owner"
	CLIOptionNotOwner           = "skip-owner"
	CLIOptionPruneIndices       = "prune"
	CLIOptionAllIndexOperations = "all-ops"
	CLIOptionVerboseKey         = "verbose"
	CLIOptionVerboseShortKey    = "v"
	CLIOptionOpen               = "open"
	CLIOptionWebBrowser         = "web-browser"
	CLIOptionFileBrowser        = "file-browser"
)

const (
	DefaultMaxTraversalDepth = -1
)

const (
	GoogleApiClientIdEnvKey     = "GOOGLE_API_CLIENT_ID"
	GoogleApiClientSecretEnvKey = "GOOGLE_API_CLIENT_SECRET"
	DriveGoMaxProcsKey          = "DRIVE_GOMAXPROCS"
	GoMaxProcsKey               = "GOMAXPROCS"
)

const (
	DesktopExtension = "desktop"
)

const (
	InfiniteDepth = -1
)

var skipChecksumNote = fmt.Sprintf(
	"\nNote: You can skip checksum verification by passing in flag `-%s`", CLIOptionIgnoreChecksum)

var docMap = map[string][]string{
	AboutKey: []string{
		DescAbout,
	},
	CopyKey: []string{
		DescCopy,
	},
	DeleteKey: []string{
		DescDelete,
	},
	DiffKey: []string{
		DescDiff, "Accepts multiple remote paths for line by line comparison",
		skipChecksumNote,
	},
	EmptyTrashKey: []string{
		DescEmptyTrash,
	},
	FeaturesKey: []string{
		DescFeatures,
	},
	InitKey: []string{
		DescInit, "Requests for access to your Google Drive",
		"Creating a folder that contains your credentials",
		"Note: `init` in an already initialized drive will erase the old credentials",
	},
	PullKey: []string{
		DescPull, "Downloads content from the remote drive or modifies",
		" local content to match that on your Google Drive",
		skipChecksumNote,
	},
	PushKey: []string{
		DescPush, "Uploads content to your Google Drive from your local path",
		"Push comes in a couple of flavors",
		"\t* Ordinary push: `drive push path1 path2 path3`",
		"\t* Mounted push: `drive push -m path1 [path2 path3] drive_context_path`",
		skipChecksumNote,
	},
	ListKey: []string{
		DescList,
		"List the information of a remote path not necessarily present locally",
		"Allows printing of long options and by default does minimal printing",
	},
	MoveKey: []string{
		DescMove,
		"Moves files/folders between folders",
	},
	PubKey: []string{
		DescPublish, "Accepts multiple paths",
	},
	RenameKey: []string{
		DescRename, "Accepts <src> <newName>",
	},
	QuotaKey: []string{DescQuota},
	ShareKey: []string{
		DescShare, "Accepts multiple paths",
		"Specify the emails to share with as well as the message to send them on notification",
		"Accepted values for:\n+ accountType: ",
		DescAccountTypes, "\n+ roles:", DescRoles,
	},
	StatKey: []string{
		DescStat, "provides detailed information about a remote file",
		"Accepts multiple paths",
	},
	TouchKey: []string{
		DescTouch, "Given a list of remote files `touch` updates their",
		"last edit times to that currently on the server",
	},
	TrashKey: []string{
		DescTrash, "Sends a list of remote files to trash",
	},
	UnshareKey: []string{
		DescUnshare, "Accepts multiple paths",
		"Accepted values for accountTypes::", DescAccountTypes,
	},
	UntrashKey: []string{
		DescUntrash, "takes remote files out of the trash",
		"Note: untrash is a relative path command so any resolutions are made",
		"relative to the current working directory i.e",
		"\n\t$ drive trash mnt/logos",
	},
	UnpubKey: []string{
		DescUnpublish, "revokes public access to a list of remote files",
	},
	VersionKey: []string{
		DescVersion, fmt.Sprintf("current version is: %s", Version),
	},
}

var Aliases = map[string][]string{
	CopyKey: []string{"cp"},
	ListKey: []string{"ls"},
	MoveKey: []string{"mv"},
}

func ShowAllDescriptions() {
	for key, _ := range docMap {
		ShowDescription(key)
	}
}

func ShowDescriptions(topics ...string) {
	if len(topics) < 1 {
		topics = append(topics, AllKey)
	}

	for _, topic := range topics {
		ShowDescription(topic)
	}
}

func ShowDescription(topic string) {
	if topic == AllKey {
		ShowAllDescriptions()
		return
	}

	help, ok := docMap[topic]
	if !ok {
		PrintfShadow("Unkown command '%s' type `drive help all` for entire usage documentation", topic)
		ShowAllDescriptions()
	} else {
		description, documentation := help[0], help[1:]
		PrintfShadow("Name\n\t%s - %s\n", topic, description)
		if len(documentation) >= 1 {
			PrintfShadow("Description\n")
			for _, line := range documentation {
				segments := formatText(line)
				for _, segment := range segments {
					PrintfShadow("\t%s", segment)
				}
			}
			PrintfShadow("\n* For usage flags: \033[32m`drive %s -h`\033[00m\n\n", topic)
		}
	}
}

func formatText(text string) []string {
	splits := strings.Split(text, " ")

	pr := prettywords.PrettyRubric{
		Limit: 80,
		Body:  splits,
	}

	return pr.Format()
}

func PrintfShadow(fmt_ string, args ...interface{}) {
	FprintfShadow(os.Stdout, fmt_, args...)
}

func FprintfShadow(f io.Writer, fmt_ string, args ...interface{}) {
	sprinted := fmt.Sprintf(fmt_, args...)
	splits := formatText(sprinted)
	joined := strings.Join(splits, "\n")
	fmt.Fprintf(f, joined)
}
<commit_msg>help: fixed up new widther package<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package drive

import (
	"fmt"
	"io"
	"os"
	"strings"

	prettywords "github.com/odeke-em/pretty-words"
)

const (
	AboutKey      = "about"
	AllKey        = "all"
	CopyKey       = "copy"
	DeleteKey     = "delete"
	DiffKey       = "diff"
	EmptyTrashKey = "emptytrash"
	FeaturesKey   = "features"
	HelpKey       = "help"
	InitKey       = "init"
	DeInitKey     = "deinit"
	LinkKey       = "Link"
	ListKey       = "list"
	MoveKey       = "move"
	OSLinuxKey    = "linux"
	PullKey       = "pull"
	PushKey       = "push"
	PubKey        = "pub"
	RenameKey     = "rename"
	QuotaKey      = "quota"
	ShareKey      = "share"
	StatKey       = "stat"
	TouchKey      = "touch"
	TrashKey      = "trash"
	UnshareKey    = "unshare"
	UntrashKey    = "untrash"
	UnpubKey      = "unpub"
	VersionKey    = "version"
	Md5sumKey     = "md5sum"
	NewKey        = "new"
	IndexKey      = "index"
	PruneKey      = "prune"

	CoercedMimeKeyKey     = "coerced-mime"
	DepthKey              = "depth"
	EmailsKey             = "emails"
	EmailMessageKey       = "emailMessage"
	ForceKey              = "force"
	QuietKey              = "quiet"
	QuitShortKey          = "q"
	YesShortKey           = "Y"
	QuitLongKey           = "quit"
	MatchesKey            = "matches"
	HiddenKey             = "hidden"
	Md5Key                = "md5"
	NoPromptKey           = "no-prompt"
	SizeKey               = "size"
	NameKey               = "name"
	OpenKey               = "open"
	OriginalNameKey       = "oname"
	ModTimeKey            = "modt"
	LastViewedByMeTimeKey = "lvt"
	RoleKey               = "role"
	TypeKey               = "type"
	TrashedKey            = "trashed"
	SkipMimeKeyKey        = "skip-mime"
	MatchMimeKeyKey       = "exact-mime"
	ExactTitleKey         = "exact-title"
	MatchOwnerKey         = "match-owner"
	ExactOwnerKey         = "exact-owner"
	NotOwnerKey           = "skip-owner"
	SortKey               = "sort"
	FolderKey             = "folder"
	MimeKey               = "mime-key"
	DriveRepoRelPath      = "github.com/odeke-em/drive"
	UrlKey                = "url"
)

const (
	DescAbout                 = "print out information about your Google drive"
	DescAll                   = "print out the entire help section"
	DescCopy                  = "copy remote paths to a destination"
	DescDelete                = "deletes the items permanently. This operation is irreversible"
	DescDiff                  = "compares local files with their remote equivalent"
	DescEmptyTrash            = "permanently cleans out your trash"
	DescExcludeOps            = "exclude operations"
	DescFeatures              = "returns information about the features of your drive"
	DescIndex                 = "fetch indices from remote"
	DescHelp                  = "Get help for a topic"
	DescInit                  = "initializes a directory and authenticates user"
	DescDeInit                = "removes the user's credentials and initialized files"
	DescList                  = "lists the contents of remote path"
	DescMove                  = "move files/folders"
	DescQuota                 = "prints out information related to your quota space"
	DescPublish               = "publishes a file and prints its publicly available url"
	DescRename                = "renames a file/folder"
	DescPull                  = "pulls remote changes from Google Drive"
	DescPruneIndices          = "remove stale indices"
	DescPush                  = "push local changes to Google Drive"
	DescShare                 = "share files with specific emails giving the specified users specifies roles and permissions"
	DescStat                  = "display information about a file"
	DescTouch                 = "updates a remote file's modification time to that currently on the server"
	DescTrash                 = "moves files to trash"
	DescUnshare               = "revoke a user's access to a file"
	DescUntrash               = "restores files from trash to their original locations"
	DescUnpublish             = "revokes public access to a file"
	DescVersion               = "prints the version"
	DescMd5sum                = "prints a list compatible with md5sum(1)"
	DescAccountTypes          = "\n\t* anyone.\n\t* user.\n\t* domain.\n\t* group"
	DescRoles                 = "\n\t* owner.\n\t* reader.\n\t* writer.\n\t* commenter."
	DescExplicitylPullExports = "explicitly pull exports"
	DescIgnoreChecksum        = "avoids computation of checksums as a final check." +
		"\nUse cases may include:\n\t* when you are low on bandwidth e.g SSHFS." +
		"\n\t* Are on a low power device"
	DescIgnoreConflict     = "turns off the conflict resolution safety"
	DescIgnoreNameClashes  = "ignore name clashes"
	DescSort               = "sort items in the order\n\t* md5.\n\t* name.\n\t* size.\n\t* type.\n\t* version"
	DescSkipMime           = "skip elements with mimeTypes derived from these extensison"
	DescMatchMime          = "get elements with the exact mimeTypes derived from extensisons"
	DescMatchTitle         = "elements with matching titles"
	DescExactTitle         = "get elements with the exact titles"
	DescMatchOwner         = "elements with matching owners"
	DescExactOwner         = "elements with the exact owner"
	DescNotOwner           = "ignore elements owned by these users"
	DescNew                = "create a new file/folder"
	DescAllIndexOperations = "perform all the index related operations"
	DescOpen               = "open a file in the appropriate filemanager or default browser"
	DescUrl                = "returns the url of each file"
	DescVerbose            = "show step by step information verbosely"
)

const (
	CLIOptionExplicitlyExport   = "explicitly-export"
	CLIOptionIgnoreChecksum     = "ignore-checksum"
	CLIOptionIgnoreConflict     = "ignore-conflict"
	CLIOptionIgnoreNameClashes  = "ignore-name-clashes"
	CLIOptionExcludeOperations  = "exclude-ops"
	CLIOptionId                 = "id"
	CLIOptionNoClobber          = "no-clobber"
	CLIOptionNotify             = "notify"
	CLIOptionSkipMime           = "skip-mime"
	CLIOptionMatchMime          = "exact-mime"
	CLIOptionExactTitle         = "exact-title"
	CLIOptionMatchTitle         = "match-mime"
	CLIOptionExactOwner         = "exact-owner"
	CLIOptionMatchOwner         = "match-owner"
	CLIOptionNotOwner           = "skip-owner"
	CLIOptionPruneIndices       = "prune"
	CLIOptionAllIndexOperations = "all-ops"
	CLIOptionVerboseKey         = "verbose"
	CLIOptionVerboseShortKey    = "v"
	CLIOptionOpen               = "open"
	CLIOptionWebBrowser         = "web-browser"
	CLIOptionFileBrowser        = "file-browser"
)

const (
	DefaultMaxTraversalDepth = -1
)

const (
	GoogleApiClientIdEnvKey     = "GOOGLE_API_CLIENT_ID"
	GoogleApiClientSecretEnvKey = "GOOGLE_API_CLIENT_SECRET"
	DriveGoMaxProcsKey          = "DRIVE_GOMAXPROCS"
	GoMaxProcsKey               = "GOMAXPROCS"
)

const (
	DesktopExtension = "desktop"
)

const (
	InfiniteDepth = -1
)

var skipChecksumNote = fmt.Sprintf(
	"\nNote: You can skip checksum verification by passing in flag `-%s`", CLIOptionIgnoreChecksum)

var docMap = map[string][]string{
	AboutKey: []string{
		DescAbout,
	},
	CopyKey: []string{
		DescCopy,
	},
	DeleteKey: []string{
		DescDelete,
	},
	DiffKey: []string{
		DescDiff, "Accepts multiple remote paths for line by line comparison",
		skipChecksumNote,
	},
	EmptyTrashKey: []string{
		DescEmptyTrash,
	},
	FeaturesKey: []string{
		DescFeatures,
	},
	InitKey: []string{
		DescInit, "Requests for access to your Google Drive",
		"Creating a folder that contains your credentials",
		"Note: `init` in an already initialized drive will erase the old credentials",
	},
	PullKey: []string{
		DescPull, "Downloads content from the remote drive or modifies",
		" local content to match that on your Google Drive",
		skipChecksumNote,
	},
	PushKey: []string{
		DescPush, "Uploads content to your Google Drive from your local path",
		"Push comes in a couple of flavors",
		"\t* Ordinary push: `drive push path1 path2 path3`",
		"\t* Mounted push: `drive push -m path1 [path2 path3] drive_context_path`",
		skipChecksumNote,
	},
	ListKey: []string{
		DescList,
		"List the information of a remote path not necessarily present locally",
		"Allows printing of long options and by default does minimal printing",
	},
	MoveKey: []string{
		DescMove,
		"Moves files/folders between folders",
	},
	PubKey: []string{
		DescPublish, "Accepts multiple paths",
	},
	RenameKey: []string{
		DescRename, "Accepts <src> <newName>",
	},
	QuotaKey: []string{DescQuota},
	ShareKey: []string{
		DescShare, "Accepts multiple paths",
		"Specify the emails to share with as well as the message to send them on notification",
		"Accepted values for:\n+ accountType: ",
		DescAccountTypes, "\n+ roles:", DescRoles,
	},
	StatKey: []string{
		DescStat, "provides detailed information about a remote file",
		"Accepts multiple paths",
	},
	TouchKey: []string{
		DescTouch, "Given a list of remote files `touch` updates their",
		"last edit times to that currently on the server",
	},
	TrashKey: []string{
		DescTrash, "Sends a list of remote files to trash",
	},
	UnshareKey: []string{
		DescUnshare, "Accepts multiple paths",
		"Accepted values for accountTypes::", DescAccountTypes,
	},
	UntrashKey: []string{
		DescUntrash, "takes remote files out of the trash",
		"Note: untrash is a relative path command so any resolutions are made",
		"relative to the current working directory i.e",
		"\n\t$ drive trash mnt/logos",
	},
	UnpubKey: []string{
		DescUnpublish, "revokes public access to a list of remote files",
	},
	VersionKey: []string{
		DescVersion, fmt.Sprintf("current version is: %s", Version),
	},
}

var Aliases = map[string][]string{
	CopyKey: []string{"cp"},
	ListKey: []string{"ls"},
	MoveKey: []string{"mv"},
}

func ShowAllDescriptions() {
	for key, _ := range docMap {
		ShowDescription(key)
	}
}

func ShowDescriptions(topics ...string) {
	if len(topics) < 1 {
		topics = append(topics, AllKey)
	}

	for _, topic := range topics {
		ShowDescription(topic)
	}
}

func ShowDescription(topic string) {
	if topic == AllKey {
		ShowAllDescriptions()
		return
	}

	help, ok := docMap[topic]
	if !ok {
		PrintfShadow("Unkown command '%s' type `drive help all` for entire usage documentation", topic)
		ShowAllDescriptions()
	} else {
		description, documentation := help[0], help[1:]
		PrintfShadow("Name\n\t%s - %s\n", topic, description)
		if len(documentation) >= 1 {
			PrintfShadow("Description\n")
			for _, line := range documentation {
				segments := formatText(line)
				for _, segment := range segments {
					PrintfShadow("\t%s", segment)
				}
			}
			PrintfShadow("\n* For usage flags: \033[32m`drive %s -h`\033[00m\n\n", topic)
		}
	}
}

func formatText(text string) []string {
	splits := strings.Split(text, " ")

	pr := prettywords.PrettyRubric{
		Limit: 80,
		Body:  splits,
	}

	return pr.Format()
}

func PrintfShadow(fmt_ string, args ...interface{}) {
	FprintfShadow(os.Stdout, fmt_, args...)
}

func FprintfShadow(f io.Writer, fmt_ string, args ...interface{}) {
	sprinted := fmt.Sprintf(fmt_, args...)
	splits := formatText(sprinted)
	for _, split := range splits {
		fmt.Fprintf(f, "%s\n", split)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"os/exec"

	"github.com/99designs/aws-vault/keyring"
	"github.com/aws/aws-sdk-go/aws/credentials"
)

type ExecCommandInput struct {
	Profile string
	Command string
	Args    []string
	Keyring keyring.Keyring
}

func ExecCommand(ui Ui, input ExecCommandInput) {
	provider, err := NewVaultProvider(input.Keyring, input.Profile)
	if err != nil {
		ui.Error.Fatal(err)
	}

	creds := credentials.NewCredentials(provider)
	val, err := creds.Get()
	if err != nil {
		ui.Error.Fatal(err)
	}

	env := append(os.Environ(),
		"AWS_ACCESS_KEY_ID="+val.AccessKeyID,
		"AWS_SECRET_ACCESS_KEY="+val.SecretAccessKey,
	)

	if val.SessionToken != "" {
		env = append(env, "AWS_SESSION_TOKEN="+val.SessionToken)
	}

	cmd := exec.Command(input.Command, input.Args...)
	cmd.Env = env
	cmd.Stdout = &logWriter{ui.Logger}
	cmd.Stderr = &logWriter{ui.Error}

	if err := cmd.Run(); err != nil {
		ui.Error.Fatal(err)
	}
}
<commit_msg>Better error for when exec doesn't find any credentials<commit_after>package main

import (
	"os"
	"os/exec"

	"github.com/99designs/aws-vault/keyring"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/credentials"
)

type ExecCommandInput struct {
	Profile string
	Command string
	Args    []string
	Keyring keyring.Keyring
}

func ExecCommand(ui Ui, input ExecCommandInput) {
	provider, err := NewVaultProvider(input.Keyring, input.Profile)
	if err != nil {
		ui.Error.Fatal(err)
	}

	creds := credentials.NewCredentials(provider)
	val, err := creds.Get()
	if err != nil {
		if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == "NoCredentialProviders" {
			ui.Error.Fatalf("No credentials found for profile %q", input.Profile)
		} else {
			ui.Error.Fatal(err)
		}
	}

	env := append(os.Environ(),
		"AWS_ACCESS_KEY_ID="+val.AccessKeyID,
		"AWS_SECRET_ACCESS_KEY="+val.SecretAccessKey,
	)

	if val.SessionToken != "" {
		env = append(env, "AWS_SESSION_TOKEN="+val.SessionToken)
	}

	cmd := exec.Command(input.Command, input.Args...)
	cmd.Env = env
	cmd.Stdout = &logWriter{ui.Logger}
	cmd.Stderr = &logWriter{ui.Error}

	if err := cmd.Run(); err != nil {
		ui.Error.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"os"

	"github.com/codegangsta/cli"
)

var exitCmd = cli.Command{
	Name:   "exit",
	Action: exit,
	Usage:  "exit god, or use ctrl+D.",
}

func exit(*cli.Context) {
	os.Exit(0)
}
<commit_msg>save history when press hotkey for exit<commit_after>package main

import (
	"os"

	"github.com/codegangsta/cli"
)

var exitCmd = cli.Command{
	Name:   "exit",
	Action: cmder.exit,
	Usage:  "exit god, or use ctrl+D.",
}

func (c *Cmder) exitOn(*cli.Context) {

}
func (c *Cmder) exit(*cli.Context) {
	if c.line != nil && c.history != nil {
		c.line.WriteHistory(c.history)
	}

	os.Exit(0)
}
<|endoftext|>


<commit_before>package renter

import (
	"io"
	"os"
	"sync"
	"time"

	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/modules/renter/hostdb"
	"github.com/NebulousLabs/Sia/types"
)

const (
	// When a file contract is within this many blocks of expiring, the renter
	// will attempt to renew the contract.
	renewThreshold = 2000

	hostTimeout = 15 * time.Second
)

// repair attempts to repair a file chunk by uploading its pieces to more
// hosts.
func (f *file) repair(chunkIndex uint64, missingPieces []uint64, r io.ReaderAt, hosts []hostdb.Uploader) error {
	// read chunk data and encode
	chunk := make([]byte, f.chunkSize())
	_, err := r.ReadAt(chunk, int64(chunkIndex*f.chunkSize()))
	if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
		return err
	}
	pieces, err := f.erasureCode.Encode(chunk)
	if err != nil {
		return err
	}
	// encrypt pieces
	for i := range pieces {
		key := deriveKey(f.masterKey, chunkIndex, uint64(i))
		pieces[i], err = key.EncryptBytes(pieces[i])
		if err != nil {
			return err
		}
	}

	// upload one piece per host
	numPieces := len(missingPieces)
	if len(hosts) < numPieces {
		numPieces = len(hosts)
	}
	var wg sync.WaitGroup
	wg.Add(numPieces)
	for i := 0; i < numPieces; i++ {
		go func(host hostdb.Uploader, pieceIndex uint64, piece []byte) {
			defer wg.Done()
			offset, err := host.Upload(piece)
			if err != nil {
				return
			}

			// create contract entry, if necessary
			f.mu.Lock()
			defer f.mu.Unlock()
			contract, ok := f.contracts[host.ContractID()]
			if !ok {
				contract = fileContract{
					ID:          host.ContractID(),
					IP:          host.Address(),
					WindowStart: host.EndHeight(),
				}
			}

			// update contract
			contract.Pieces = append(contract.Pieces, pieceData{
				Chunk:  chunkIndex,
				Piece:  pieceIndex,
				Offset: offset,
			})
			f.contracts[host.ContractID()] = contract
		}(hosts[i], uint64(i), pieces[missingPieces[i]])
	}
	wg.Wait()

	return nil
}

// threadedRepairLoop improves the health of files tracked by the renter by
// reuploading their missing pieces. Multiple repair attempts may be necessary
// before the file reaches full redundancy.
func (r *Renter) threadedRepairLoop() {
	for {
		time.Sleep(5 * time.Second)

		if !r.wallet.Unlocked() {
			continue
		}

		// make copy of repair set under lock
		repairing := make(map[string]trackedFile)
		id := r.mu.RLock()
		for name, meta := range r.tracking {
			repairing[name] = meta
		}
		r.mu.RUnlock(id)

		for name, meta := range repairing {
			r.threadedRepairFile(name, meta)
		}
	}
}

// incompleteChunks returns a map of chunks containing pieces that have not
// been uploaded.
func (f *file) incompleteChunks() map[uint64][]uint64 {
	f.mu.RLock()
	defer f.mu.RUnlock()

	present := make([][]bool, f.numChunks())
	for i := range present {
		present[i] = make([]bool, f.erasureCode.NumPieces())
	}
	for _, fc := range f.contracts {
		for _, p := range fc.Pieces {
			present[p.Chunk][p.Piece] = true
		}
	}

	incomplete := make(map[uint64][]uint64)
	for chunkIndex, pieceBools := range present {
		for pieceIndex, ok := range pieceBools {
			if !ok {
				incomplete[uint64(chunkIndex)] = append(incomplete[uint64(chunkIndex)], uint64(pieceIndex))
			}
		}
	}
	return incomplete
}

// chunkHosts returns the hosts storing the given chunk.
func (f *file) chunkHosts(chunk uint64) []modules.NetAddress {
	f.mu.RLock()
	defer f.mu.RUnlock()

	var old []modules.NetAddress
	for _, fc := range f.contracts {
		for _, p := range fc.Pieces {
			if p.Chunk == chunk {
				old = append(old, fc.IP)
				break
			}
		}
	}
	return old
}

// expiringContracts returns the contracts that will expire soon.
// TODO: what if contract has fully expired?
func (f *file) expiringContracts(height types.BlockHeight) []fileContract {
	f.mu.RLock()
	defer f.mu.RUnlock()

	var expiring []fileContract
	for _, fc := range f.contracts {
		if height > fc.WindowStart-renewThreshold {
			expiring = append(expiring, fc)
		}
	}
	return expiring
}

// threadedRepairFile repairs and saves an individual file.
func (r *Renter) threadedRepairFile(name string, meta trackedFile) {
	// helper function
	logAndRemove := func(fmt string, args ...interface{}) {
		r.log.Printf(fmt, args...)
		id := r.mu.Lock()
		delete(r.tracking, name)
		r.mu.Unlock(id)
	}

	id := r.mu.RLock()
	f, ok := r.files[name]
	r.mu.RUnlock(id)
	if !ok {
		logAndRemove("removing %v from repair set: no longer tracking that file", name)
		return
	}

	// check for expiration
	height := r.cs.Height()
	if meta.EndHeight != 0 && meta.EndHeight < height {
		logAndRemove("removing %v from repair set: storage period has ended", name)
		return
	}

	// open file handle
	handle, err := os.Open(meta.RepairPath)
	if err != nil {
		logAndRemove("removing %v from repair set: %v", name, err)
		return
	}
	defer handle.Close()

	// check for un-uploaded pieces
	badChunks := f.incompleteChunks()
	if len(badChunks) == 0 {
		return
	}

	r.log.Printf("repairing %v chunks of %v", len(badChunks), name)

	// create host pool
	contractSize := f.pieceSize * f.numChunks() // each host gets one piece of each chunk
	var duration types.BlockHeight = defaultDuration
	if meta.EndHeight != 0 {
		duration = meta.EndHeight - height
	}
	pool, err := r.hostDB.NewPool(contractSize, duration)
	if err != nil {
		r.log.Printf("failed to repair %v: %v", name, err)
		return
	}
	defer pool.Close() // heh

	for chunk, pieces := range badChunks {
		// determine host set
		old := f.chunkHosts(chunk)
		hosts := pool.UniqueHosts(f.erasureCode.NumPieces()-len(old), old)
		if len(hosts) == 0 {
			r.log.Printf("aborting repair of %v: not enough hosts", name)
			break
		}
		// upload to new hosts
		err = f.repair(chunk, pieces, handle, hosts)
		if err != nil {
			r.log.Printf("aborting repair of %v: %v", name, err)
			break
		}
	}

	// renew expiring contracts
	if meta.EndHeight == 0 {
		var expiringContracts []fileContract
		expiringContracts = f.expiringContracts(height)
		r.log.Printf("renewing %v contracts of %v", len(expiringContracts), name)
		for _, c := range expiringContracts {
			newHeight := height + defaultDuration
			newID, err := r.hostDB.Renew(c.ID, newHeight)
			if err != nil {
				r.log.Printf("failed to renew contract %v: %v", c.ID, err)
				continue
			}
			f.mu.Lock()
			f.contracts[newID] = fileContract{
				ID:          newID,
				IP:          c.IP,
				Pieces:      c.Pieces,
				WindowStart: newHeight,
			}
			// need to delete the old contract; otherwise f.expiringContracts
			// will continue to return it
			delete(f.contracts, c.ID)
			f.mu.Unlock()
		}
	}

	// save the repaired file data
	err = r.saveFile(f)
	if err != nil {
		// definitely bad, but we probably shouldn't delete from the
		// repair set if this happens
		r.log.Printf("failed to save repaired file %v: %v", name, err)
	}
}
<commit_msg>fix critical pieceIndex bug in f.repair<commit_after>package renter

import (
	"io"
	"os"
	"sync"
	"time"

	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/modules/renter/hostdb"
	"github.com/NebulousLabs/Sia/types"
)

const (
	// When a file contract is within this many blocks of expiring, the renter
	// will attempt to renew the contract.
	renewThreshold = 2000

	hostTimeout = 15 * time.Second
)

// repair attempts to repair a file chunk by uploading its pieces to more
// hosts.
func (f *file) repair(chunkIndex uint64, missingPieces []uint64, r io.ReaderAt, hosts []hostdb.Uploader) error {
	// read chunk data and encode
	chunk := make([]byte, f.chunkSize())
	_, err := r.ReadAt(chunk, int64(chunkIndex*f.chunkSize()))
	if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
		return err
	}
	pieces, err := f.erasureCode.Encode(chunk)
	if err != nil {
		return err
	}
	// encrypt pieces
	for i := range pieces {
		key := deriveKey(f.masterKey, chunkIndex, uint64(i))
		pieces[i], err = key.EncryptBytes(pieces[i])
		if err != nil {
			return err
		}
	}

	// upload one piece per host
	numPieces := len(missingPieces)
	if len(hosts) < numPieces {
		numPieces = len(hosts)
	}
	var wg sync.WaitGroup
	wg.Add(numPieces)
	for i := 0; i < numPieces; i++ {
		go func(i int) {
			defer wg.Done()
			host := hosts[i]
			pieceIndex := missingPieces[i]

			// upload data to host
			offset, err := host.Upload(pieces[pieceIndex])
			if err != nil {
				return
			}

			// create contract entry, if necessary
			f.mu.Lock()
			defer f.mu.Unlock()
			contract, ok := f.contracts[host.ContractID()]
			if !ok {
				contract = fileContract{
					ID:          host.ContractID(),
					IP:          host.Address(),
					WindowStart: host.EndHeight(),
				}
			}

			// update contract
			contract.Pieces = append(contract.Pieces, pieceData{
				Chunk:  chunkIndex,
				Piece:  pieceIndex,
				Offset: offset,
			})
			f.contracts[host.ContractID()] = contract
		}(i)
	}
	wg.Wait()

	return nil
}

// threadedRepairLoop improves the health of files tracked by the renter by
// reuploading their missing pieces. Multiple repair attempts may be necessary
// before the file reaches full redundancy.
func (r *Renter) threadedRepairLoop() {
	for {
		time.Sleep(5 * time.Second)

		if !r.wallet.Unlocked() {
			continue
		}

		// make copy of repair set under lock
		repairing := make(map[string]trackedFile)
		id := r.mu.RLock()
		for name, meta := range r.tracking {
			repairing[name] = meta
		}
		r.mu.RUnlock(id)

		for name, meta := range repairing {
			r.threadedRepairFile(name, meta)
		}
	}
}

// incompleteChunks returns a map of chunks containing pieces that have not
// been uploaded.
func (f *file) incompleteChunks() map[uint64][]uint64 {
	f.mu.RLock()
	defer f.mu.RUnlock()

	present := make([][]bool, f.numChunks())
	for i := range present {
		present[i] = make([]bool, f.erasureCode.NumPieces())
	}
	for _, fc := range f.contracts {
		for _, p := range fc.Pieces {
			present[p.Chunk][p.Piece] = true
		}
	}

	incomplete := make(map[uint64][]uint64)
	for chunkIndex, pieceBools := range present {
		for pieceIndex, ok := range pieceBools {
			if !ok {
				incomplete[uint64(chunkIndex)] = append(incomplete[uint64(chunkIndex)], uint64(pieceIndex))
			}
		}
	}
	return incomplete
}

// chunkHosts returns the hosts storing the given chunk.
func (f *file) chunkHosts(chunk uint64) []modules.NetAddress {
	f.mu.RLock()
	defer f.mu.RUnlock()

	var old []modules.NetAddress
	for _, fc := range f.contracts {
		for _, p := range fc.Pieces {
			if p.Chunk == chunk {
				old = append(old, fc.IP)
				break
			}
		}
	}
	return old
}

// expiringContracts returns the contracts that will expire soon.
// TODO: what if contract has fully expired?
func (f *file) expiringContracts(height types.BlockHeight) []fileContract {
	f.mu.RLock()
	defer f.mu.RUnlock()

	var expiring []fileContract
	for _, fc := range f.contracts {
		if height > fc.WindowStart-renewThreshold {
			expiring = append(expiring, fc)
		}
	}
	return expiring
}

// threadedRepairFile repairs and saves an individual file.
func (r *Renter) threadedRepairFile(name string, meta trackedFile) {
	// helper function
	logAndRemove := func(fmt string, args ...interface{}) {
		r.log.Printf(fmt, args...)
		id := r.mu.Lock()
		delete(r.tracking, name)
		r.mu.Unlock(id)
	}

	id := r.mu.RLock()
	f, ok := r.files[name]
	r.mu.RUnlock(id)
	if !ok {
		logAndRemove("removing %v from repair set: no longer tracking that file", name)
		return
	}

	// check for expiration
	height := r.cs.Height()
	if meta.EndHeight != 0 && meta.EndHeight < height {
		logAndRemove("removing %v from repair set: storage period has ended", name)
		return
	}

	// open file handle
	handle, err := os.Open(meta.RepairPath)
	if err != nil {
		logAndRemove("removing %v from repair set: %v", name, err)
		return
	}
	defer handle.Close()

	// check for un-uploaded pieces
	badChunks := f.incompleteChunks()
	if len(badChunks) == 0 {
		return
	}

	r.log.Printf("repairing %v chunks of %v", len(badChunks), name)

	// create host pool
	contractSize := f.pieceSize * f.numChunks() // each host gets one piece of each chunk
	var duration types.BlockHeight = defaultDuration
	if meta.EndHeight != 0 {
		duration = meta.EndHeight - height
	}
	pool, err := r.hostDB.NewPool(contractSize, duration)
	if err != nil {
		r.log.Printf("failed to repair %v: %v", name, err)
		return
	}
	defer pool.Close() // heh

	for chunk, pieces := range badChunks {
		// determine host set
		old := f.chunkHosts(chunk)
		hosts := pool.UniqueHosts(f.erasureCode.NumPieces()-len(old), old)
		if len(hosts) == 0 {
			r.log.Printf("aborting repair of %v: not enough hosts", name)
			break
		}
		// upload to new hosts
		err = f.repair(chunk, pieces, handle, hosts)
		if err != nil {
			r.log.Printf("aborting repair of %v: %v", name, err)
			break
		}
	}

	// renew expiring contracts
	if meta.EndHeight == 0 {
		var expiringContracts []fileContract
		expiringContracts = f.expiringContracts(height)
		r.log.Printf("renewing %v contracts of %v", len(expiringContracts), name)
		for _, c := range expiringContracts {
			newHeight := height + defaultDuration
			newID, err := r.hostDB.Renew(c.ID, newHeight)
			if err != nil {
				r.log.Printf("failed to renew contract %v: %v", c.ID, err)
				continue
			}
			f.mu.Lock()
			f.contracts[newID] = fileContract{
				ID:          newID,
				IP:          c.IP,
				Pieces:      c.Pieces,
				WindowStart: newHeight,
			}
			// need to delete the old contract; otherwise f.expiringContracts
			// will continue to return it
			delete(f.contracts, c.ID)
			f.mu.Unlock()
		}
	}

	// save the repaired file data
	err = r.saveFile(f)
	if err != nil {
		// definitely bad, but we probably shouldn't delete from the
		// repair set if this happens
		r.log.Printf("failed to save repaired file %v: %v", name, err)
	}
}
<|endoftext|>


<commit_before>package renter

import (
	"errors"
	"io"
	"os"
	"sync"
	"sync/atomic"

	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

const (
	defaultDuration     = 6000 // Duration that hosts will hold onto the file
	defaultDataPieces   = 2    // Data pieces per erasure-coded chunk
	defaultParityPieces = 10   // Parity pieces per erasure-coded chunk

	// piece sizes
	// NOTE: The encryption overhead is subtracted so that encrypted piece
	// will always be a multiple of 64 (i.e. crypto.SegmentSize). Without this
	// property, revisions break the file's Merkle root.
	defaultPieceSize = 1<<22 - crypto.TwofishOverhead // 4 MiB
	smallPieceSize   = 1<<16 - crypto.TwofishOverhead // 64 KiB
)

type uploadPiece struct {
	data       []byte
	chunkIndex uint64
	pieceIndex uint64
}

// An uploader uploads pieces to a host. This interface exists to facilitate
// easy testing.
type uploader interface {
	// addPiece uploads a piece to the uploader.
	addPiece(uploadPiece) error

	// fileContract returns the fileContract containing the metadata of all
	// previously added pieces.
	fileContract() fileContract
}

// upload reads chunks from r and uploads them to hosts. It spawns a worker
// for each host, and instructs them to upload pieces of each chunk.
func (f *file) upload(r io.Reader, hosts []uploader) error {
	// encode and upload each chunk
	var wg sync.WaitGroup
	for i := uint64(0); ; i++ {
		// read next chunk
		chunk := make([]byte, f.chunkSize())
		_, err := io.ReadFull(r, chunk)
		if err == io.EOF {
			break
		} else if err != nil && err != io.ErrUnexpectedEOF {
			return err
		}
		// encode
		pieces, err := f.erasureCode.Encode(chunk)
		if err != nil {
			return err
		}
		// upload pieces, split evenly among hosts
		wg.Add(len(pieces))
		for j, data := range pieces {
			go func(j int, data []byte) {
				err := hosts[j%len(hosts)].addPiece(uploadPiece{data, i, uint64(j)})
				if err == nil {
					atomic.AddUint64(&f.bytesUploaded, uint64(len(data)))
				}
				wg.Done()
			}(j, data)
		}
		wg.Wait()
		atomic.AddUint64(&f.chunksUploaded, 1)
	}

	// gather final contracts
	for _, h := range hosts {
		contract := h.fileContract()
		f.contracts[contract.IP] = contract
	}

	return nil
}

// checkWalletBalance looks at an upload and determines if there is enough
// money in the wallet to support such an upload. An error is returned if it is
// determined that there is not enough money.
func (r *Renter) checkWalletBalance(up modules.FileUploadParams) error {
	// Get the size of the file.
	fileInfo, err := os.Stat(up.Filename)
	if err != nil {
		return err
	}
	curSize := types.NewCurrency64(uint64(fileInfo.Size()))

	var averagePrice types.Currency
	sampleSize := up.ErasureCode.NumPieces() * 3 / 2
	hosts := r.hostDB.RandomHosts(sampleSize)
	for _, host := range hosts {
		averagePrice = averagePrice.Add(host.Price)
	}
	if len(hosts) == 0 {
		return errors.New("no hosts!")
	}
	averagePrice = averagePrice.Div(types.NewCurrency64(uint64(len(hosts))))
	estimatedCost := averagePrice.Mul(types.NewCurrency64(uint64(up.Duration))).Mul(curSize)
	bufferedCost := estimatedCost.Mul(types.NewCurrency64(2))

	siacoinBalance, _, _ := r.wallet.ConfirmedBalance()
	if bufferedCost.Cmp(siacoinBalance) > 0 {
		return errors.New("insufficient balance for upload")
	}
	return nil
}

// Upload takes an upload parameters, which contain a file to upload, and then
// creates a redundant copy of the file on the Sia network.
func (r *Renter) Upload(up modules.FileUploadParams) error {
	// Open the file.
	handle, err := os.Open(up.Filename)
	if err != nil {
		return err
	}

	// Check for a nickname conflict.
	lockID := r.mu.RLock()
	_, exists := r.files[up.Nickname]
	r.mu.RUnlock(lockID)
	if exists {
		return errors.New("file with that nickname already exists")
	}

	// Check that the file is less than 5 GiB.
	fileInfo, err := handle.Stat()
	if err != nil {
		return err
	}
	// NOTE: The upload max of 5 GiB is temporary and therefore does not have
	// a constant. This should be removed once micropayments + upload resuming
	// are in place. 5 GiB is chosen to prevent confusion - on anybody's
	// machine any file appearing to be under 5 GB will be below the hard
	// limit.
	if fileInfo.Size() > 5*1024*1024*1024 {
		return errors.New("cannot upload a file larger than 5 GB")
	}

	// Fill in any missing upload params with sensible defaults.
	if up.Duration == 0 {
		up.Duration = defaultDuration
	}
	if up.ErasureCode == nil {
		up.ErasureCode, _ = NewRSCode(defaultDataPieces, defaultParityPieces)
	}
	if up.PieceSize == 0 {
		if fileInfo.Size() > defaultPieceSize {
			up.PieceSize = defaultPieceSize
		} else {
			up.PieceSize = smallPieceSize
		}
	}

	// Check that we have enough money to finance the upload.
	err = r.checkWalletBalance(up)
	if err != nil {
		return err
	}

	// Create file object.
	f := newFile(up.Nickname, up.ErasureCode, up.PieceSize, uint64(fileInfo.Size()))
	f.mode = uint32(fileInfo.Mode())

	// Select and connect to hosts.
	totalsize := up.PieceSize * uint64(up.ErasureCode.NumPieces()) * f.numChunks()
	var hosts []uploader
	randHosts := r.hostDB.RandomHosts(up.ErasureCode.NumPieces())
	for i := range randHosts {
		hostUploader, err := r.newHostUploader(randHosts[i], totalsize, up.Duration, f.masterKey)
		if err != nil {
			continue
		}
		defer hostUploader.Close()
		hosts = append(hosts, hostUploader)
	}
	if len(hosts) < up.ErasureCode.MinPieces() {
		return errors.New("not enough hosts to support upload")
	}

	// Add file to renter.
	lockID = r.mu.Lock()
	r.files[up.Nickname] = f
	r.save()
	r.mu.Unlock(lockID)

	// Upload in parallel.
	err = f.upload(handle, hosts)
	if err != nil {
		// Upload failed; remove the file object.
		lockID = r.mu.Lock()
		delete(r.files, up.Nickname)
		r.save()
		r.mu.Unlock(lockID)
		return errors.New("failed to upload any file pieces")
	}

	// Save the .sia file to the renter directory.
	err = r.saveFile(f)
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>update contracts inside chunk loop<commit_after>package renter

import (
	"errors"
	"io"
	"os"
	"sync"
	"sync/atomic"

	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

const (
	defaultDuration     = 6000 // Duration that hosts will hold onto the file
	defaultDataPieces   = 2    // Data pieces per erasure-coded chunk
	defaultParityPieces = 10   // Parity pieces per erasure-coded chunk

	// piece sizes
	// NOTE: The encryption overhead is subtracted so that encrypted piece
	// will always be a multiple of 64 (i.e. crypto.SegmentSize). Without this
	// property, revisions break the file's Merkle root.
	defaultPieceSize = 1<<22 - crypto.TwofishOverhead // 4 MiB
	smallPieceSize   = 1<<16 - crypto.TwofishOverhead // 64 KiB
)

type uploadPiece struct {
	data       []byte
	chunkIndex uint64
	pieceIndex uint64
}

// An uploader uploads pieces to a host. This interface exists to facilitate
// easy testing.
type uploader interface {
	// addPiece uploads a piece to the uploader.
	addPiece(uploadPiece) error

	// fileContract returns the fileContract containing the metadata of all
	// previously added pieces.
	fileContract() fileContract
}

// upload reads chunks from r and uploads them to hosts. It spawns a worker
// for each host, and instructs them to upload pieces of each chunk.
func (f *file) upload(r io.Reader, hosts []uploader) error {
	// encode and upload each chunk
	var wg sync.WaitGroup
	for i := uint64(0); ; i++ {
		// read next chunk
		chunk := make([]byte, f.chunkSize())
		_, err := io.ReadFull(r, chunk)
		if err == io.EOF {
			break
		} else if err != nil && err != io.ErrUnexpectedEOF {
			return err
		}
		// encode
		pieces, err := f.erasureCode.Encode(chunk)
		if err != nil {
			return err
		}
		// upload pieces, split evenly among hosts
		wg.Add(len(pieces))
		for j, data := range pieces {
			go func(j int, data []byte) {
				err := hosts[j%len(hosts)].addPiece(uploadPiece{data, i, uint64(j)})
				if err == nil {
					atomic.AddUint64(&f.bytesUploaded, uint64(len(data)))
				}
				wg.Done()
			}(j, data)
		}
		wg.Wait()
		atomic.AddUint64(&f.chunksUploaded, 1)

		// update contracts
		for _, h := range hosts {
			contract := h.fileContract()
			f.contracts[contract.IP] = contract
		}
	}

	return nil
}

// checkWalletBalance looks at an upload and determines if there is enough
// money in the wallet to support such an upload. An error is returned if it is
// determined that there is not enough money.
func (r *Renter) checkWalletBalance(up modules.FileUploadParams) error {
	// Get the size of the file.
	fileInfo, err := os.Stat(up.Filename)
	if err != nil {
		return err
	}
	curSize := types.NewCurrency64(uint64(fileInfo.Size()))

	var averagePrice types.Currency
	sampleSize := up.ErasureCode.NumPieces() * 3 / 2
	hosts := r.hostDB.RandomHosts(sampleSize)
	for _, host := range hosts {
		averagePrice = averagePrice.Add(host.Price)
	}
	if len(hosts) == 0 {
		return errors.New("no hosts!")
	}
	averagePrice = averagePrice.Div(types.NewCurrency64(uint64(len(hosts))))
	estimatedCost := averagePrice.Mul(types.NewCurrency64(uint64(up.Duration))).Mul(curSize)
	bufferedCost := estimatedCost.Mul(types.NewCurrency64(2))

	siacoinBalance, _, _ := r.wallet.ConfirmedBalance()
	if bufferedCost.Cmp(siacoinBalance) > 0 {
		return errors.New("insufficient balance for upload")
	}
	return nil
}

// Upload takes an upload parameters, which contain a file to upload, and then
// creates a redundant copy of the file on the Sia network.
func (r *Renter) Upload(up modules.FileUploadParams) error {
	// Open the file.
	handle, err := os.Open(up.Filename)
	if err != nil {
		return err
	}

	// Check for a nickname conflict.
	lockID := r.mu.RLock()
	_, exists := r.files[up.Nickname]
	r.mu.RUnlock(lockID)
	if exists {
		return errors.New("file with that nickname already exists")
	}

	// Check that the file is less than 5 GiB.
	fileInfo, err := handle.Stat()
	if err != nil {
		return err
	}
	// NOTE: The upload max of 5 GiB is temporary and therefore does not have
	// a constant. This should be removed once micropayments + upload resuming
	// are in place. 5 GiB is chosen to prevent confusion - on anybody's
	// machine any file appearing to be under 5 GB will be below the hard
	// limit.
	if fileInfo.Size() > 5*1024*1024*1024 {
		return errors.New("cannot upload a file larger than 5 GB")
	}

	// Fill in any missing upload params with sensible defaults.
	if up.Duration == 0 {
		up.Duration = defaultDuration
	}
	if up.ErasureCode == nil {
		up.ErasureCode, _ = NewRSCode(defaultDataPieces, defaultParityPieces)
	}
	if up.PieceSize == 0 {
		if fileInfo.Size() > defaultPieceSize {
			up.PieceSize = defaultPieceSize
		} else {
			up.PieceSize = smallPieceSize
		}
	}

	// Check that we have enough money to finance the upload.
	err = r.checkWalletBalance(up)
	if err != nil {
		return err
	}

	// Create file object.
	f := newFile(up.Nickname, up.ErasureCode, up.PieceSize, uint64(fileInfo.Size()))
	f.mode = uint32(fileInfo.Mode())

	// Select and connect to hosts.
	totalsize := up.PieceSize * uint64(up.ErasureCode.NumPieces()) * f.numChunks()
	var hosts []uploader
	randHosts := r.hostDB.RandomHosts(up.ErasureCode.NumPieces())
	for i := range randHosts {
		hostUploader, err := r.newHostUploader(randHosts[i], totalsize, up.Duration, f.masterKey)
		if err != nil {
			continue
		}
		defer hostUploader.Close()
		hosts = append(hosts, hostUploader)
	}
	if len(hosts) < up.ErasureCode.MinPieces() {
		return errors.New("not enough hosts to support upload")
	}

	// Add file to renter.
	lockID = r.mu.Lock()
	r.files[up.Nickname] = f
	r.save()
	r.mu.Unlock(lockID)

	// Upload in parallel.
	err = f.upload(handle, hosts)
	if err != nil {
		// Upload failed; remove the file object.
		lockID = r.mu.Lock()
		delete(r.files, up.Nickname)
		r.save()
		r.mu.Unlock(lockID)
		return errors.New("failed to upload any file pieces")
	}

	// Save the .sia file to the renter directory.
	err = r.saveFile(f)
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package renter

import (
	"errors"
	"fmt"
	"os"
	"strings"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/modules"
)

var (
	errInsufficientContracts = errors.New("not enough contracts to upload file")

	// Erasure-coded piece size
	pieceSize = modules.SectorSize - crypto.TwofishOverhead

	// defaultDataPieces is the number of data pieces per erasure-coded chunk
	defaultDataPieces = func() int {
		switch build.Release {
		case "dev":
			return 1
		case "standard":
			return 10
		case "testing":
			return 1
		}
		panic("undefined defaultDataPieces")
	}()

	// defaultParityPieces is the number of parity pieces per erasure-coded
	// chunk
	defaultParityPieces = func() int {
		switch build.Release {
		case "dev":
			return 1
		case "standard":
			return 20
		case "testing":
			return 8
		}
		panic("undefined defaultParityPieces")
	}()
)

// Upload instructs the renter to start tracking a file. The renter will
// automatically upload and repair tracked files using a background loop.
func (r *Renter) Upload(up modules.FileUploadParams) error {
	// Enforce nickname rules.
	if strings.HasPrefix(up.SiaPath, "/") {
		return errors.New("nicknames cannot begin with /")
	}
	if up.SiaPath == "" {
		return ErrEmptyFilename
	}

	// Check for a nickname conflict.
	lockID := r.mu.RLock()
	_, exists := r.files[up.SiaPath]
	r.mu.RUnlock(lockID)
	if exists {
		return ErrPathOverload
	}

	// Fill in any missing upload params with sensible defaults.
	fileInfo, err := os.Stat(up.Source)
	if err != nil {
		return err
	}
	if up.ErasureCode == nil {
		up.ErasureCode, _ = NewRSCode(defaultDataPieces, defaultParityPieces)
	}

	// Check that we have contracts to upload to. We need at least (data +
	// parity/2) contracts; since NumPieces = data + parity, we arrive at the
	// expression below.
	if nContracts := len(r.hostContractor.Contracts()); nContracts < (up.ErasureCode.NumPieces()+up.ErasureCode.MinPieces())/2 && build.Release != "testing" {
		return fmt.Errorf("not enough contracts to upload file: got %v, needed %v", nContracts, (up.ErasureCode.NumPieces()+up.ErasureCode.MinPieces())/2)
	}

	// Create file object.
	f := newFile(up.SiaPath, up.ErasureCode, pieceSize, uint64(fileInfo.Size()))
	f.mode = uint32(fileInfo.Mode())

	// Add file to renter.
	lockID = r.mu.Lock()
	r.files[up.SiaPath] = f
	r.tracking[up.SiaPath] = trackedFile{
		RepairPath: up.Source,
	}
	r.saveSync()
	err = r.saveFile(f)
	r.mu.Unlock(lockID)
	if err != nil {
		return err
	}

	// Send the upload to the repair loop.
	r.newRepairs <- f
	return nil
}
<commit_msg>add validateSiapath to validate renter siapaths on upload<commit_after>package renter

import (
	"errors"
	"fmt"
	"os"
	"strings"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/modules"
)

var (
	errInsufficientContracts = errors.New("not enough contracts to upload file")

	// Erasure-coded piece size
	pieceSize = modules.SectorSize - crypto.TwofishOverhead

	// defaultDataPieces is the number of data pieces per erasure-coded chunk
	defaultDataPieces = func() int {
		switch build.Release {
		case "dev":
			return 1
		case "standard":
			return 10
		case "testing":
			return 1
		}
		panic("undefined defaultDataPieces")
	}()

	// defaultParityPieces is the number of parity pieces per erasure-coded
	// chunk
	defaultParityPieces = func() int {
		switch build.Release {
		case "dev":
			return 1
		case "standard":
			return 20
		case "testing":
			return 8
		}
		panic("undefined defaultParityPieces")
	}()
)

// validateSiapath checks that a Siapath is a legal filename.
// ../ is disallowed to prevent directory traversal,
// and paths must not begin with / or be empty.
func validateSiapath(siapath string) error {
	if strings.HasPrefix(siapath, "/") {
		return errors.New("nicknames cannot begin with /")
	}

	if siapath == "" {
		return ErrEmptyFilename
	}

	if strings.Contains(siapath, "../") {
		return errors.New("directory traversal is not allowed")
	}

	return nil
}

// Upload instructs the renter to start tracking a file. The renter will
// automatically upload and repair tracked files using a background loop.
func (r *Renter) Upload(up modules.FileUploadParams) error {
	// Enforce nickname rules.
	if err := validateSiapath(up.SiaPath); err != nil {
		return err
	}

	// Check for a nickname conflict.
	lockID := r.mu.RLock()
	_, exists := r.files[up.SiaPath]
	r.mu.RUnlock(lockID)
	if exists {
		return ErrPathOverload
	}

	// Fill in any missing upload params with sensible defaults.
	fileInfo, err := os.Stat(up.Source)
	if err != nil {
		return err
	}
	if up.ErasureCode == nil {
		up.ErasureCode, _ = NewRSCode(defaultDataPieces, defaultParityPieces)
	}

	// Check that we have contracts to upload to. We need at least (data +
	// parity/2) contracts; since NumPieces = data + parity, we arrive at the
	// expression below.
	if nContracts := len(r.hostContractor.Contracts()); nContracts < (up.ErasureCode.NumPieces()+up.ErasureCode.MinPieces())/2 && build.Release != "testing" {
		return fmt.Errorf("not enough contracts to upload file: got %v, needed %v", nContracts, (up.ErasureCode.NumPieces()+up.ErasureCode.MinPieces())/2)
	}

	// Create file object.
	f := newFile(up.SiaPath, up.ErasureCode, pieceSize, uint64(fileInfo.Size()))
	f.mode = uint32(fileInfo.Mode())

	// Add file to renter.
	lockID = r.mu.Lock()
	r.files[up.SiaPath] = f
	r.tracking[up.SiaPath] = trackedFile{
		RepairPath: up.Source,
	}
	r.saveSync()
	err = r.saveFile(f)
	r.mu.Unlock(lockID)
	if err != nil {
		return err
	}

	// Send the upload to the repair loop.
	r.newRepairs <- f
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package inode_test

import (
	"math"
	"os"
	"testing"
	"time"

	"golang.org/x/net/context"

	"github.com/googlecloudplatform/gcsfuse/fs/inode"
	"github.com/googlecloudplatform/gcsfuse/lease"
	"github.com/googlecloudplatform/gcsfuse/timeutil"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/gcs/gcsfake"
	"github.com/jacobsa/gcloud/gcs/gcsutil"
	. "github.com/jacobsa/ogletest"
)

func TestFile(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Boilerplate
////////////////////////////////////////////////////////////////////////

const fileInodeID = 17
const fileInodeName = "foo/bar"

type FileTest struct {
	ctx    context.Context
	bucket gcs.Bucket
	leaser lease.FileLeaser
	clock  timeutil.SimulatedClock

	initialContents string
	backingObj      *gcs.Object

	in *inode.FileInode
}

var _ SetUpInterface = &FileTest{}
var _ TearDownInterface = &FileTest{}

func init() { RegisterTestSuite(&FileTest{}) }

func (t *FileTest) SetUp(ti *TestInfo) {
	t.ctx = ti.Ctx
	t.clock.SetTime(time.Date(2012, 8, 15, 22, 56, 0, 0, time.Local))
	t.leaser = lease.NewFileLeaser("", math.MaxInt64)
	t.bucket = gcsfake.NewFakeBucket(&t.clock, "some_bucket")

	// Set up the backing object.
	var err error

	t.initialContents = "taco"
	t.backingObj, err = gcsutil.CreateObject(
		t.ctx,
		t.bucket,
		fileInodeName,
		t.initialContents)

	AssertEq(nil, err)

	// Create the inode.
	t.in = inode.NewFileInode(
		fileInodeID,
		t.backingObj,
		math.MaxUint64, // GCS chunk size
		false,          // Support nlink
		t.bucket,
		t.leaser,
		&t.clock)

	t.in.Lock()
}

func (t *FileTest) TearDown() {
	t.in.Unlock()
}

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *FileTest) ID() {
	ExpectEq(fileInodeID, t.in.ID())
}

func (t *FileTest) Name() {
	ExpectEq(fileInodeName, t.in.Name())
}

func (t *FileTest) InitialSourceGeneration() {
	ExpectEq(t.backingObj.Generation, t.in.SourceGeneration())
}

func (t *FileTest) InitialAttributes() {
	attrs, err := t.in.Attributes(t.ctx)
	AssertEq(nil, err)

	ExpectEq(len(t.initialContents), attrs.Size)
	ExpectEq(1, attrs.Nlink)
	ExpectEq(os.FileMode(0700), attrs.Mode)
	ExpectThat(attrs.Mtime, timeutil.TimeEq(t.backingObj.Updated))
}

func (t *FileTest) Read() {
	// TODO(jacobsa): Test various ranges in a table-driven test. Make sure no
	// EOF.
	AssertTrue(false, "TODO")
}

func (t *FileTest) Write() {
	// TODO(jacobsa): Check attributes and read afterward.
	AssertTrue(false, "TODO")
}

func (t *FileTest) Truncate() {
	// TODO(jacobsa): Check attributes and read afterward.
	AssertTrue(false, "TODO")
}

func (t *FileTest) Sync_NotClobbered() {
	// TODO(jacobsa): Check generation and bucket afterward.
	AssertTrue(false, "TODO")
}

func (t *FileTest) Sync_Clobbered() {
	AssertTrue(false, "TODO")
}
<commit_msg>FileTest.Read<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package inode_test

import (
	"fmt"
	"math"
	"os"
	"testing"
	"time"

	"golang.org/x/net/context"

	"github.com/googlecloudplatform/gcsfuse/fs/inode"
	"github.com/googlecloudplatform/gcsfuse/lease"
	"github.com/googlecloudplatform/gcsfuse/timeutil"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/gcs/gcsfake"
	"github.com/jacobsa/gcloud/gcs/gcsutil"
	. "github.com/jacobsa/ogletest"
)

func TestFile(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Boilerplate
////////////////////////////////////////////////////////////////////////

const fileInodeID = 17
const fileInodeName = "foo/bar"

type FileTest struct {
	ctx    context.Context
	bucket gcs.Bucket
	leaser lease.FileLeaser
	clock  timeutil.SimulatedClock

	initialContents string
	backingObj      *gcs.Object

	in *inode.FileInode
}

var _ SetUpInterface = &FileTest{}
var _ TearDownInterface = &FileTest{}

func init() { RegisterTestSuite(&FileTest{}) }

func (t *FileTest) SetUp(ti *TestInfo) {
	t.ctx = ti.Ctx
	t.clock.SetTime(time.Date(2012, 8, 15, 22, 56, 0, 0, time.Local))
	t.leaser = lease.NewFileLeaser("", math.MaxInt64)
	t.bucket = gcsfake.NewFakeBucket(&t.clock, "some_bucket")

	// Set up the backing object.
	var err error

	t.initialContents = "taco"
	t.backingObj, err = gcsutil.CreateObject(
		t.ctx,
		t.bucket,
		fileInodeName,
		t.initialContents)

	AssertEq(nil, err)

	// Create the inode.
	t.in = inode.NewFileInode(
		fileInodeID,
		t.backingObj,
		math.MaxUint64, // GCS chunk size
		false,          // Support nlink
		t.bucket,
		t.leaser,
		&t.clock)

	t.in.Lock()
}

func (t *FileTest) TearDown() {
	t.in.Unlock()
}

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *FileTest) ID() {
	ExpectEq(fileInodeID, t.in.ID())
}

func (t *FileTest) Name() {
	ExpectEq(fileInodeName, t.in.Name())
}

func (t *FileTest) InitialSourceGeneration() {
	ExpectEq(t.backingObj.Generation, t.in.SourceGeneration())
}

func (t *FileTest) InitialAttributes() {
	attrs, err := t.in.Attributes(t.ctx)
	AssertEq(nil, err)

	ExpectEq(len(t.initialContents), attrs.Size)
	ExpectEq(1, attrs.Nlink)
	ExpectEq(os.FileMode(0700), attrs.Mode)
	ExpectThat(attrs.Mtime, timeutil.TimeEq(t.backingObj.Updated))
}

func (t *FileTest) Read() {
	AssertEq("taco", t.initialContents)

	// Make several reads, checking the expected contents. We should never get an
	// EOF error, since fuseops.ReadFileOp is not supposed to see those.
	testCases := []struct {
		offset   int64
		size     int
		expected string
	}{
		{0, 1, "t"},
		{0, 2, "ta"},
		{0, 3, "tac"},
		{0, 4, "taco"},
		{0, 5, "taco"},

		{1, 1, "a"},
		{1, 2, "ac"},
		{1, 3, "aco"},
		{1, 4, "aco"},

		{3, 1, "o"},
		{3, 2, "o"},

		// Empty ranges
		{0, 0, ""},
		{3, 0, ""},
		{4, 0, ""},
		{4, 1, ""},
		{5, 0, ""},
		{5, 1, ""},
		{5, 2, ""},
	}

	for _, tc := range testCases {
		desc := fmt.Sprintf("offset: %d, size: %d", tc.offset, tc.size)

		data, err := t.in.Read(t.ctx, tc.offset, tc.size)
		AssertEq(nil, err, "%s", desc)
		ExpectEq(tc.expected, string(data), "%s", desc)
	}
}

func (t *FileTest) Write() {
	// TODO(jacobsa): Check attributes and read afterward.
	AssertTrue(false, "TODO")
}

func (t *FileTest) Truncate() {
	// TODO(jacobsa): Check attributes and read afterward.
	AssertTrue(false, "TODO")
}

func (t *FileTest) Sync_NotClobbered() {
	// TODO(jacobsa): Check generation and bucket afterward.
	AssertTrue(false, "TODO")
}

func (t *FileTest) Sync_Clobbered() {
	AssertTrue(false, "TODO")
}
<|endoftext|>


<commit_before>package db

// MemDB is an in-memory database repository implementing the DB interface
// used for testing
type MemDB struct {
	installations map[int]int // accountID -> installationID
}

// Ensure MemDB implements DB
var _ DB = (*MemDB)(nil)

// NewMemDB returns an MemDB
func NewMemDB() *MemDB {
	return &MemDB{
		installations: make(map[int]int),
	}
}

// AddGHInstallation implements DB interface
func (db *MemDB) AddGHInstallation(installationID, accountID int) error {
	db.installations[accountID] = installationID
	return nil
}

// RemoveGHInstallation implements DB interface
func (db *MemDB) RemoveGHInstallation(accountID int) error {
	delete(db.installations, accountID)
	return nil
}

// FindGHInstallation implements DB interface
func (db *MemDB) FindGHInstallation(accountID int) (*GHInstallation, error) {
	if installationID, ok := db.installations[accountID]; ok {
		return &GHInstallation{AccountID: accountID, InstallationID: installationID}, nil
	}
	return nil, nil
}
<commit_msg>db.MemDB allow forcing of errors<commit_after>package db

// MemDB is an in-memory database repository implementing the DB interface
// used for testing
type MemDB struct {
	installations map[int]int // accountID -> installationID
	err           error
}

// Ensure MemDB implements DB
var _ DB = (*MemDB)(nil)

// NewMemDB returns an MemDB
func NewMemDB() *MemDB {
	return &MemDB{
		installations: make(map[int]int),
	}
}

func (db *MemDB) ForceError(err error) {
	db.err = err
}

// AddGHInstallation implements DB interface
func (db *MemDB) AddGHInstallation(installationID, accountID int) error {
	db.installations[accountID] = installationID
	return db.err
}

// RemoveGHInstallation implements DB interface
func (db *MemDB) RemoveGHInstallation(accountID int) error {
	delete(db.installations, accountID)
	return db.err
}

// FindGHInstallation implements DB interface
func (db *MemDB) FindGHInstallation(accountID int) (*GHInstallation, error) {
	if installationID, ok := db.installations[accountID]; ok {
		return &GHInstallation{AccountID: accountID, InstallationID: installationID}, db.err
	}
	return nil, db.err
}
<|endoftext|>


<commit_before>// Package internal centralizes a lot of other boring configuration and startup logic into a common place.
package internal

import (
	"context"
	"flag"
	"fmt"
	"os"

	"go4.org/legal"
	"within.website/confyg/flagconfyg"
	"within.website/ln"
	"within.website/ln/opname"
	"within.website/x/internal/flagenv"
	"within.website/x/internal/manpage"

	// Debug routes
	_ "net/http/pprof"

	// Older projects use .env files, shim in compatibility
	_ "github.com/joho/godotenv/autoload"

	// User agent init hook
	_ "within.website/x/web"
)

var (
	licenseShow = flag.Bool("license", false, "show software licenses?")
	config      = flag.String("config", "", "configuration file, if set (see flagconfyg(4))")
	manpageGen  = flag.Bool("manpage", false, "generate a manpage template?")
)

// HandleStartup optionally shows all software licenses or other things.
// This always loads from the following configuration sources in the following
// order:
//
//     - command line flags (to get -config)
//     - environment variables
//     - configuration file (if -config is set)
//     - command line flags
//
// This is done this way to ensure that command line flags always are the deciding
// factor as an escape hatch.
func HandleStartup() {
	flag.Parse()
	flagenv.Parse()

	ctx := opname.With(context.Background(), "internal.HandleStartup")
	if *config != "" {
		ln.Log(ctx, ln.Info("loading config"), ln.F{"path": *config})

		flagconfyg.CmdParse(*config)
	}
	flag.Parse()

	if *licenseShow {
		fmt.Printf("Licenses for %v\n", os.Args)

		for _, li := range legal.Licenses() {
			fmt.Println(li)
			fmt.Println()
		}

		os.Exit(0)
	}

	if *manpageGen {
		manpage.Spew()
	}
}
<commit_msg>internal: add -write-config global flag<commit_after>// Package internal centralizes a lot of other boring configuration and startup logic into a common place.
package internal

import (
	"context"
	"flag"
	"fmt"
	"io/ioutil"
	"os"

	"go4.org/legal"
	"within.website/confyg/flagconfyg"
	"within.website/ln"
	"within.website/ln/opname"
	"within.website/x/internal/flagenv"
	"within.website/x/internal/manpage"

	// Debug routes
	_ "net/http/pprof"

	// Older projects use .env files, shim in compatibility
	_ "github.com/joho/godotenv/autoload"

	// User agent init hook
	_ "within.website/x/web"
)

var (
	licenseShow = flag.Bool("license", false, "show software licenses?")
	config      = flag.String("config", "", "configuration file, if set (see flagconfyg(4))")
	writeConfig = flag.String("write-config", "", "if set, write flags to this file by name/path")
	manpageGen  = flag.Bool("manpage", false, "generate a manpage template?")
)

// HandleStartup optionally shows all software licenses or other things.
// This always loads from the following configuration sources in the following
// order:
//
//     - command line flags (to get -config)
//     - environment variables
//     - configuration file (if -config is set)
//     - command line flags
//
// This is done this way to ensure that command line flags always are the deciding
// factor as an escape hatch.
func HandleStartup() {
	flag.Parse()
	flagenv.Parse()

	ctx := opname.With(context.Background(), "internal.HandleStartup")
	if val := *writeConfig; val != "" {
		ln.Log(ctx, ln.Info("writing flags to file, remember to remove write-config"), ln.F{"fname": val})
		data := flagconfyg.Dump(flag.CommandLine)
		err := ioutil.WriteFile(val, data, 0644)
		if err != nil {
			ln.FatalErr(ctx, err)
		}
		os.Exit(0)
	}

	if *config != "" {
		ln.Log(ctx, ln.Info("loading config"), ln.F{"path": *config})

		flagconfyg.CmdParse(*config)
	}
	flag.Parse()

	if *licenseShow {
		fmt.Printf("Licenses for %v\n", os.Args)

		for _, li := range legal.Licenses() {
			fmt.Println(li)
			fmt.Println()
		}

		os.Exit(0)
	}

	if *manpageGen {
		manpage.Spew()
	}
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/jaroszan/sip"
	"log"
	"runtime"
	"sync"
	"time"
)

//Create mutex to protect existingSessions
var mu = &sync.RWMutex{}

type sessionData struct {
	ReceivedOK uint8
}

var existingSessions map[string]sessionData

func init() {
	existingSessions = make(map[string]sessionData)
}

func handleIncomingPacket(inbound chan sip.SipMessage, outbound chan sip.SipMessage) {
	for sipMessage := range inbound {
		sipMessage := sipMessage
		go func() {
			mType, mValue, err := sip.ParseFirstLine(sipMessage.FirstLine)
			if err != nil {
				log.Println(err)
				log.Println("Dropping request")
				runtime.Goexit()
			}

			if mType == sip.REQUEST {
				/*if mValue == "INVITE" {
					outboundTrying := sip.PrepareResponse(sipHeaders, 100, "Trying")
					outbound180 := sip.PrepareResponse(sipHeaders, 180, "Ringing")
					outbound180 = sip.AddHeader(outbound180, "Contact", "sip:bob@localhost:5060")
					outboundOK := sip.PrepareResponse(sipHeaders, 200, "OK")
					outboundOK = sip.AddHeader(outboundOK, "Contact", "sip:alice@localhost:5060")
					outbound <- []byte(outboundTrying)
					outbound <- []byte(outbound180)
					outbound <- []byte(outboundOK)
				} else if mValue == "BYE" {
					outboundOK := sip.PrepareResponse(sipHeaders, 200, "OK")
					outbound <- []byte(outboundOK)
				} else {
					log.Println(mValue + " received")
				}*/
			} else if mType == sip.RESPONSE {
				mu.Lock()
				if _, ok := existingSessions[sipMessage.Headers["call-id"]]; !ok {
					existingSessions[sipMessage.Headers["call-id"]] = sessionData{0}
				}
				mu.Unlock()
				if mValue == "200" {
					if sipMessage.Headers["cseq"] == "1 INVITE" {
						mu.Lock()
						isOkReceived := existingSessions[sipMessage.Headers["call-id"]].ReceivedOK
						mu.Unlock()
						if isOkReceived == 0 {
							mu.Lock()
							existingSessions[sipMessage.Headers["call-id"]] = sessionData{1}
							mu.Unlock()
							ackRequest := sip.PrepareInDialogRequest("ACK", "1", sipMessage.Headers)
							outbound <- ackRequest
							byeRequest := sip.PrepareInDialogRequest("BYE", "2", sipMessage.Headers)
							time.Sleep(time.Second * 2)
							outbound <- byeRequest
						} else {
							log.Println("Retransmission received")
						}
					} else if sipMessage.Headers["cseq"] == "2 BYE" {
						mu.Lock()
						delete(existingSessions, sipMessage.Headers["call-id"])
						mu.Unlock()
					}
				} else if mValue < "200" {
					//log.Println("Provisional response received: " + mValue)
				} else {
					log.Println("Response received: " + mValue)
				}
			}
		}()
	}
}

func main() {
	// Initiate TCP connection to remote peer, inbound/outbound are channels are used
	// for receiving and sending messages respectively
	localAddr := "localhost:5160"
	remoteAddr := "localhost:5060"
	transport := "TCP"
	inbound, outbound := sip.StartTCPClient(localAddr, remoteAddr)
	// Goroutine for processing incoming messages
	go handleIncomingPacket(inbound, outbound)

	ticker := time.NewTicker(time.Millisecond * 25)
	go func() {
		for _ = range ticker.C {
			// Prepare INVITE
			newRequest := sip.NewDialog("sip:bob@"+localAddr, "sip:alice@"+remoteAddr, transport)
			outbound <- newRequest
		}
	}()
	time.Sleep(time.Millisecond * 30)
	ticker.Stop()
	time.Sleep(time.Second * 5)
}
<commit_msg>adding license header<commit_after>// Copyright 2016 sip authors. All rights reserved.
// Use of this source code is governed by a MIT-style license that can be
// found in the LICENSE file.

package main

import (
	"github.com/jaroszan/sip"
	"log"
	"runtime"
	"sync"
	"time"
)

//Create mutex to protect existingSessions
var mu = &sync.RWMutex{}

type sessionData struct {
	ReceivedOK uint8
}

var existingSessions map[string]sessionData

func init() {
	existingSessions = make(map[string]sessionData)
}

func handleIncomingPacket(inbound chan sip.SipMessage, outbound chan sip.SipMessage) {
	for sipMessage := range inbound {
		sipMessage := sipMessage
		go func() {
			mType, mValue, err := sip.ParseFirstLine(sipMessage.FirstLine)
			if err != nil {
				log.Println(err)
				log.Println("Dropping request")
				runtime.Goexit()
			}

			if mType == sip.REQUEST {
				/*if mValue == "INVITE" {
					outboundTrying := sip.PrepareResponse(sipHeaders, 100, "Trying")
					outbound180 := sip.PrepareResponse(sipHeaders, 180, "Ringing")
					outbound180 = sip.AddHeader(outbound180, "Contact", "sip:bob@localhost:5060")
					outboundOK := sip.PrepareResponse(sipHeaders, 200, "OK")
					outboundOK = sip.AddHeader(outboundOK, "Contact", "sip:alice@localhost:5060")
					outbound <- []byte(outboundTrying)
					outbound <- []byte(outbound180)
					outbound <- []byte(outboundOK)
				} else if mValue == "BYE" {
					outboundOK := sip.PrepareResponse(sipHeaders, 200, "OK")
					outbound <- []byte(outboundOK)
				} else {
					log.Println(mValue + " received")
				}*/
			} else if mType == sip.RESPONSE {
				mu.Lock()
				if _, ok := existingSessions[sipMessage.Headers["call-id"]]; !ok {
					existingSessions[sipMessage.Headers["call-id"]] = sessionData{0}
				}
				mu.Unlock()
				if mValue == "200" {
					if sipMessage.Headers["cseq"] == "1 INVITE" {
						mu.Lock()
						isOkReceived := existingSessions[sipMessage.Headers["call-id"]].ReceivedOK
						mu.Unlock()
						if isOkReceived == 0 {
							mu.Lock()
							existingSessions[sipMessage.Headers["call-id"]] = sessionData{1}
							mu.Unlock()
							ackRequest := sip.PrepareInDialogRequest("ACK", "1", sipMessage.Headers)
							outbound <- ackRequest
							byeRequest := sip.PrepareInDialogRequest("BYE", "2", sipMessage.Headers)
							time.Sleep(time.Second * 2)
							outbound <- byeRequest
						} else {
							log.Println("Retransmission received")
						}
					} else if sipMessage.Headers["cseq"] == "2 BYE" {
						mu.Lock()
						delete(existingSessions, sipMessage.Headers["call-id"])
						mu.Unlock()
					}
				} else if mValue < "200" {
					//log.Println("Provisional response received: " + mValue)
				} else {
					log.Println("Response received: " + mValue)
				}
			}
		}()
	}
}

func main() {
	// Initiate TCP connection to remote peer, inbound/outbound are channels are used
	// for receiving and sending messages respectively
	localAddr := "localhost:5160"
	remoteAddr := "localhost:5060"
	transport := "TCP"
	inbound, outbound := sip.StartTCPClient(localAddr, remoteAddr)
	// Goroutine for processing incoming messages
	go handleIncomingPacket(inbound, outbound)

	ticker := time.NewTicker(time.Millisecond * 25)
	go func() {
		for _ = range ticker.C {
			// Prepare INVITE
			newRequest := sip.NewDialog("sip:bob@"+localAddr, "sip:alice@"+remoteAddr, transport)
			outbound <- newRequest
		}
	}()
	time.Sleep(time.Millisecond * 30)
	ticker.Stop()
	time.Sleep(time.Second * 5)
}
<|endoftext|>


<commit_before>// Package feed presets rss/atom reader.
package news

import (
	"bytes"
	"encoding/xml"
	"errors"
	"io"
	"io/ioutil"
	"time"

	"github.com/lufia/news/atom"
	"github.com/lufia/news/rss1"
	"github.com/lufia/news/rss2"
)

type distinctElement struct {
	XMLName xml.Name
	Version string `xml:"version,attr"`
}

func (rule distinctElement) Match(v distinctElement) bool {
	x1 := rule.XMLName
	x2 := v.XMLName
	if x1.Space != "" && x1.Space != x2.Space {
		return false
	}
	if x1.Local != x2.Local {
		return false
	}
	if rule.Version != "" && rule.Version != v.Version {
		return false
	}
	return true
}

type Dialect struct {
	Type  string
	Parse func(r io.Reader) (feed interface{}, err error)
}

var (
	rss1Dialect = &Dialect{
		Type: "rss1.0",
		Parse: func(r io.Reader) (feed interface{}, err error) {
			return rss1.Parse(r)
		},
	}
	rss2Dialect = &Dialect{
		Type: "rss2.0",
		Parse: func(r io.Reader) (feed interface{}, err error) {
			return rss2.Parse(r)
		},
	}
	atomDialect = &Dialect{
		Type: "atom",
		Parse: func(r io.Reader) (feed interface{}, err error) {
			return atom.Parse(r)
		},
	}
)

var decisionTable = []struct {
	elem    distinctElement
	dialect *Dialect
}{
	{
		elem: distinctElement{
			XMLName: xml.Name{
				Space: "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
				Local: "RDF",
			},
		},
		dialect: rss1Dialect,
	},
	{
		elem: distinctElement{
			XMLName: xml.Name{
				Local: "rss",
			},
			Version: "2.0",
		},
		dialect: rss2Dialect,
	},
	{
		elem: distinctElement{
			XMLName: xml.Name{
				Space: "http://www.w3.org/2005/Atom",
				Local: "feed",
			},
		},
		dialect: atomDialect,
	},
	{
		elem: distinctElement{
			XMLName: xml.Name{
				Space: "http://purl.org/atom/ns#",
				Local: "feed",
			},
		},
		dialect: atomDialect,
	},
}

var (
	errUnknownDialect = errors.New("unknown dialect")
)

func DetectDialect(r io.Reader) (*Dialect, error) {
	var x distinctElement
	d := xml.NewDecoder(r)
	if err := d.Decode(&x); err != nil {
		return nil, err
	}
	for _, v := range decisionTable {
		if v.elem.Match(x) {
			return v.dialect, nil
		}
	}
	return nil, errUnknownDialect
}

func parse(r io.Reader) (feed interface{}, err error) {
	buf, err := ioutil.ReadAll(r)
	if err != nil {
		return
	}
	fin := bytes.NewReader(buf)
	d, err := DetectDialect(fin)
	if err != nil {
		return
	}
	_, err = fin.Seek(0, 0)
	if err != nil {
		return
	}
	return d.Parse(fin)
}

type Feed struct {
	Title    string
	URL      string
	Summary  string
	Articles []*Article
}

type Article struct {
	Title      string
	ID         string
	URL        string
	Authors    []string
	Published  time.Time
	Categories []string
	Content    string
}

func Parse(r io.Reader) (feed *Feed, err error) {
	p, err := parse(r)
	if err != nil {
		return
	}
	feed = &Feed{}
	switch v := p.(type) {
	case *rss1.Feed:
		err = feed.ImportFromRSS1(v)
		return
	case *rss2.Feed:
		err = feed.ImportFromRSS2(v)
		return
	case *atom.Feed:
		err = feed.ImportFromAtom(v)
		return
	default:
		return nil, errors.New("unknown feed type")
	}
}

func (feed *Feed) ImportFromRSS1(r *rss1.Feed) (err error) {
	feed.Title = r.Channel.Title
	feed.URL = r.Channel.Link
	feed.Summary = r.Channel.Description
	feed.Articles = make([]*Article, len(r.Items))
	for i, item := range r.Items {
		p := &Article{
			Title:     item.Title,
			ID:        r.Channel.Indexes[i].URL,
			URL:       item.Link,
			Authors:   []string{item.Creator},
			Published: item.Date,
			Content:   item.Description,
		}
		feed.Articles[i] = p
	}
	return nil
}

type rss2Item rss2.Item

func (v *rss2Item) Authors() []string {
	if v.Author == "" {
		return []string{}
	}
	return []string{v.Author}
}

func (v *rss2Item) Published() time.Time {
	return time.Time(v.PubDate)
}

func (feed *Feed) ImportFromRSS2(r *rss2.Feed) (err error) {
	feed.Title = r.Channel.Title
	feed.URL = r.Channel.Link
	feed.Summary = r.Channel.Description
	feed.Articles = make([]*Article, len(r.Channel.Items))
	for i, item := range r.Channel.Items {
		v := (*rss2Item)(item)
		p := &Article{
			Title:     item.Title,
			URL:       item.Link,
			Authors:   v.Authors(),
			Published: v.Published(),
			Content:   item.Content(),
		}
		if p.ID, err = item.ID(); err != nil {
			return
		}
		feed.Articles[i] = p
	}
	return
}

func (feed *Feed) ImportFromAtom(r *atom.Feed) (err error) {
	feed.Title = r.Title.Content
	feed.URL = r.AlternateURL()
	feed.Summary = r.Summary
	feed.Articles = make([]*Article, len(r.Entries))
	for i, entry := range r.Entries {
		p := &Article{
			Title:     entry.Title.Content,
			ID:        entry.ID,
			URL:       entry.AlternateURL(),
			Authors:   feed.atomAuthors(entry.Authors),
			Published: entry.PublishedTime(),
		}
		var s string
		s, err = entry.Content.HTML()
		if err != nil {
			return
		}
		p.Content = s
		feed.Articles[i] = p
	}
	return
}

func (feed *Feed) atomAuthors(authors []atom.Person) []string {
	a := make([]string, len(authors))
	for i, p := range authors {
		a[i] = p.Name
	}
	return a
}
<commit_msg>remove vertical tab from feed<commit_after>// Package feed presets rss/atom reader.
package news

import (
	"bytes"
	"encoding/xml"
	"errors"
	"io"
	"io/ioutil"
	"time"
	"unicode/utf8"

	"github.com/lufia/news/atom"
	"github.com/lufia/news/rss1"
	"github.com/lufia/news/rss2"
)

type distinctElement struct {
	XMLName xml.Name
	Version string `xml:"version,attr"`
}

func (rule distinctElement) Match(v distinctElement) bool {
	x1 := rule.XMLName
	x2 := v.XMLName
	if x1.Space != "" && x1.Space != x2.Space {
		return false
	}
	if x1.Local != x2.Local {
		return false
	}
	if rule.Version != "" && rule.Version != v.Version {
		return false
	}
	return true
}

type Dialect struct {
	Type  string
	Parse func(r io.Reader) (feed interface{}, err error)
}

var (
	rss1Dialect = &Dialect{
		Type: "rss1.0",
		Parse: func(r io.Reader) (feed interface{}, err error) {
			return rss1.Parse(r)
		},
	}
	rss2Dialect = &Dialect{
		Type: "rss2.0",
		Parse: func(r io.Reader) (feed interface{}, err error) {
			return rss2.Parse(r)
		},
	}
	atomDialect = &Dialect{
		Type: "atom",
		Parse: func(r io.Reader) (feed interface{}, err error) {
			return atom.Parse(r)
		},
	}
)

var decisionTable = []struct {
	elem    distinctElement
	dialect *Dialect
}{
	{
		elem: distinctElement{
			XMLName: xml.Name{
				Space: "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
				Local: "RDF",
			},
		},
		dialect: rss1Dialect,
	},
	{
		elem: distinctElement{
			XMLName: xml.Name{
				Local: "rss",
			},
			Version: "2.0",
		},
		dialect: rss2Dialect,
	},
	{
		elem: distinctElement{
			XMLName: xml.Name{
				Space: "http://www.w3.org/2005/Atom",
				Local: "feed",
			},
		},
		dialect: atomDialect,
	},
	{
		elem: distinctElement{
			XMLName: xml.Name{
				Space: "http://purl.org/atom/ns#",
				Local: "feed",
			},
		},
		dialect: atomDialect,
	},
}

var (
	errUnknownDialect = errors.New("unknown dialect")
)

func DetectDialect(r io.Reader) (*Dialect, error) {
	var x distinctElement
	d := xml.NewDecoder(r)
	if err := d.Decode(&x); err != nil {
		return nil, err
	}
	for _, v := range decisionTable {
		if v.elem.Match(x) {
			return v.dialect, nil
		}
	}
	return nil, errUnknownDialect
}

// Cleanup discard invalid chars in XML 1.0.
func Cleanup(p []byte) []byte {
	tab := []rune{'\v'}
	for _, c := range tab {
		n := utf8.RuneLen(c)
		for {
			i := bytes.IndexRune(p, c)
			if i < 0 {
				break
			}
			copy(p[i:], p[i+n:])
			p = p[:len(p)-n]
		}
	}
	return p
}

func parse(r io.Reader) (feed interface{}, err error) {
	buf, err := ioutil.ReadAll(r)
	if err != nil {
		return
	}
	buf = Cleanup(buf)
	fin := bytes.NewReader(buf)
	d, err := DetectDialect(fin)
	if err != nil {
		return
	}
	_, err = fin.Seek(0, 0)
	if err != nil {
		return
	}
	return d.Parse(fin)
}

type Feed struct {
	Title    string
	URL      string
	Summary  string
	Articles []*Article
}

type Article struct {
	Title      string
	ID         string
	URL        string
	Authors    []string
	Published  time.Time
	Categories []string
	Content    string
}

func Parse(r io.Reader) (feed *Feed, err error) {
	p, err := parse(r)
	if err != nil {
		return
	}
	feed = &Feed{}
	switch v := p.(type) {
	case *rss1.Feed:
		err = feed.ImportFromRSS1(v)
		return
	case *rss2.Feed:
		err = feed.ImportFromRSS2(v)
		return
	case *atom.Feed:
		err = feed.ImportFromAtom(v)
		return
	default:
		return nil, errors.New("unknown feed type")
	}
}

func (feed *Feed) ImportFromRSS1(r *rss1.Feed) (err error) {
	feed.Title = r.Channel.Title
	feed.URL = r.Channel.Link
	feed.Summary = r.Channel.Description
	feed.Articles = make([]*Article, len(r.Items))
	for i, item := range r.Items {
		p := &Article{
			Title:     item.Title,
			ID:        r.Channel.Indexes[i].URL,
			URL:       item.Link,
			Authors:   []string{item.Creator},
			Published: item.Date,
			Content:   item.Description,
		}
		feed.Articles[i] = p
	}
	return nil
}

type rss2Item rss2.Item

func (v *rss2Item) Authors() []string {
	if v.Author == "" {
		return []string{}
	}
	return []string{v.Author}
}

func (v *rss2Item) Published() time.Time {
	return time.Time(v.PubDate)
}

func (feed *Feed) ImportFromRSS2(r *rss2.Feed) (err error) {
	feed.Title = r.Channel.Title
	feed.URL = r.Channel.Link
	feed.Summary = r.Channel.Description
	feed.Articles = make([]*Article, len(r.Channel.Items))
	for i, item := range r.Channel.Items {
		v := (*rss2Item)(item)
		p := &Article{
			Title:     item.Title,
			URL:       item.Link,
			Authors:   v.Authors(),
			Published: v.Published(),
			Content:   item.Content(),
		}
		if p.ID, err = item.ID(); err != nil {
			return
		}
		feed.Articles[i] = p
	}
	return
}

func (feed *Feed) ImportFromAtom(r *atom.Feed) (err error) {
	feed.Title = r.Title.Content
	feed.URL = r.AlternateURL()
	feed.Summary = r.Summary
	feed.Articles = make([]*Article, len(r.Entries))
	for i, entry := range r.Entries {
		p := &Article{
			Title:     entry.Title.Content,
			ID:        entry.ID,
			URL:       entry.AlternateURL(),
			Authors:   feed.atomAuthors(entry.Authors),
			Published: entry.PublishedTime(),
		}
		var s string
		s, err = entry.Content.HTML()
		if err != nil {
			return
		}
		p.Content = s
		feed.Articles[i] = p
	}
	return
}

func (feed *Feed) atomAuthors(authors []atom.Person) []string {
	a := make([]string, len(authors))
	for i, p := range authors {
		a[i] = p.Name
	}
	return a
}
<|endoftext|>