<commit_before>package fuseklient

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"time"

	"koding/fuseklient/transport"

	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/fuseops"
	"github.com/jacobsa/fuse/fuseutil"
)

// Dir represents a file system directory and implements Node interface. It can
// contain one or more files and directories.
type Dir struct {
	// Entry is generic structure that contains commonality between File and Dir.
	*Entry

	// IDGen is responsible for generating ids for newly created nodes.
	IDGen *IDGen

	////// Entry#RWLock protects the fields below.

	// Entries contains list of files and directories that belong to this
	// directory.
	//
	// Note even if an entry is deleted, it'll still be in this list however
	// the deleted entry's type will be set to `fuseutil.DT_Unknown`, so requests
	// to return entries can be filtered. This is done so we set proper offset
	// position for newly created entries. In other words once an entry is set in
	// an offset position, it should maintain that position always.
	Entries []fuseutil.Dirent

	// EntriesList contains list of files and directories that belong to this
	// directory mapped by entry name for easy lookup.
	EntriesList map[string]Node
}

// NewDir is the required initializer for Dir.
func NewDir(e *Entry, idGen *IDGen) *Dir {
	return &Dir{
		Entry:       e,
		IDGen:       idGen,
		Entries:     []fuseutil.Dirent{},
		EntriesList: map[string]Node{},
	}
}

///// Directory operations

// ReadEntries returns entries starting from specified offset position. If local
// cache is empty, it'll fetch from remote.
func (d *Dir) ReadEntries(offset fuseops.DirOffset) ([]fuseutil.Dirent, error) {
	d.Lock()
	defer d.Unlock()

	var entries = d.Entries
	if len(entries) == 0 {
		if err := d.updateEntriesFromRemote(); err != nil {
			return nil, err
		}

		entries = d.Entries
	}

	// return err if offset is greather than list of entries
	if offset > fuseops.DirOffset(len(entries)) {
		return nil, fuse.EIO
	}

	// filter by entries whose type is not to set fuse.DT_Unknown
	var liveEntries []fuseutil.Dirent
	for _, e := range entries[offset:] {
		if e.Type != fuseutil.DT_Unknown {
			liveEntries = append(liveEntries, e)
		}
	}

	return liveEntries, nil
}

// FindEntryDir finds a directory with the specified name.
func (d *Dir) FindEntryDir(name string) (*Dir, error) {
	d.RLock()
	defer d.RUnlock()

	n, err := d.findEntry(name)
	if err != nil {
		return nil, err
	}

	d, ok := n.(*Dir)
	if !ok {
		return nil, fuse.ENOTDIR
	}

	return d, nil
}

// CreateEntryDir creates an empty directory with specified name and mode.
func (d *Dir) CreateEntryDir(name string, mode os.FileMode) (*Dir, error) {
	d.Lock()
	defer d.Unlock()

	if _, err := d.findEntry(name); err != fuse.ENOENT {
		return nil, fuse.EEXIST
	}

	// write to remote before saving to local or else this'll become divergent
	// when there's network disruptions.
	path := filepath.Join(d.Path, name)
	if err := d.Transport.CreateDir(path, 0755); err != nil {
		return nil, err
	}

	e := &tempEntry{
		Name: name,
		Type: fuseutil.DT_Directory,
		Mode: d.Attrs.Mode,
	}

	child, err := d.initializeChild(e)
	if err != nil {
		return nil, err
	}

	dir, _ := child.(*Dir)
	dir.Attrs.Mode = mode

	return dir, nil
}

///// File operations

// FindEntryFile finds file with specified name.
func (d *Dir) FindEntryFile(name string) (*File, error) {
	d.RLock()
	defer d.RUnlock()

	n, err := d.findEntry(name)
	if err != nil {
		return nil, err
	}

	f, ok := n.(*File)
	if !ok {
		return nil, fuse.EIO
	}

	return f, nil
}

// CreateEntryFile creates an empty file with specified name and mode.
func (d *Dir) CreateEntryFile(name string, mode os.FileMode) (*File, error) {
	d.Lock()
	defer d.Unlock()

	if _, err := d.findEntry(name); err != fuse.ENOENT {
		return nil, fuse.EEXIST
	}

	// write to remote before saving to local or else this'll become divergent
	// when there's network disruptions.
	path := filepath.Join(d.Path, name)
	if err := d.Transport.WriteFile(path, nil); err != nil {
		return nil, err
	}

	e := &tempEntry{
		Name: name,
		Type: fuseutil.DT_File,
		Mode: d.Attrs.Mode,
	}

	child, err := d.initializeChild(e)
	if err != nil {
		return nil, err
	}

	file, _ := child.(*File)
	file.Attrs.Mode = mode

	return file, nil
}

///// Entry operations

// FindEntry finds an entry with specified name.
func (d *Dir) FindEntry(name string) (Node, error) {
	d.RLock()
	defer d.RUnlock()

	return d.findEntry(name)
}

// MoveEntry moves specified entry from here to specified directory. Note
// "move" actually means delete from current directory and add to new directory,
// ie InodeID will be different.
func (d *Dir) MoveEntry(oldName, newName string, newDir *Dir) (Node, error) {
	d.Lock()
	defer d.Unlock()

	child, err := d.findEntry(oldName)
	if err != nil {
		return nil, err
	}

	removedEntry, err := d.removeChild(oldName)
	if err != nil {
		return nil, err
	}

	oldPath := d.GetPathForEntry(oldName)
	newPath := newDir.GetPathForEntry(newName)

	if err := d.Transport.Rename(oldPath, newPath); err != nil {
		return nil, err
	}

	e := &tempEntry{
		Name: newName,
		Type: child.GetType(),
		Mode: child.GetAttrs().Mode,
	}

	newEntry, err := newDir.initializeChild(e)
	if err != nil {
		return nil, err
	}

	switch child.GetType() {
	case fuseutil.DT_Directory:
		dir1 := removedEntry.(*Dir)
		dir2 := newEntry.(*Dir)

		dir2.Entries = dir1.Entries
		dir2.EntriesList = dir1.EntriesList
		dir2.Entry.Parent = newDir
	case fuseutil.DT_File:
		file1 := removedEntry.(*File)

		file2 := newEntry.(*File)
		file2.Entry.Parent = newDir

		if err := file2.updateContentFromRemote(); err != nil {
			return nil, err
		}

		if len(file1.Content) >= len(file2.Content) {
			n := make([]byte, len(file1.Content))
			copy(n, file1.Content)

			file2.Content = n
			file2.Attrs.Size = file1.Attrs.Size
		}
	}

	return newEntry, nil
}

// RemoveEntry removes entry with specified name.
func (d *Dir) RemoveEntry(name string) (Node, error) {
	d.Lock()
	defer d.Unlock()

	return d.removeEntry(name)
}

///// Node interface

// GetType returns fuseutil.DT_Directory for identification for fuse library.
func (d *Dir) GetType() fuseutil.DirentType {
	return fuseutil.DT_Directory
}

// Expire updates the internal cache of the directory. This is used when watcher
// indicates directory has changed in remote. If file exists in local already,
// we update the attributes.
func (d *Dir) Expire() error {
	d.Lock()
	defer d.Unlock()

	return d.updateEntriesFromRemote()
}

func (d *Dir) ToString() string {
	d.RLock()
	defer d.RUnlock()

	eToS := d.Entry.ToString()
	return fmt.Sprintf("%s\ndir: entriesCount=%d", eToS, len(d.EntriesList))
}

///// Helpers

// FindEntryRecursive finds entry with specified path by recursively traversing
// all directories.
func (d *Dir) FindEntryRecursive(path string) (Node, error) {
	d.RLock()
	defer d.RUnlock()

	var (
		last  Node = d
		paths      = strings.Split(path, folderSeparator)
	)

	for _, p := range paths {
		d, ok := last.(*Dir)
		if !ok {
			return nil, fuse.ENOENT
		}

		var err error
		if last, err = d.findEntry(p); err != nil {
			return nil, fuse.ENOENT
		}
	}

	return last, nil
}

// Reset removes internal cache of files and directories.
func (d *Dir) Reset() error {
	d.Lock()
	defer d.Unlock()

	d.Entries = []fuseutil.Dirent{}
	d.EntriesList = map[string]Node{}

	return nil
}

// GetPathForEntry returns full relative path for entry, ie. it combines the
// full path of dir from the mount with the entry. It does not check if entry
// exists.
func (d *Dir) GetPathForEntry(name string) string {
	return filepath.Join(d.Path, name)
}

///// Private helpers

func (d *Dir) removeEntry(name string) (Node, error) {
	entry, err := d.findEntry(name)
	if err != nil {
		return nil, err
	}

	path := filepath.Join(d.Path, name)
	if err := d.Transport.Remove(path); err != nil {
		return nil, err
	}

	if _, err := d.removeChild(name); err != nil {
		return nil, err
	}

	return entry, nil
}

func (d *Dir) findEntry(name string) (Node, error) {
	child, ok := d.EntriesList[name]
	if !ok {
		return nil, fuse.ENOENT
	}

	return child, nil
}

func (d *Dir) updateEntriesFromRemote() error {
	var prevEntries = make(map[string]bool, len(d.Entries))
	for _, e := range d.Entries {
		prevEntries[e.Name] = false
	}

	entries, err := d.getEntriesFromRemote()
	if err != nil {
		return err
	}

	for _, e := range entries {
		prevEntries[e.Name] = true

		localEntry, err := d.findEntry(e.Name)
		if err != nil {
			if _, err := d.initializeChild(e); err != nil {
				return err
			}
			continue
		}

		attrs := d.initializeAttrs(e)
		localEntry.SetAttrs(attrs)
	}

	// remove entries not in recently fetched list, ie they've been
	// deleted since last seen
	for entryName, wasSeen := range prevEntries {
		if !wasSeen {
			d.removeEntry(entryName)
		}
	}

	return nil
}

func newTempEntry(file *transport.GetInfoRes) *tempEntry {
	fileType := fuseutil.DT_File
	if file.IsDir {
		fileType = fuseutil.DT_Directory
	}

	return &tempEntry{
		Name: file.Name,
		Type: fileType,
		Mode: file.Mode,
		Size: uint64(file.Size),
		Time: file.Time,
	}
}

func (d *Dir) getEntriesFromRemote() ([]*tempEntry, error) {
	res, err := d.Transport.ReadDir(d.Path, false)
	if err != nil {
		return nil, err
	}

	var entries []*tempEntry
	for _, file := range res.Files {
		e := newTempEntry(file)
		entries = append(entries, e)
	}

	return entries, nil
}

func (d *Dir) initializeAttrs(e *tempEntry) fuseops.InodeAttributes {
	var t = e.Time
	if t.IsZero() {
		t = time.Now()
	}

	return fuseops.InodeAttributes{
		Size:   e.Size,
		Uid:    d.Attrs.Uid,
		Gid:    d.Attrs.Gid,
		Mode:   e.Mode,
		Atime:  t,
		Mtime:  t,
		Ctime:  t,
		Crtime: t,
	}
}

func (d *Dir) initializeChild(e *tempEntry) (Node, error) {
	node, err := d.findEntry(e.Name)
	if err == nil {
		return node, nil
	}

	var t = e.Time
	if t.IsZero() {
		t = time.Now()
	}

	attrs := fuseops.InodeAttributes{
		Size:   e.Size,
		Uid:    d.Attrs.Uid,
		Gid:    d.Attrs.Gid,
		Mode:   e.Mode,
		Atime:  t,
		Mtime:  t,
		Ctime:  t,
		Crtime: t,
	}

	n := NewEntry(d, e.Name)
	n.Attrs = attrs

	dirEntry := fuseutil.Dirent{
		Offset: fuseops.DirOffset(len(d.Entries)) + 1, // offset is 1 indexed
		Inode:  n.ID,
		Name:   e.Name,
		Type:   e.Type,
	}

	d.Entries = append(d.Entries, dirEntry)

	var dt Node
	switch e.Type {
	case fuseutil.DT_Directory:
		dt = NewDir(n, d.IDGen)
	case fuseutil.DT_File:
		dt = NewFile(n)
	default:
		return nil, fmt.Errorf("Unknown file type: %v", e.Type)
	}

	d.EntriesList[e.Name] = dt

	return dt, nil
}

func (d *Dir) removeChild(name string) (Node, error) {
	listEntry, err := d.findEntry(name)
	if err != nil {
		return nil, err
	}

	listEntry.Forget()

	delete(d.EntriesList, name)

	for index, mapEntry := range d.Entries {
		if mapEntry.Name == name {
			mapEntry.Type = fuseutil.DT_Unknown
			d.Entries[index] = mapEntry
		}
	}

	return listEntry, nil
}

type tempEntry struct {
	Offset fuseops.DirOffset
	Name   string
	Type   fuseutil.DirentType
	Mode   os.FileMode
	Size   uint64
	Time   time.Time
}
<commit_msg>fuseklient: use mode to create dir in transport<commit_after>package fuseklient

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"time"

	"koding/fuseklient/transport"

	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/fuseops"
	"github.com/jacobsa/fuse/fuseutil"
)

// Dir represents a file system directory and implements Node interface. It can
// contain one or more files and directories.
type Dir struct {
	// Entry is generic structure that contains commonality between File and Dir.
	*Entry

	// IDGen is responsible for generating ids for newly created nodes.
	IDGen *IDGen

	////// Entry#RWLock protects the fields below.

	// Entries contains list of files and directories that belong to this
	// directory.
	//
	// Note even if an entry is deleted, it'll still be in this list however
	// the deleted entry's type will be set to `fuseutil.DT_Unknown`, so requests
	// to return entries can be filtered. This is done so we set proper offset
	// position for newly created entries. In other words once an entry is set in
	// an offset position, it should maintain that position always.
	Entries []fuseutil.Dirent

	// EntriesList contains list of files and directories that belong to this
	// directory mapped by entry name for easy lookup.
	EntriesList map[string]Node
}

// NewDir is the required initializer for Dir.
func NewDir(e *Entry, idGen *IDGen) *Dir {
	return &Dir{
		Entry:       e,
		IDGen:       idGen,
		Entries:     []fuseutil.Dirent{},
		EntriesList: map[string]Node{},
	}
}

///// Directory operations

// ReadEntries returns entries starting from specified offset position. If local
// cache is empty, it'll fetch from remote.
func (d *Dir) ReadEntries(offset fuseops.DirOffset) ([]fuseutil.Dirent, error) {
	d.Lock()
	defer d.Unlock()

	var entries = d.Entries
	if len(entries) == 0 {
		if err := d.updateEntriesFromRemote(); err != nil {
			return nil, err
		}

		entries = d.Entries
	}

	// return err if offset is greather than list of entries
	if offset > fuseops.DirOffset(len(entries)) {
		return nil, fuse.EIO
	}

	// filter by entries whose type is not to set fuse.DT_Unknown
	var liveEntries []fuseutil.Dirent
	for _, e := range entries[offset:] {
		if e.Type != fuseutil.DT_Unknown {
			liveEntries = append(liveEntries, e)
		}
	}

	return liveEntries, nil
}

// FindEntryDir finds a directory with the specified name.
func (d *Dir) FindEntryDir(name string) (*Dir, error) {
	d.RLock()
	defer d.RUnlock()

	n, err := d.findEntry(name)
	if err != nil {
		return nil, err
	}

	d, ok := n.(*Dir)
	if !ok {
		return nil, fuse.ENOTDIR
	}

	return d, nil
}

// CreateEntryDir creates an empty directory with specified name and mode.
func (d *Dir) CreateEntryDir(name string, mode os.FileMode) (*Dir, error) {
	d.Lock()
	defer d.Unlock()

	if _, err := d.findEntry(name); err != fuse.ENOENT {
		return nil, fuse.EEXIST
	}

	// write to remote before saving to local or else this'll become divergent
	// when there's network disruptions.
	path := filepath.Join(d.Path, name)
	if err := d.Transport.CreateDir(path, mode); err != nil {
		return nil, err
	}

	e := &tempEntry{
		Name: name,
		Type: fuseutil.DT_Directory,
		Mode: d.Attrs.Mode,
	}

	child, err := d.initializeChild(e)
	if err != nil {
		return nil, err
	}

	dir, _ := child.(*Dir)
	dir.Attrs.Mode = mode

	return dir, nil
}

///// File operations

// FindEntryFile finds file with specified name.
func (d *Dir) FindEntryFile(name string) (*File, error) {
	d.RLock()
	defer d.RUnlock()

	n, err := d.findEntry(name)
	if err != nil {
		return nil, err
	}

	f, ok := n.(*File)
	if !ok {
		return nil, fuse.EIO
	}

	return f, nil
}

// CreateEntryFile creates an empty file with specified name and mode.
func (d *Dir) CreateEntryFile(name string, mode os.FileMode) (*File, error) {
	d.Lock()
	defer d.Unlock()

	if _, err := d.findEntry(name); err != fuse.ENOENT {
		return nil, fuse.EEXIST
	}

	// write to remote before saving to local or else this'll become divergent
	// when there's network disruptions.
	path := filepath.Join(d.Path, name)
	if err := d.Transport.WriteFile(path, nil); err != nil {
		return nil, err
	}

	e := &tempEntry{
		Name: name,
		Type: fuseutil.DT_File,
		Mode: d.Attrs.Mode,
	}

	child, err := d.initializeChild(e)
	if err != nil {
		return nil, err
	}

	file, _ := child.(*File)
	file.Attrs.Mode = mode

	return file, nil
}

///// Entry operations

// FindEntry finds an entry with specified name.
func (d *Dir) FindEntry(name string) (Node, error) {
	d.RLock()
	defer d.RUnlock()

	return d.findEntry(name)
}

// MoveEntry moves specified entry from here to specified directory. Note
// "move" actually means delete from current directory and add to new directory,
// ie InodeID will be different.
func (d *Dir) MoveEntry(oldName, newName string, newDir *Dir) (Node, error) {
	d.Lock()
	defer d.Unlock()

	child, err := d.findEntry(oldName)
	if err != nil {
		return nil, err
	}

	removedEntry, err := d.removeChild(oldName)
	if err != nil {
		return nil, err
	}

	oldPath := d.GetPathForEntry(oldName)
	newPath := newDir.GetPathForEntry(newName)

	if err := d.Transport.Rename(oldPath, newPath); err != nil {
		return nil, err
	}

	e := &tempEntry{
		Name: newName,
		Type: child.GetType(),
		Mode: child.GetAttrs().Mode,
	}

	newEntry, err := newDir.initializeChild(e)
	if err != nil {
		return nil, err
	}

	switch child.GetType() {
	case fuseutil.DT_Directory:
		dir1 := removedEntry.(*Dir)
		dir2 := newEntry.(*Dir)

		dir2.Entries = dir1.Entries
		dir2.EntriesList = dir1.EntriesList
		dir2.Entry.Parent = newDir
	case fuseutil.DT_File:
		file1 := removedEntry.(*File)

		file2 := newEntry.(*File)
		file2.Entry.Parent = newDir

		if err := file2.updateContentFromRemote(); err != nil {
			return nil, err
		}

		if len(file1.Content) >= len(file2.Content) {
			n := make([]byte, len(file1.Content))
			copy(n, file1.Content)

			file2.Content = n
			file2.Attrs.Size = file1.Attrs.Size
		}
	}

	return newEntry, nil
}

// RemoveEntry removes entry with specified name.
func (d *Dir) RemoveEntry(name string) (Node, error) {
	d.Lock()
	defer d.Unlock()

	return d.removeEntry(name)
}

///// Node interface

// GetType returns fuseutil.DT_Directory for identification for fuse library.
func (d *Dir) GetType() fuseutil.DirentType {
	return fuseutil.DT_Directory
}

// Expire updates the internal cache of the directory. This is used when watcher
// indicates directory has changed in remote. If file exists in local already,
// we update the attributes.
func (d *Dir) Expire() error {
	d.Lock()
	defer d.Unlock()

	return d.updateEntriesFromRemote()
}

func (d *Dir) ToString() string {
	d.RLock()
	defer d.RUnlock()

	eToS := d.Entry.ToString()
	return fmt.Sprintf("%s\ndir: entriesCount=%d", eToS, len(d.EntriesList))
}

///// Helpers

// FindEntryRecursive finds entry with specified path by recursively traversing
// all directories.
func (d *Dir) FindEntryRecursive(path string) (Node, error) {
	d.RLock()
	defer d.RUnlock()

	var (
		last  Node = d
		paths      = strings.Split(path, folderSeparator)
	)

	for _, p := range paths {
		d, ok := last.(*Dir)
		if !ok {
			return nil, fuse.ENOENT
		}

		var err error
		if last, err = d.findEntry(p); err != nil {
			return nil, fuse.ENOENT
		}
	}

	return last, nil
}

// Reset removes internal cache of files and directories.
func (d *Dir) Reset() error {
	d.Lock()
	defer d.Unlock()

	d.Entries = []fuseutil.Dirent{}
	d.EntriesList = map[string]Node{}

	return nil
}

// GetPathForEntry returns full relative path for entry, ie. it combines the
// full path of dir from the mount with the entry. It does not check if entry
// exists.
func (d *Dir) GetPathForEntry(name string) string {
	return filepath.Join(d.Path, name)
}

///// Private helpers

func (d *Dir) removeEntry(name string) (Node, error) {
	entry, err := d.findEntry(name)
	if err != nil {
		return nil, err
	}

	path := filepath.Join(d.Path, name)
	if err := d.Transport.Remove(path); err != nil {
		return nil, err
	}

	if _, err := d.removeChild(name); err != nil {
		return nil, err
	}

	return entry, nil
}

func (d *Dir) findEntry(name string) (Node, error) {
	child, ok := d.EntriesList[name]
	if !ok {
		return nil, fuse.ENOENT
	}

	return child, nil
}

func (d *Dir) updateEntriesFromRemote() error {
	var prevEntries = make(map[string]bool, len(d.Entries))
	for _, e := range d.Entries {
		prevEntries[e.Name] = false
	}

	entries, err := d.getEntriesFromRemote()
	if err != nil {
		return err
	}

	for _, e := range entries {
		prevEntries[e.Name] = true

		localEntry, err := d.findEntry(e.Name)
		if err != nil {
			if _, err := d.initializeChild(e); err != nil {
				return err
			}
			continue
		}

		attrs := d.initializeAttrs(e)
		localEntry.SetAttrs(attrs)
	}

	// remove entries not in recently fetched list, ie they've been
	// deleted since last seen
	for entryName, wasSeen := range prevEntries {
		if !wasSeen {
			d.removeEntry(entryName)
		}
	}

	return nil
}

func newTempEntry(file *transport.GetInfoRes) *tempEntry {
	fileType := fuseutil.DT_File
	if file.IsDir {
		fileType = fuseutil.DT_Directory
	}

	return &tempEntry{
		Name: file.Name,
		Type: fileType,
		Mode: file.Mode,
		Size: uint64(file.Size),
		Time: file.Time,
	}
}

func (d *Dir) getEntriesFromRemote() ([]*tempEntry, error) {
	res, err := d.Transport.ReadDir(d.Path, false)
	if err != nil {
		return nil, err
	}

	var entries []*tempEntry
	for _, file := range res.Files {
		e := newTempEntry(file)
		entries = append(entries, e)
	}

	return entries, nil
}

func (d *Dir) initializeAttrs(e *tempEntry) fuseops.InodeAttributes {
	var t = e.Time
	if t.IsZero() {
		t = time.Now()
	}

	return fuseops.InodeAttributes{
		Size:   e.Size,
		Uid:    d.Attrs.Uid,
		Gid:    d.Attrs.Gid,
		Mode:   e.Mode,
		Atime:  t,
		Mtime:  t,
		Ctime:  t,
		Crtime: t,
	}
}

func (d *Dir) initializeChild(e *tempEntry) (Node, error) {
	node, err := d.findEntry(e.Name)
	if err == nil {
		return node, nil
	}

	var t = e.Time
	if t.IsZero() {
		t = time.Now()
	}

	attrs := fuseops.InodeAttributes{
		Size:   e.Size,
		Uid:    d.Attrs.Uid,
		Gid:    d.Attrs.Gid,
		Mode:   e.Mode,
		Atime:  t,
		Mtime:  t,
		Ctime:  t,
		Crtime: t,
	}

	n := NewEntry(d, e.Name)
	n.Attrs = attrs

	dirEntry := fuseutil.Dirent{
		Offset: fuseops.DirOffset(len(d.Entries)) + 1, // offset is 1 indexed
		Inode:  n.ID,
		Name:   e.Name,
		Type:   e.Type,
	}

	d.Entries = append(d.Entries, dirEntry)

	var dt Node
	switch e.Type {
	case fuseutil.DT_Directory:
		dt = NewDir(n, d.IDGen)
	case fuseutil.DT_File:
		dt = NewFile(n)
	default:
		return nil, fmt.Errorf("Unknown file type: %v", e.Type)
	}

	d.EntriesList[e.Name] = dt

	return dt, nil
}

func (d *Dir) removeChild(name string) (Node, error) {
	listEntry, err := d.findEntry(name)
	if err != nil {
		return nil, err
	}

	listEntry.Forget()

	delete(d.EntriesList, name)

	for index, mapEntry := range d.Entries {
		if mapEntry.Name == name {
			mapEntry.Type = fuseutil.DT_Unknown
			d.Entries[index] = mapEntry
		}
	}

	return listEntry, nil
}

type tempEntry struct {
	Offset fuseops.DirOffset
	Name   string
	Type   fuseutil.DirentType
	Mode   os.FileMode
	Size   uint64
	Time   time.Time
}
<|endoftext|>


<commit_before>// Copyright (c) 2014, Google Inc.  All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tao

import (
	"crypto/rand"
	"crypto/sha256"
	"encoding/base64"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"os/signal"
	"path"
	"strings"
	"syscall"

	"github.com/jlmucb/cloudproxy/go/tao/auth"
	"github.com/jlmucb/cloudproxy/go/util"
)

// A LinuxProcessFactory supports methods for creating Linux processes as
// hosted programs. LinuxProcessFactory implements HostedProgramFactory.
type LinuxProcessFactory struct {
	channelType string
	socketPath  string
}

// NewLinuxProcessFactory returns a new HostedProgramFactory that can create
// linux processes.
func NewLinuxProcessFactory(channelType, socketPath string) HostedProgramFactory {
	return &LinuxProcessFactory{
		channelType: channelType,
		socketPath:  socketPath,
	}
}

// A LinuxProcess represents a hosted program that executes as a linux process.
type HostedProcess struct {

	// The spec from which this process was created.
	spec HostedProgramSpec

	// The value to be used as argv[0]
	Argv0 string

	// A secured, private copy of the executable.
	Temppath string

	// A temporary directory for storing the temporary executable.
	Tempdir string

	// Hash of the executable.
	Hash []byte

	// The underlying process.
	Cmd exec.Cmd

	// The factory responsible for the hosted process.
	Factory *LinuxProcessFactory

	// A channel to be signaled when the process is done.
	Done chan bool
}

// NewHostedProgram initializes, but does not start, a hosted process.
func (lpf *LinuxProcessFactory) NewHostedProgram(spec HostedProgramSpec) (child HostedProgram, err error) {

	// The argv[0] for the child is given by spec.ContainerArgs
	argv0 := spec.Path
	if len(spec.ContainerArgs) == 1 {
		argv0 = spec.ContainerArgs[0]
	} else if len(spec.ContainerArgs) > 0 {
		err = fmt.Errorf("Too many container arguments for process")
		return
	}

	// To avoid a time-of-check-to-time-of-use error, we copy the file
	// bytes to a temp file as we read them. This temp-file path is
	// returned so it can be used to start the program.
	tempdir, err := ioutil.TempDir("/tmp", "cloudproxy_linux_host")
	if err != nil {
		return
	}
	defer func() {
		if err != nil {
			os.RemoveAll(tempdir)
		}
	}()
	if err = os.Chmod(tempdir, 0755); err != nil {
		return
	}

	temppath := path.Join(tempdir, "hosted_program")
	tf, err := os.OpenFile(temppath, os.O_CREATE|os.O_RDWR, 0700)
	defer tf.Close()
	if err != nil {
		return
	}
	if err = tf.Chmod(0755); err != nil {
		return
	}

	inf, err := os.Open(spec.Path)
	defer inf.Close()
	if err != nil {
		return
	}

	// Read from the input file and write to the temp file.
	tr := io.TeeReader(inf, tf)
	b, err := ioutil.ReadAll(tr)
	if err != nil {
		return
	}

	h := sha256.Sum256(b)

	child = &HostedProcess{
		spec:     spec,
		Argv0:    argv0,
		Temppath: temppath,
		Tempdir:  tempdir,
		Hash:     h[:],
		Factory:  lpf,
		Done:     make(chan bool, 1),
	}
	return
}

// Use 24 bytes for the socket name.
const sockNameLen = 24

// Start starts the the hosted process and returns a tao channel to it.
func (p *HostedProcess) Start() (channel io.ReadWriteCloser, err error) {
	var extraFiles []*os.File
	var evar string
	switch p.Factory.channelType {
	case "pipe":
		// Get a pipe pair for communication with the child.
		var serverRead, clientRead, serverWrite, clientWrite *os.File
		serverRead, clientWrite, err = os.Pipe()
		if err != nil {
			return
		}
		defer clientWrite.Close()

		clientRead, serverWrite, err = os.Pipe()
		if err != nil {
			serverRead.Close()
			return
		}
		defer clientRead.Close()

		channel = util.NewPairReadWriteCloser(serverRead, serverWrite)
		extraFiles = []*os.File{clientRead, clientWrite} // fd 3, fd 4

		// Note: ExtraFiles below ensures readfd=3, writefd=4 in child
		evar = HostSpecEnvVar + "=tao::RPC+tao::FDMessageChannel(3, 4)"
	case "unix":
		// Get a random name for the socket.
		nameBytes := make([]byte, sockNameLen)
		if _, err = rand.Read(nameBytes); err != nil {
			return
		}
		sockName := base64.URLEncoding.EncodeToString(nameBytes)
		sockPath := path.Join(p.Factory.socketPath, sockName)
		channel = util.NewUnixSingleReadWriteCloser(sockPath)
		if channel == nil {
			err = fmt.Errorf("Couldn't create a new Unix channel\n")
			return
		}
		evar = HostSpecEnvVar + "=" + sockPath
	default:
		err = fmt.Errorf("invalid channel type '%s'\n", p.Factory.channelType)
		return
	}
	defer func() {
		if err != nil {
			channel.Close()
			channel = nil
		}
	}()

	env := p.spec.Env
	if env == nil {
		env = os.Environ()
	}
	// Make sure that the child knows to use the right kind of channel.
	etvar := HostChannelTypeEnvVar + "=" + p.Factory.channelType
	replaced := false
	replacedType := false
	for i, pair := range env {
		if strings.HasPrefix(pair, HostSpecEnvVar+"=") {
			env[i] = evar
			replaced = true
		}

		if strings.HasPrefix(pair, HostChannelTypeEnvVar+"=") {
			env[i] = etvar
			replacedType = true
		}
	}
	if !replaced {
		env = append(env, evar)
	}

	if !replacedType {
		env = append(env, etvar)
	}

	if (p.spec.Uid == 0 || p.spec.Gid == 0) && !p.spec.Superuser {
		err = fmt.Errorf("Uid and Gid must be nonzero unless Superuser is set\n")
		return
	}

	wd := p.spec.Dir
	if wd == "" {
		wd = p.Tempdir
	}

	// Every hosted process is given its own process group (Setpgid=true). This
	// ensures that hosted processes will not be in orphaned process groups,
	// allowing them to receive job control signals (SIGTTIN, SIGTTOU, and
	// SIGTSTP).
	//
	// If this host is running in "daemon" mode, i.e. without a controlling tty
	// and in our own session and process group, then this host will be (a) the
	// parent of a process in the child's group, (b) in the same session, and
	// (c) not in the same group as the child, so it will serve as the anchor
	// that keeps the child process groups from being considered orphaned.
	//
	// If this host is running in "foreground" mode, i.e. with a controlling tty
	// and as part of our parent process's session but in our own process group,
	// then the same three conditions are satisified, so this host can still
	// serve as the anchor that keeps the child process groups from being
	// considered orphaned. (Note: We could also use Setpid=false in this case,
	// since the host would be part of the child process group and our parent
	// would then meet the requirements.)

	spa := &syscall.SysProcAttr{
		Credential: &syscall.Credential{
			Uid: uint32(p.spec.Uid),
			Gid: uint32(p.spec.Uid),
		},
		// Setsid: true, // Create session.
		Setpgid: true, // Set process group ID to new pid (SYSV setpgrp)
		// Setctty: true, // Set controlling terminal to fd Ctty (only meaningful if Setsid is set)
		// Noctty: true, // Detach fd 0 from controlling terminal
		// Ctty: 0, // Controlling TTY fd (Linux only)
	}
	argv := []string{p.Argv0}
	argv = append(argv, p.spec.Args...)
	p.Cmd = exec.Cmd{
		Path:        p.Temppath,
		Dir:         wd,
		Args:        argv,
		Stdin:       p.spec.Stdin,
		Stdout:      p.spec.Stdout,
		Stderr:      p.spec.Stderr,
		Env:         env,
		ExtraFiles:  extraFiles,
		SysProcAttr: spa,
	}

	if err = p.Cmd.Start(); err != nil {
		return
	}

	// Reap the child when the process dies.
	sc := make(chan os.Signal, 1)
	signal.Notify(sc, syscall.SIGCHLD)
	go func() {
		<-sc
		p.Cmd.Wait()
		signal.Stop(sc)
		os.RemoveAll(p.Tempdir)
		p.Done <- true
		close(p.Done) // prevent any more blocking
	}()

	// TODO(kwalsh) put channel into p, remove the struct in linux_host.go

	return
}

// ExitStatus returns an exit code for the process.
func (p *HostedProcess) ExitStatus() (int, error) {
	s := p.Cmd.ProcessState
	if s == nil {
		return -1, fmt.Errorf("Child has not exited")
	}
	if code, ok := (*s).Sys().(syscall.WaitStatus); ok {
		return int(code), nil
	}
	return -1, fmt.Errorf("Couldn't get exit status\n")
}

// WaitChan returns a chan that will be signaled when the hosted process is
// done.
func (p *HostedProcess) WaitChan() <-chan bool {
	return p.Done
}

// Kill kills an os/exec.Cmd process.
func (p *HostedProcess) Kill() error {
	return p.Cmd.Process.Kill()
}

// Stop tries to send SIGTERM to a process.
func (p *HostedProcess) Stop() error {
	err := syscall.Kill(p.Cmd.Process.Pid, syscall.SIGTERM)
	syscall.Kill(p.Cmd.Process.Pid, syscall.SIGCONT)
	return err
}

// Spec returns the specification used to start the hosted process.
func (p *HostedProcess) Spec() HostedProgramSpec {
	return p.spec
}

// Pid returns the pid of the underlying os/exec.Cmd instance.
func (p *HostedProcess) Pid() int {
	return p.Cmd.Process.Pid
}

// Subprin returns the subprincipal representing the hosted process.
func (p *HostedProcess) Subprin() auth.SubPrin {
	return FormatProcessSubprin(p.spec.Id, p.Hash)
}

// FormatProcessSubprin produces a string that represents a subprincipal with
// the given ID and hash.
func FormatProcessSubprin(id uint, hash []byte) auth.SubPrin {
	var args []auth.Term
	if id != 0 {
		args = append(args, auth.Int(id))
	}
	args = append(args, auth.Bytes(hash))
	return auth.SubPrin{auth.PrinExt{Name: "Program", Arg: args}}
}

func (p *HostedProcess) Cleanup() error {
	// TODO(kwalsh) close channel, maybe also kill process if still running?
	os.RemoveAll(p.Tempdir)
	return nil
}
<commit_msg>return 0 on error case, not -1<commit_after>// Copyright (c) 2014, Google Inc.  All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tao

import (
	"crypto/rand"
	"crypto/sha256"
	"encoding/base64"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"os/signal"
	"path"
	"strings"
	"syscall"

	"github.com/jlmucb/cloudproxy/go/tao/auth"
	"github.com/jlmucb/cloudproxy/go/util"
)

// A LinuxProcessFactory supports methods for creating Linux processes as
// hosted programs. LinuxProcessFactory implements HostedProgramFactory.
type LinuxProcessFactory struct {
	channelType string
	socketPath  string
}

// NewLinuxProcessFactory returns a new HostedProgramFactory that can create
// linux processes.
func NewLinuxProcessFactory(channelType, socketPath string) HostedProgramFactory {
	return &LinuxProcessFactory{
		channelType: channelType,
		socketPath:  socketPath,
	}
}

// A LinuxProcess represents a hosted program that executes as a linux process.
type HostedProcess struct {

	// The spec from which this process was created.
	spec HostedProgramSpec

	// The value to be used as argv[0]
	Argv0 string

	// A secured, private copy of the executable.
	Temppath string

	// A temporary directory for storing the temporary executable.
	Tempdir string

	// Hash of the executable.
	Hash []byte

	// The underlying process.
	Cmd exec.Cmd

	// The factory responsible for the hosted process.
	Factory *LinuxProcessFactory

	// A channel to be signaled when the process is done.
	Done chan bool
}

// NewHostedProgram initializes, but does not start, a hosted process.
func (lpf *LinuxProcessFactory) NewHostedProgram(spec HostedProgramSpec) (child HostedProgram, err error) {

	// The argv[0] for the child is given by spec.ContainerArgs
	argv0 := spec.Path
	if len(spec.ContainerArgs) == 1 {
		argv0 = spec.ContainerArgs[0]
	} else if len(spec.ContainerArgs) > 0 {
		err = fmt.Errorf("Too many container arguments for process")
		return
	}

	// To avoid a time-of-check-to-time-of-use error, we copy the file
	// bytes to a temp file as we read them. This temp-file path is
	// returned so it can be used to start the program.
	tempdir, err := ioutil.TempDir("/tmp", "cloudproxy_linux_host")
	if err != nil {
		return
	}
	defer func() {
		if err != nil {
			os.RemoveAll(tempdir)
		}
	}()
	if err = os.Chmod(tempdir, 0755); err != nil {
		return
	}

	temppath := path.Join(tempdir, "hosted_program")
	tf, err := os.OpenFile(temppath, os.O_CREATE|os.O_RDWR, 0700)
	defer tf.Close()
	if err != nil {
		return
	}
	if err = tf.Chmod(0755); err != nil {
		return
	}

	inf, err := os.Open(spec.Path)
	defer inf.Close()
	if err != nil {
		return
	}

	// Read from the input file and write to the temp file.
	tr := io.TeeReader(inf, tf)
	b, err := ioutil.ReadAll(tr)
	if err != nil {
		return
	}

	h := sha256.Sum256(b)

	child = &HostedProcess{
		spec:     spec,
		Argv0:    argv0,
		Temppath: temppath,
		Tempdir:  tempdir,
		Hash:     h[:],
		Factory:  lpf,
		Done:     make(chan bool, 1),
	}
	return
}

// Use 24 bytes for the socket name.
const sockNameLen = 24

// Start starts the the hosted process and returns a tao channel to it.
func (p *HostedProcess) Start() (channel io.ReadWriteCloser, err error) {
	var extraFiles []*os.File
	var evar string
	switch p.Factory.channelType {
	case "pipe":
		// Get a pipe pair for communication with the child.
		var serverRead, clientRead, serverWrite, clientWrite *os.File
		serverRead, clientWrite, err = os.Pipe()
		if err != nil {
			return
		}
		defer clientWrite.Close()

		clientRead, serverWrite, err = os.Pipe()
		if err != nil {
			serverRead.Close()
			return
		}
		defer clientRead.Close()

		channel = util.NewPairReadWriteCloser(serverRead, serverWrite)
		extraFiles = []*os.File{clientRead, clientWrite} // fd 3, fd 4

		// Note: ExtraFiles below ensures readfd=3, writefd=4 in child
		evar = HostSpecEnvVar + "=tao::RPC+tao::FDMessageChannel(3, 4)"
	case "unix":
		// Get a random name for the socket.
		nameBytes := make([]byte, sockNameLen)
		if _, err = rand.Read(nameBytes); err != nil {
			return
		}
		sockName := base64.URLEncoding.EncodeToString(nameBytes)
		sockPath := path.Join(p.Factory.socketPath, sockName)
		channel = util.NewUnixSingleReadWriteCloser(sockPath)
		if channel == nil {
			err = fmt.Errorf("Couldn't create a new Unix channel\n")
			return
		}
		evar = HostSpecEnvVar + "=" + sockPath
	default:
		err = fmt.Errorf("invalid channel type '%s'\n", p.Factory.channelType)
		return
	}
	defer func() {
		if err != nil {
			channel.Close()
			channel = nil
		}
	}()

	env := p.spec.Env
	if env == nil {
		env = os.Environ()
	}
	// Make sure that the child knows to use the right kind of channel.
	etvar := HostChannelTypeEnvVar + "=" + p.Factory.channelType
	replaced := false
	replacedType := false
	for i, pair := range env {
		if strings.HasPrefix(pair, HostSpecEnvVar+"=") {
			env[i] = evar
			replaced = true
		}

		if strings.HasPrefix(pair, HostChannelTypeEnvVar+"=") {
			env[i] = etvar
			replacedType = true
		}
	}
	if !replaced {
		env = append(env, evar)
	}

	if !replacedType {
		env = append(env, etvar)
	}

	if (p.spec.Uid == 0 || p.spec.Gid == 0) && !p.spec.Superuser {
		err = fmt.Errorf("Uid and Gid must be nonzero unless Superuser is set\n")
		return
	}

	wd := p.spec.Dir
	if wd == "" {
		wd = p.Tempdir
	}

	// Every hosted process is given its own process group (Setpgid=true). This
	// ensures that hosted processes will not be in orphaned process groups,
	// allowing them to receive job control signals (SIGTTIN, SIGTTOU, and
	// SIGTSTP).
	//
	// If this host is running in "daemon" mode, i.e. without a controlling tty
	// and in our own session and process group, then this host will be (a) the
	// parent of a process in the child's group, (b) in the same session, and
	// (c) not in the same group as the child, so it will serve as the anchor
	// that keeps the child process groups from being considered orphaned.
	//
	// If this host is running in "foreground" mode, i.e. with a controlling tty
	// and as part of our parent process's session but in our own process group,
	// then the same three conditions are satisified, so this host can still
	// serve as the anchor that keeps the child process groups from being
	// considered orphaned. (Note: We could also use Setpid=false in this case,
	// since the host would be part of the child process group and our parent
	// would then meet the requirements.)

	spa := &syscall.SysProcAttr{
		Credential: &syscall.Credential{
			Uid: uint32(p.spec.Uid),
			Gid: uint32(p.spec.Uid),
		},
		// Setsid: true, // Create session.
		Setpgid: true, // Set process group ID to new pid (SYSV setpgrp)
		// Setctty: true, // Set controlling terminal to fd Ctty (only meaningful if Setsid is set)
		// Noctty: true, // Detach fd 0 from controlling terminal
		// Ctty: 0, // Controlling TTY fd (Linux only)
	}
	argv := []string{p.Argv0}
	argv = append(argv, p.spec.Args...)
	p.Cmd = exec.Cmd{
		Path:        p.Temppath,
		Dir:         wd,
		Args:        argv,
		Stdin:       p.spec.Stdin,
		Stdout:      p.spec.Stdout,
		Stderr:      p.spec.Stderr,
		Env:         env,
		ExtraFiles:  extraFiles,
		SysProcAttr: spa,
	}

	if err = p.Cmd.Start(); err != nil {
		return
	}

	// Reap the child when the process dies.
	sc := make(chan os.Signal, 1)
	signal.Notify(sc, syscall.SIGCHLD)
	go func() {
		<-sc
		p.Cmd.Wait()
		signal.Stop(sc)
		os.RemoveAll(p.Tempdir)
		p.Done <- true
		close(p.Done) // prevent any more blocking
	}()

	// TODO(kwalsh) put channel into p, remove the struct in linux_host.go

	return
}

// ExitStatus returns an exit code for the process.
func (p *HostedProcess) ExitStatus() (int, error) {
	s := p.Cmd.ProcessState
	if s == nil {
		return 0, fmt.Errorf("Child has not exited")
	}
	if code, ok := (*s).Sys().(syscall.WaitStatus); ok {
		return int(code), nil
	}
	return 0, fmt.Errorf("Couldn't get exit status\n")
}

// WaitChan returns a chan that will be signaled when the hosted process is
// done.
func (p *HostedProcess) WaitChan() <-chan bool {
	return p.Done
}

// Kill kills an os/exec.Cmd process.
func (p *HostedProcess) Kill() error {
	return p.Cmd.Process.Kill()
}

// Stop tries to send SIGTERM to a process.
func (p *HostedProcess) Stop() error {
	err := syscall.Kill(p.Cmd.Process.Pid, syscall.SIGTERM)
	syscall.Kill(p.Cmd.Process.Pid, syscall.SIGCONT)
	return err
}

// Spec returns the specification used to start the hosted process.
func (p *HostedProcess) Spec() HostedProgramSpec {
	return p.spec
}

// Pid returns the pid of the underlying os/exec.Cmd instance.
func (p *HostedProcess) Pid() int {
	return p.Cmd.Process.Pid
}

// Subprin returns the subprincipal representing the hosted process.
func (p *HostedProcess) Subprin() auth.SubPrin {
	return FormatProcessSubprin(p.spec.Id, p.Hash)
}

// FormatProcessSubprin produces a string that represents a subprincipal with
// the given ID and hash.
func FormatProcessSubprin(id uint, hash []byte) auth.SubPrin {
	var args []auth.Term
	if id != 0 {
		args = append(args, auth.Int(id))
	}
	args = append(args, auth.Bytes(hash))
	return auth.SubPrin{auth.PrinExt{Name: "Program", Arg: args}}
}

func (p *HostedProcess) Cleanup() error {
	// TODO(kwalsh) close channel, maybe also kill process if still running?
	os.RemoveAll(p.Tempdir)
	return nil
}
<|endoftext|>


<commit_before>package gorums

import (
	"context"
	"math"
	"math/rand"
	"sync"
	"sync/atomic"
	"time"

	"github.com/relab/gorums/ordering"
	"google.golang.org/grpc"
	"google.golang.org/grpc/backoff"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
	"google.golang.org/protobuf/reflect/protoreflect"
)

type gorumsStreamRequest struct {
	ctx  context.Context
	msg  *Message
	opts callOptions
}

type gorumsStreamResult struct {
	nid   uint32
	reply protoreflect.ProtoMessage
	err   error
}

type receiveQueue struct {
	msgID    uint64
	recvQ    map[uint64]chan *gorumsStreamResult
	recvQMut sync.RWMutex
}

func newReceiveQueue() *receiveQueue {
	return &receiveQueue{
		recvQ: make(map[uint64]chan *gorumsStreamResult),
	}
}

func (m *receiveQueue) nextMsgID() uint64 {
	return atomic.AddUint64(&m.msgID, 1)
}

func (m *receiveQueue) putChan(id uint64, c chan *gorumsStreamResult) {
	m.recvQMut.Lock()
	m.recvQ[id] = c
	m.recvQMut.Unlock()
}

func (m *receiveQueue) deleteChan(id uint64) {
	m.recvQMut.Lock()
	delete(m.recvQ, id)
	m.recvQMut.Unlock()
}

func (m *receiveQueue) putResult(id uint64, result *gorumsStreamResult) {
	m.recvQMut.RLock()
	c, ok := m.recvQ[id]
	m.recvQMut.RUnlock()
	if ok {
		c <- result
	}
}

type orderedNodeStream struct {
	*receiveQueue
	sendQ        chan gorumsStreamRequest
	node         *Node // needed for ID and setLastError
	backoff      backoff.Config
	rand         *rand.Rand
	gorumsClient ordering.GorumsClient
	gorumsStream ordering.Gorums_NodeStreamClient
	streamMut    sync.RWMutex
	streamBroken bool
	parentCtx    context.Context
	streamCtx    context.Context
	cancelStream context.CancelFunc
}

func (s *orderedNodeStream) connectOrderedStream(ctx context.Context, conn *grpc.ClientConn) error {
	var err error
	s.parentCtx = ctx
	s.streamCtx, s.cancelStream = context.WithCancel(s.parentCtx)
	s.gorumsClient = ordering.NewGorumsClient(conn)
	s.gorumsStream, err = s.gorumsClient.NodeStream(s.streamCtx)
	if err != nil {
		return err
	}
	go s.sendMsgs()
	go s.recvMsgs()
	return nil
}

func (s *orderedNodeStream) sendMsg(req gorumsStreamRequest) (err error) {
	s.streamMut.RLock()
	defer s.streamMut.RUnlock()

	c := make(chan struct{})

	// wait for either the message to be sent, or the request context being cancelled.
	// if the request context was cancelled, then we most likely have a blocked stream.
	go func() {
		select {
		case <-c:
		case <-req.ctx.Done():
			s.cancelStream()
		}
	}()

	err = s.gorumsStream.SendMsg(req.msg)
	if err != nil {
		s.node.setLastErr(err)
		s.streamBroken = true
	}
	c <- struct{}{}

	// unblock the waiting caller when sendAsync is not enabled
	if req.opts.callType == E_Multicast || req.opts.callType == E_Unicast && !req.opts.sendAsync {
		s.putResult(req.msg.Metadata.MessageID, &gorumsStreamResult{})
	}

	return err
}

func (s *orderedNodeStream) sendMsgs() {
	var req gorumsStreamRequest
	for {
		select {
		case <-s.parentCtx.Done():
			return
		case req = <-s.sendQ:
		}
		// return error if stream is broken
		if s.streamBroken {
			err := status.Errorf(codes.Unavailable, "stream is down")
			s.putResult(req.msg.Metadata.MessageID, &gorumsStreamResult{nid: s.node.ID(), reply: nil, err: err})
			continue
		}
		// else try to send message
		err := s.sendMsg(req)
		if err != nil {
			// return the error
			s.putResult(req.msg.Metadata.MessageID, &gorumsStreamResult{nid: s.node.ID(), reply: nil, err: err})
		}
	}
}

func (s *orderedNodeStream) recvMsgs() {
	for {
		resp := newMessage(responseType)
		s.streamMut.RLock()
		err := s.gorumsStream.RecvMsg(resp)
		if err != nil {
			s.streamBroken = true
			s.streamMut.RUnlock()
			s.node.setLastErr(err)
			// attempt to reconnect
			s.reconnectStream()
		} else {
			s.streamMut.RUnlock()
			err := status.FromProto(resp.Metadata.GetStatus()).Err()
			s.putResult(resp.Metadata.MessageID, &gorumsStreamResult{nid: s.node.ID(), reply: resp.Message, err: err})
		}

		select {
		case <-s.parentCtx.Done():
			return
		default:
		}
	}
}

func (s *orderedNodeStream) reconnectStream() {
	s.streamMut.Lock()
	defer s.streamMut.Unlock()

	var retries float64
	for {
		var err error

		s.streamCtx, s.cancelStream = context.WithCancel(s.parentCtx)
		s.gorumsStream, err = s.gorumsClient.NodeStream(s.streamCtx)
		if err == nil {
			s.streamBroken = false
			return
		}
		s.cancelStream()
		s.node.setLastErr(err)
		delay := float64(s.backoff.BaseDelay)
		max := float64(s.backoff.MaxDelay)
		for r := retries; delay < max && r > 0; r-- {
			delay *= s.backoff.Multiplier
		}
		delay = math.Min(delay, max)
		delay *= 1 + s.backoff.Jitter*(rand.Float64()*2-1)
		select {
		case <-time.After(time.Duration(delay)):
			retries++
		case <-s.parentCtx.Done():
			return
		}
	}
}
<commit_msg>Added newCall to receiveQueue<commit_after>package gorums

import (
	"context"
	"math"
	"math/rand"
	"sync"
	"sync/atomic"
	"time"

	"github.com/relab/gorums/ordering"
	"google.golang.org/grpc"
	"google.golang.org/grpc/backoff"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
	"google.golang.org/protobuf/reflect/protoreflect"
)

type gorumsStreamRequest struct {
	ctx  context.Context
	msg  *Message
	opts callOptions
}

type gorumsStreamResult struct {
	nid   uint32
	reply protoreflect.ProtoMessage
	err   error
}

type receiveQueue struct {
	msgID    uint64
	recvQ    map[uint64]chan *gorumsStreamResult
	recvQMut sync.RWMutex
}

func newReceiveQueue() *receiveQueue {
	return &receiveQueue{
		recvQ: make(map[uint64]chan *gorumsStreamResult),
	}
}

// newCall returns metadata for the call and a function to be called for clean up.
func (m *receiveQueue) newCall(method string, replyChan chan *gorumsStreamResult) (*ordering.Metadata, func()) {
	var msgID uint64
	m.recvQMut.Lock()
	m.msgID++
	msgID = m.msgID
	m.recvQ[msgID] = replyChan
	m.recvQMut.Unlock()
	md := &ordering.Metadata{
		MessageID: msgID,
		Method:    method,
	}
	return md, func() { m.deleteChan(msgID) }
}

func (m *receiveQueue) nextMsgID() uint64 {
	return atomic.AddUint64(&m.msgID, 1)
}

func (m *receiveQueue) putChan(id uint64, c chan *gorumsStreamResult) {
	m.recvQMut.Lock()
	m.recvQ[id] = c
	m.recvQMut.Unlock()
}

func (m *receiveQueue) deleteChan(id uint64) {
	m.recvQMut.Lock()
	delete(m.recvQ, id)
	m.recvQMut.Unlock()
}

func (m *receiveQueue) putResult(id uint64, result *gorumsStreamResult) {
	m.recvQMut.RLock()
	c, ok := m.recvQ[id]
	m.recvQMut.RUnlock()
	if ok {
		c <- result
	}
}

type orderedNodeStream struct {
	*receiveQueue
	sendQ        chan gorumsStreamRequest
	node         *Node // needed for ID and setLastError
	backoff      backoff.Config
	rand         *rand.Rand
	gorumsClient ordering.GorumsClient
	gorumsStream ordering.Gorums_NodeStreamClient
	streamMut    sync.RWMutex
	streamBroken bool
	parentCtx    context.Context
	streamCtx    context.Context
	cancelStream context.CancelFunc
}

func (s *orderedNodeStream) connectOrderedStream(ctx context.Context, conn *grpc.ClientConn) error {
	var err error
	s.parentCtx = ctx
	s.streamCtx, s.cancelStream = context.WithCancel(s.parentCtx)
	s.gorumsClient = ordering.NewGorumsClient(conn)
	s.gorumsStream, err = s.gorumsClient.NodeStream(s.streamCtx)
	if err != nil {
		return err
	}
	go s.sendMsgs()
	go s.recvMsgs()
	return nil
}

func (s *orderedNodeStream) sendMsg(req gorumsStreamRequest) (err error) {
	s.streamMut.RLock()
	defer s.streamMut.RUnlock()

	c := make(chan struct{})

	// wait for either the message to be sent, or the request context being cancelled.
	// if the request context was cancelled, then we most likely have a blocked stream.
	go func() {
		select {
		case <-c:
		case <-req.ctx.Done():
			s.cancelStream()
		}
	}()

	err = s.gorumsStream.SendMsg(req.msg)
	if err != nil {
		s.node.setLastErr(err)
		s.streamBroken = true
	}
	c <- struct{}{}

	// unblock the waiting caller when sendAsync is not enabled
	if req.opts.callType == E_Multicast || req.opts.callType == E_Unicast && !req.opts.sendAsync {
		s.putResult(req.msg.Metadata.MessageID, &gorumsStreamResult{})
	}

	return err
}

func (s *orderedNodeStream) sendMsgs() {
	var req gorumsStreamRequest
	for {
		select {
		case <-s.parentCtx.Done():
			return
		case req = <-s.sendQ:
		}
		// return error if stream is broken
		if s.streamBroken {
			err := status.Errorf(codes.Unavailable, "stream is down")
			s.putResult(req.msg.Metadata.MessageID, &gorumsStreamResult{nid: s.node.ID(), reply: nil, err: err})
			continue
		}
		// else try to send message
		err := s.sendMsg(req)
		if err != nil {
			// return the error
			s.putResult(req.msg.Metadata.MessageID, &gorumsStreamResult{nid: s.node.ID(), reply: nil, err: err})
		}
	}
}

func (s *orderedNodeStream) recvMsgs() {
	for {
		resp := newMessage(responseType)
		s.streamMut.RLock()
		err := s.gorumsStream.RecvMsg(resp)
		if err != nil {
			s.streamBroken = true
			s.streamMut.RUnlock()
			s.node.setLastErr(err)
			// attempt to reconnect
			s.reconnectStream()
		} else {
			s.streamMut.RUnlock()
			err := status.FromProto(resp.Metadata.GetStatus()).Err()
			s.putResult(resp.Metadata.MessageID, &gorumsStreamResult{nid: s.node.ID(), reply: resp.Message, err: err})
		}

		select {
		case <-s.parentCtx.Done():
			return
		default:
		}
	}
}

func (s *orderedNodeStream) reconnectStream() {
	s.streamMut.Lock()
	defer s.streamMut.Unlock()

	var retries float64
	for {
		var err error

		s.streamCtx, s.cancelStream = context.WithCancel(s.parentCtx)
		s.gorumsStream, err = s.gorumsClient.NodeStream(s.streamCtx)
		if err == nil {
			s.streamBroken = false
			return
		}
		s.cancelStream()
		s.node.setLastErr(err)
		delay := float64(s.backoff.BaseDelay)
		max := float64(s.backoff.MaxDelay)
		for r := retries; delay < max && r > 0; r-- {
			delay *= s.backoff.Multiplier
		}
		delay = math.Min(delay, max)
		delay *= 1 + s.backoff.Jitter*(rand.Float64()*2-1)
		select {
		case <-time.After(time.Duration(delay)):
			retries++
		case <-s.parentCtx.Done():
			return
		}
	}
}
<|endoftext|>


<commit_before>package render

import (
    "os"
    "fmt"
    "image"
    "errors"
    "unsafe"
    "reflect"
    "io/ioutil"
    _ "image/png"
    _ "image/jpeg"
    gl "github.com/GlenKelley/go-gl32"
)

func ArrayPtr(data interface{}) (gl.Pointer, gl.Sizeiptr) {
    var size gl.Sizeiptr
    var ptr gl.Pointer
    switch data := data.(type) {
    case []gl.Float:
        var v gl.Float
        size = gl.Sizeiptr(len(data) * int(unsafe.Sizeof(v)))
        ptr = gl.Pointer(&data[0])
    case []gl.Ushort:
        var v gl.Ushort
        size = gl.Sizeiptr(len(data) * int(unsafe.Sizeof(v)))
        ptr = gl.Pointer(&data[0])
    case []gl.Int:
        var v gl.Int
        size = gl.Sizeiptr(len(data) * int(unsafe.Sizeof(v)))
        ptr = gl.Pointer(&data[0])
    default:
        panic("unknown data type")
    }
    return ptr, size
}

func BindBufferData(buffer gl.Uint, target gl.Enum, data interface{}) {
    ptr, size := ArrayPtr(data)
    gl.BindBuffer(target, buffer)
    gl.BufferData(target, size, ptr, gl.STATIC_DRAW)
}

func ImageData(img image.Image) (gl.Sizei, gl.Sizei, gl.Enum, gl.Enum, gl.Pointer) {
    switch img := img.(type) {
    case *image.NRGBA:
        return gl.Sizei(img.Rect.Dx()), gl.Sizei(img.Rect.Dy()), gl.RGBA, gl.UNSIGNED_BYTE, gl.Pointer(&img.Pix[0])
    case *image.RGBA:
        return gl.Sizei(img.Rect.Dx()), gl.Sizei(img.Rect.Dy()), gl.RGBA, gl.UNSIGNED_BYTE, gl.Pointer(&img.Pix[0])
    default:
        panic(reflect.TypeOf(img))
    }
    return 0,0,gl.RGB,gl.UNSIGNED_BYTE,nil
}

//gl.Sizei(width), gl.Sizei(height)
//gl.RGB, gl.UNSIGNED_BYTE
func LoadTexture(filename string) (gl.Uint, error) {
    var texture gl.Uint
    file, err := os.Open(filename)
    if err != nil {
        return 0, err
    }
    defer file.Close()
    img, _, err := image.Decode(file)
    if err != nil {
        return 0, err
    }
    width, height, format, channelType, pixels := ImageData(img)
    gl.GenTextures(1, &texture)
    gl.BindTexture(gl.TEXTURE_2D, texture)
    gl.TexParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR)
    gl.TexParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR)
    gl.TexParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S,     gl.CLAMP_TO_EDGE)
    gl.TexParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T,     gl.CLAMP_TO_EDGE)
    gl.TexImage2D(
          gl.TEXTURE_2D, 0,           /* target, level of detail */
          gl.RGB8,                    /* internal format */
          width, height, 0,           /* width, height, border */
          format, channelType,          /* external format, type */
          pixels,                      /* pixels */
    )
    return texture, nil
}

func LoadShader(shaderType gl.Enum, filename string) (gl.Uint, error){
    bytes, err := ioutil.ReadFile(filename)
    if err != nil {
        return 0, err
    }
    shader := gl.CreateShader(shaderType)
    source := string(bytes)
    gl.ShaderSource(shader, []string{source})
    gl.CompileShader(shader)
    var ok gl.Int
    gl.GetShaderiv(shader, gl.COMPILE_STATUS, &ok)
    if (ok == 0) {
        fmt.Println(gl.GetShaderInfoLog(shader))
        gl.DeleteShader(shader)
        return 0, errors.New("Failed to compile " + filename + "\n")
    }
    return shader, nil
}

func CreateProgram(vertexShader, fragmentShader gl.Uint) (gl.Uint, error) {
    program := gl.CreateProgram()
    gl.AttachShader(program, vertexShader)
    gl.AttachShader(program, fragmentShader)
    gl.LinkProgram(program)
    var ok gl.Int
    gl.GetProgramiv(program, gl.LINK_STATUS, &ok)
     if (ok == 0) {
         fmt.Println(gl.GetProgramInfoLog(program))
         gl.DeleteProgram(program)
         return 0, errors.New("Failed to link shader program")
     }
     return program, nil
}
<commit_msg>print errors to stderr<commit_after>package render

import (
    "os"
    "fmt"
    "image"
    "errors"
    "unsafe"
    "reflect"
    "io/ioutil"
    _ "image/png"
    _ "image/jpeg"
    gl "github.com/GlenKelley/go-gl32"
)

func ArrayPtr(data interface{}) (gl.Pointer, gl.Sizeiptr) {
    var size gl.Sizeiptr
    var ptr gl.Pointer
    switch data := data.(type) {
    case []gl.Float:
        var v gl.Float
        size = gl.Sizeiptr(len(data) * int(unsafe.Sizeof(v)))
        ptr = gl.Pointer(&data[0])
    case []gl.Ushort:
        var v gl.Ushort
        size = gl.Sizeiptr(len(data) * int(unsafe.Sizeof(v)))
        ptr = gl.Pointer(&data[0])
    case []gl.Int:
        var v gl.Int
        size = gl.Sizeiptr(len(data) * int(unsafe.Sizeof(v)))
        ptr = gl.Pointer(&data[0])
    default:
        panic("unknown data type")
    }
    return ptr, size
}

func BindBufferData(buffer gl.Uint, target gl.Enum, data interface{}) {
    ptr, size := ArrayPtr(data)
    gl.BindBuffer(target, buffer)
    gl.BufferData(target, size, ptr, gl.STATIC_DRAW)
}

func ImageData(img image.Image) (gl.Sizei, gl.Sizei, gl.Enum, gl.Enum, gl.Pointer) {
    switch img := img.(type) {
    case *image.NRGBA:
        return gl.Sizei(img.Rect.Dx()), gl.Sizei(img.Rect.Dy()), gl.RGBA, gl.UNSIGNED_BYTE, gl.Pointer(&img.Pix[0])
    case *image.RGBA:
        return gl.Sizei(img.Rect.Dx()), gl.Sizei(img.Rect.Dy()), gl.RGBA, gl.UNSIGNED_BYTE, gl.Pointer(&img.Pix[0])
    default:
        panic(reflect.TypeOf(img))
    }
    return 0,0,gl.RGB,gl.UNSIGNED_BYTE,nil
}

func LoadTexture(filename string) (gl.Uint, error) {
    var texture gl.Uint
    file, err := os.Open(filename)
    if err != nil {
        return 0, err
    }
    defer file.Close()
    img, _, err := image.Decode(file)
    if err != nil {
        return 0, err
    }
    width, height, format, channelType, pixels := ImageData(img)
    gl.GenTextures(1, &texture)
    gl.BindTexture(gl.TEXTURE_2D, texture)
    gl.TexParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR)
    gl.TexParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR)
    gl.TexParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S,     gl.CLAMP_TO_EDGE)
    gl.TexParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T,     gl.CLAMP_TO_EDGE)
    gl.TexImage2D(
          gl.TEXTURE_2D, 0,           /* target, level of detail */
          gl.RGB8,                    /* internal format */
          width, height, 0,           /* width, height, border */
          format, channelType,          /* external format, type */
          pixels,                      /* pixels */
    )
    return texture, nil
}

func LoadShader(shaderType gl.Enum, filename string) (gl.Uint, error){
    bytes, err := ioutil.ReadFile(filename)
    if err != nil {
        return 0, err
    }
    shader := gl.CreateShader(shaderType)
    source := string(bytes)
    gl.ShaderSource(shader, []string{source})
    gl.CompileShader(shader)
    var ok gl.Int
    gl.GetShaderiv(shader, gl.COMPILE_STATUS, &ok)
    if (ok == 0) {
        fmt.Fprintln(os.Stderr, gl.GetShaderInfoLog(shader))
        gl.DeleteShader(shader)
        return 0, errors.New("Failed to compile " + filename + "\n")
    }
    return shader, nil
}

func CreateProgram(vertexShader, fragmentShader gl.Uint) (gl.Uint, error) {
    program := gl.CreateProgram()
    gl.AttachShader(program, vertexShader)
    gl.AttachShader(program, fragmentShader)
    gl.LinkProgram(program)
    var ok gl.Int
    gl.GetProgramiv(program, gl.LINK_STATUS, &ok)
     if (ok == 0) {
        fmt.Fprintln(os.Stderr, gl.GetProgramInfoLog(program))
         gl.DeleteProgram(program)
         return 0, errors.New("Failed to link shader program")
     }
     return program, nil
}
<|endoftext|>


<commit_before>// Package pastebin is a simple modern and powerful pastebin service
package main

import (
	"crypto/sha1"
	"database/sql"
	"encoding/base64"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"html"
	"html/template"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"time"

	// uniuri is used for easy random string generation
	"github.com/dchest/uniuri"
	// pygments is used for syntax highlighting
	"github.com/ewhal/pygments"
	// mysql driver
	_ "github.com/go-sql-driver/mysql"
	// mux is used for url routing
	"github.com/gorilla/mux"
)

const (
	// ADDRESS that pastebin will return links for
	ADDRESS = "http://localhost:9900"
	// LENGTH of paste id
	LENGTH = 6
	// PORT that pastebin will listen on
	PORT = ":9900"
	// USERNAME for database
	USERNAME = ""
	// PASS database password
	PASS = ""
	// NAME database name
	NAME = ""
	// DATABASE connection String
	DATABASE = USERNAME + ":" + PASS + "@/" + NAME + "?charset=utf8"
)

// Template pages
var templates = template.Must(template.ParseFiles("assets/paste.html", "assets/index.html", "assets/clone.html"))
var syntax, _ = ioutil.ReadFile("assets/syntax.html")

// Response API struct
type Response struct {
	ID     string `json:"id"`
	TITLE  string `json:"title"`
	HASH   string `json:"hash"`
	URL    string `json:"url"`
	SIZE   int    `json:"size"`
	DELKEY string `json:"delkey"`
}

// Page generation struct
type Page struct {
	Title    string
	Body     []byte
	Raw      string
	Home     string
	Download string
	Clone    string
}

// check error handling function
func Check(err error) {
	if err != nil {
		log.Println(err)
	}
}

// GenerateName uses uniuri to generate a random string that isn't in the
// database
func GenerateName() string {
	// use uniuri to generate random string
	id := uniuri.NewLen(LENGTH)

	db, err := sql.Open("mysql", DATABASE)
	Check(err)
	defer db.Close()
	// query database if id exists and if it does call generateName again
	query, err := db.Query("select id from pastebin where id=?", id)
	if err != sql.ErrNoRows {
		for query.Next() {
			GenerateName()
		}
	}

	return id

}

// Hash hashes paste into a sha1 hash
func Hash(paste string) string {
	hasher := sha1.New()

	hasher.Write([]byte(paste))
	sha := base64.URLEncoding.EncodeToString(hasher.Sum(nil))
	return sha
}

// DurationFromExpiry takes the expiry in string format and returns the duration
// that the paste will exist for
func DurationFromExpiry(expiry string) time.Duration {
	switch expiry {
	case "5 minutes":
		return time.Minute * 5
	case "1 hour":
		return time.Hour + 1 // XXX: did you mean '*'?
	case "1 day":
		return time.Hour * 24
	case "1 week":
		return time.Hour * 24 * 7
	case "1 month":
		return time.Hour * 24 * 30
	case "1 year":
		return time.Hour * 24 * 365
	case "forever":
		return time.Hour * 24 * (365 * 20)
	}
	return time.Hour * 24 * (365 * 20)
}

// Save function handles the saving of each paste.
// raw string is the raw paste input
// lang string is the user specified language for syntax highlighting
// title string user customized title
// expiry string duration that the paste will exist for
// Returns Response struct
func Save(raw string, lang string, title string, expiry string) Response {

	db, err := sql.Open("mysql", DATABASE)
	Check(err)
	defer db.Close()

	// hash paste data and query database to see if paste exists
	sha := Hash(raw)
	query, err := db.Query("select id, title, hash, data, delkey from pastebin where hash=?", sha)

	if err != sql.ErrNoRows {
		for query.Next() {
			var id, title, hash, paste, delkey string
			err := query.Scan(&id, &title, &hash, &paste, &delkey)
			Check(err)
			url := ADDRESS + "/p/" + id
			return Response{id, title, hash, url, len(paste), delkey}
		}
	}
	id := GenerateName()
	url := ADDRESS + "/p/" + id
	if lang != "" {
		url += "/" + lang
	}

	const timeFormat = "2006-01-02 15:04:05"
	expiryTime := time.Now().Add(DurationFromExpiry(expiry)).Format(timeFormat)

	delKey := uniuri.NewLen(40)
	dataEscaped := html.EscapeString(raw)

	stmt, err := db.Prepare("INSERT INTO pastebin(id, title, hash, data, delkey, expiry) values(?,?,?,?,?,?)")
	Check(err)
	if title == "" {
		title = id
	}
	_, err = stmt.Exec(id, html.EscapeString(title), sha, dataEscaped, delKey, expiryTime)
	Check(err)

	return Response{id, title, sha, url, len(dataEscaped), delKey}
}

// DelHandler checks to see if delkey and pasteid exist in the database.
// if both exist and are correct the paste will be removed.
func DelHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	id := vars["pasteId"]
	delkey := vars["delKey"]

	db, err := sql.Open("mysql", DATABASE)
	Check(err)
	defer db.Close()

	stmt, err := db.Prepare("delete from pastebin where delkey=? and id=?")
	Check(err)

	res, err := stmt.Exec(html.EscapeString(delkey), html.EscapeString(id))
	Check(err)

	_, err = res.RowsAffected()
	if err != sql.ErrNoRows {
		io.WriteString(w, id+" deleted")
	}
}

// SaveHandler Handles saving pastes and outputing responses
func SaveHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	output := vars["output"]
	switch r.Method {
	case "POST":
		paste := r.FormValue("p")
		lang := r.FormValue("lang")
		title := r.FormValue("title")
		expiry := r.FormValue("expiry")
		if paste == "" {
			http.Error(w, "Empty paste", 500)
			return
		}
		b := Save(paste, lang, title, expiry)

		switch output {
		case "json":
			w.Header().Set("Content-Type", "application/json")
			err := json.NewEncoder(w).Encode(b)
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}

		case "xml":
			x, err := xml.MarshalIndent(b, "", "  ")
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}

			w.Header().Set("Content-Type", "application/xml")
			w.Write(x)

		case "html":
			w.Header().Set("Content-Type", "text/html")
			io.WriteString(w, "
URL: "+b.URL+"
")
			io.WriteString(w, "
Delete Key: "+b.DELKEY+"
")

		case "redirect":
			http.Redirect(w, r, b.URL, 301)

		default:
			w.Header().Set("Content-Type", "text/plain; charset=UTF-8; imeanit=yes")
			io.WriteString(w, b.URL+"\n")
			io.WriteString(w, "delete key: "+b.DELKEY+"\n")
		}
	}

}

// Highlight uses user specified input to call pygments library to highlight the
// paste
func Highlight(s string, lang string) (string, error) {

	highlight, err := pygments.Highlight(html.UnescapeString(s), html.EscapeString(lang), "html", "style=autumn,linenos=True, lineanchors=True,anchorlinenos=True,noclasses=True,", "utf-8")
	if err != nil {
		return "", err
	}
	return highlight, nil

}

// GetPaste takes pasteid and language
// queries the database and returns paste data
func GetPaste(paste string, lang string) (string, string) {
	param1 := html.EscapeString(paste)
	db, err := sql.Open("mysql", DATABASE)
	Check(err)
	defer db.Close()
	var title, s string
	var expiry string
	err = db.QueryRow("select title, data, expiry from pastebin where id=?", param1).Scan(&title, &s, &expiry)
	Check(err)
	if time.Now().Format("2006-01-02 15:04:05") > expiry {
		stmt, err := db.Prepare("delete from pastebin where id=?")
		Check(err)
		_, err = stmt.Exec(param1)
		Check(err)
		return "Error invalid paste", ""
	}

	if err == sql.ErrNoRows {
		return "Error invalid paste", ""
	}
	if lang != "" {
		high, err := Highlight(s, lang)
		Check(err)
		return high, html.UnescapeString(title)
	}
	return html.UnescapeString(s), html.UnescapeString(title)
}

// PasteHandler handles the generation of paste pages with the links
func PasteHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	paste := vars["pasteId"]
	lang := vars["lang"]

	s, title := GetPaste(paste, lang)

	// button links
	link := ADDRESS + "/raw/" + paste
	download := ADDRESS + "/download/" + paste
	clone := ADDRESS + "/clone/" + paste
	// Page struct
	p := &Page{
		Title:    title,
		Body:     []byte(s),
		Raw:      link,
		Home:     ADDRESS,
		Download: download,
		Clone:    clone,
	}
	if lang == "" {

		err := templates.ExecuteTemplate(w, "paste.html", p)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}

	} else {
		fmt.Fprintf(w, string(syntax), p.Title, p.Title, s, p.Home, p.Download, p.Raw, p.Clone)

	}
}

// CloneHandler handles generating the clone pages
func CloneHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	paste := vars["pasteId"]

	s, title := GetPaste(paste, "")

	// Page links
	link := ADDRESS + "/raw/" + paste
	download := ADDRESS + "/download/" + paste
	clone := ADDRESS + "/clone/" + paste

	// Clone page struct
	p := &Page{
		Title:    title,
		Body:     []byte(s),
		Raw:      link,
		Home:     ADDRESS,
		Download: download,
		Clone:    clone,
	}
	err := templates.ExecuteTemplate(w, "clone.html", p)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}

}

// DownloadHandler forces downloads of selected pastes
func DownloadHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	paste := vars["pasteId"]
	s, _ := GetPaste(paste, "")

	// Set header to an attachment so browser will automatically download it
	w.Header().Set("Content-Disposition", "attachment; filename="+paste)
	w.Header().Set("Content-Type", r.Header.Get("Content-Type"))
	io.WriteString(w, s)

}

// RawHandler displays the pastes in text/plain format
func RawHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	paste := vars["pasteId"]
	s, _ := GetPaste(paste, "")

	w.Header().Set("Content-Type", "text/plain; charset=UTF-8; imeanit=yes")
	// simply write string to browser
	io.WriteString(w, s)

}

// RootHandler handles generating the root page
func RootHandler(w http.ResponseWriter, r *http.Request) {
	err := templates.ExecuteTemplate(w, "index.html", &Page{})
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func main() {
	router := mux.NewRouter()
	router.HandleFunc("/p/{pasteId}", PasteHandler).Methods("GET")
	router.HandleFunc("/raw/{pasteId}", RawHandler).Methods("GET")
	router.HandleFunc("/p/{pasteId}/{lang}", PasteHandler).Methods("GET")
	router.HandleFunc("/clone/{pasteId}", CloneHandler).Methods("GET")
	router.HandleFunc("/download/{pasteId}", DownloadHandler).Methods("GET")
	router.HandleFunc("/p", SaveHandler).Methods("POST")
	router.HandleFunc("/p/{output}", SaveHandler).Methods("POST")
	router.HandleFunc("/p/{pasteId}/{delKey}", DelHandler).Methods("DELETE")
	router.HandleFunc("/", RootHandler)
	err := http.ListenAndServe(PORT, router)
	if err != nil {
		log.Fatal(err)
	}

}
<commit_msg>fix up godoc package name error<commit_after>// Package pastebin is a simple modern and powerful pastebin service
package pastebin

import (
	"crypto/sha1"
	"database/sql"
	"encoding/base64"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"html"
	"html/template"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"time"

	// uniuri is used for easy random string generation
	"github.com/dchest/uniuri"
	// pygments is used for syntax highlighting
	"github.com/ewhal/pygments"
	// mysql driver
	_ "github.com/go-sql-driver/mysql"
	// mux is used for url routing
	"github.com/gorilla/mux"
)

const (
	// ADDRESS that pastebin will return links for
	ADDRESS = "http://localhost:9900"
	// LENGTH of paste id
	LENGTH = 6
	// PORT that pastebin will listen on
	PORT = ":9900"
	// USERNAME for database
	USERNAME = ""
	// PASS database password
	PASS = ""
	// NAME database name
	NAME = ""
	// DATABASE connection String
	DATABASE = USERNAME + ":" + PASS + "@/" + NAME + "?charset=utf8"
)

// Template pages
var templates = template.Must(template.ParseFiles("assets/paste.html", "assets/index.html", "assets/clone.html"))
var syntax, _ = ioutil.ReadFile("assets/syntax.html")

// Response API struct
type Response struct {
	ID     string `json:"id"`
	TITLE  string `json:"title"`
	HASH   string `json:"hash"`
	URL    string `json:"url"`
	SIZE   int    `json:"size"`
	DELKEY string `json:"delkey"`
}

// Page generation struct
type Page struct {
	Title    string
	Body     []byte
	Raw      string
	Home     string
	Download string
	Clone    string
}

// check error handling function
func Check(err error) {
	if err != nil {
		log.Println(err)
	}
}

// GenerateName uses uniuri to generate a random string that isn't in the
// database
func GenerateName() string {
	// use uniuri to generate random string
	id := uniuri.NewLen(LENGTH)

	db, err := sql.Open("mysql", DATABASE)
	Check(err)
	defer db.Close()
	// query database if id exists and if it does call generateName again
	query, err := db.Query("select id from pastebin where id=?", id)
	if err != sql.ErrNoRows {
		for query.Next() {
			GenerateName()
		}
	}

	return id

}

// Hash hashes paste into a sha1 hash
func Hash(paste string) string {
	hasher := sha1.New()

	hasher.Write([]byte(paste))
	sha := base64.URLEncoding.EncodeToString(hasher.Sum(nil))
	return sha
}

// DurationFromExpiry takes the expiry in string format and returns the duration
// that the paste will exist for
func DurationFromExpiry(expiry string) time.Duration {
	switch expiry {
	case "5 minutes":
		return time.Minute * 5
	case "1 hour":
		return time.Hour + 1 // XXX: did you mean '*'?
	case "1 day":
		return time.Hour * 24
	case "1 week":
		return time.Hour * 24 * 7
	case "1 month":
		return time.Hour * 24 * 30
	case "1 year":
		return time.Hour * 24 * 365
	case "forever":
		return time.Hour * 24 * (365 * 20)
	}
	return time.Hour * 24 * (365 * 20)
}

// Save function handles the saving of each paste.
// raw string is the raw paste input
// lang string is the user specified language for syntax highlighting
// title string user customized title
// expiry string duration that the paste will exist for
// Returns Response struct
func Save(raw string, lang string, title string, expiry string) Response {

	db, err := sql.Open("mysql", DATABASE)
	Check(err)
	defer db.Close()

	// hash paste data and query database to see if paste exists
	sha := Hash(raw)
	query, err := db.Query("select id, title, hash, data, delkey from pastebin where hash=?", sha)

	if err != sql.ErrNoRows {
		for query.Next() {
			var id, title, hash, paste, delkey string
			err := query.Scan(&id, &title, &hash, &paste, &delkey)
			Check(err)
			url := ADDRESS + "/p/" + id
			return Response{id, title, hash, url, len(paste), delkey}
		}
	}
	id := GenerateName()
	url := ADDRESS + "/p/" + id
	if lang != "" {
		url += "/" + lang
	}

	const timeFormat = "2006-01-02 15:04:05"
	expiryTime := time.Now().Add(DurationFromExpiry(expiry)).Format(timeFormat)

	delKey := uniuri.NewLen(40)
	dataEscaped := html.EscapeString(raw)

	stmt, err := db.Prepare("INSERT INTO pastebin(id, title, hash, data, delkey, expiry) values(?,?,?,?,?,?)")
	Check(err)
	if title == "" {
		title = id
	}
	_, err = stmt.Exec(id, html.EscapeString(title), sha, dataEscaped, delKey, expiryTime)
	Check(err)

	return Response{id, title, sha, url, len(dataEscaped), delKey}
}

// DelHandler checks to see if delkey and pasteid exist in the database.
// if both exist and are correct the paste will be removed.
func DelHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	id := vars["pasteId"]
	delkey := vars["delKey"]

	db, err := sql.Open("mysql", DATABASE)
	Check(err)
	defer db.Close()

	stmt, err := db.Prepare("delete from pastebin where delkey=? and id=?")
	Check(err)

	res, err := stmt.Exec(html.EscapeString(delkey), html.EscapeString(id))
	Check(err)

	_, err = res.RowsAffected()
	if err != sql.ErrNoRows {
		io.WriteString(w, id+" deleted")
	}
}

// SaveHandler Handles saving pastes and outputing responses
func SaveHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	output := vars["output"]
	switch r.Method {
	case "POST":
		paste := r.FormValue("p")
		lang := r.FormValue("lang")
		title := r.FormValue("title")
		expiry := r.FormValue("expiry")
		if paste == "" {
			http.Error(w, "Empty paste", 500)
			return
		}
		b := Save(paste, lang, title, expiry)

		switch output {
		case "json":
			w.Header().Set("Content-Type", "application/json")
			err := json.NewEncoder(w).Encode(b)
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}

		case "xml":
			x, err := xml.MarshalIndent(b, "", "  ")
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}

			w.Header().Set("Content-Type", "application/xml")
			w.Write(x)

		case "html":
			w.Header().Set("Content-Type", "text/html")
			io.WriteString(w, "
URL: "+b.URL+"
")
			io.WriteString(w, "
Delete Key: "+b.DELKEY+"
")

		case "redirect":
			http.Redirect(w, r, b.URL, 301)

		default:
			w.Header().Set("Content-Type", "text/plain; charset=UTF-8; imeanit=yes")
			io.WriteString(w, b.URL+"\n")
			io.WriteString(w, "delete key: "+b.DELKEY+"\n")
		}
	}

}

// Highlight uses user specified input to call pygments library to highlight the
// paste
func Highlight(s string, lang string) (string, error) {

	highlight, err := pygments.Highlight(html.UnescapeString(s), html.EscapeString(lang), "html", "style=autumn,linenos=True, lineanchors=True,anchorlinenos=True,noclasses=True,", "utf-8")
	if err != nil {
		return "", err
	}
	return highlight, nil

}

// GetPaste takes pasteid and language
// queries the database and returns paste data
func GetPaste(paste string, lang string) (string, string) {
	param1 := html.EscapeString(paste)
	db, err := sql.Open("mysql", DATABASE)
	Check(err)
	defer db.Close()
	var title, s string
	var expiry string
	err = db.QueryRow("select title, data, expiry from pastebin where id=?", param1).Scan(&title, &s, &expiry)
	Check(err)
	if time.Now().Format("2006-01-02 15:04:05") > expiry {
		stmt, err := db.Prepare("delete from pastebin where id=?")
		Check(err)
		_, err = stmt.Exec(param1)
		Check(err)
		return "Error invalid paste", ""
	}

	if err == sql.ErrNoRows {
		return "Error invalid paste", ""
	}
	if lang != "" {
		high, err := Highlight(s, lang)
		Check(err)
		return high, html.UnescapeString(title)
	}
	return html.UnescapeString(s), html.UnescapeString(title)
}

// PasteHandler handles the generation of paste pages with the links
func PasteHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	paste := vars["pasteId"]
	lang := vars["lang"]

	s, title := GetPaste(paste, lang)

	// button links
	link := ADDRESS + "/raw/" + paste
	download := ADDRESS + "/download/" + paste
	clone := ADDRESS + "/clone/" + paste
	// Page struct
	p := &Page{
		Title:    title,
		Body:     []byte(s),
		Raw:      link,
		Home:     ADDRESS,
		Download: download,
		Clone:    clone,
	}
	if lang == "" {

		err := templates.ExecuteTemplate(w, "paste.html", p)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}

	} else {
		fmt.Fprintf(w, string(syntax), p.Title, p.Title, s, p.Home, p.Download, p.Raw, p.Clone)

	}
}

// CloneHandler handles generating the clone pages
func CloneHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	paste := vars["pasteId"]

	s, title := GetPaste(paste, "")

	// Page links
	link := ADDRESS + "/raw/" + paste
	download := ADDRESS + "/download/" + paste
	clone := ADDRESS + "/clone/" + paste

	// Clone page struct
	p := &Page{
		Title:    title,
		Body:     []byte(s),
		Raw:      link,
		Home:     ADDRESS,
		Download: download,
		Clone:    clone,
	}
	err := templates.ExecuteTemplate(w, "clone.html", p)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}

}

// DownloadHandler forces downloads of selected pastes
func DownloadHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	paste := vars["pasteId"]
	s, _ := GetPaste(paste, "")

	// Set header to an attachment so browser will automatically download it
	w.Header().Set("Content-Disposition", "attachment; filename="+paste)
	w.Header().Set("Content-Type", r.Header.Get("Content-Type"))
	io.WriteString(w, s)

}

// RawHandler displays the pastes in text/plain format
func RawHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	paste := vars["pasteId"]
	s, _ := GetPaste(paste, "")

	w.Header().Set("Content-Type", "text/plain; charset=UTF-8; imeanit=yes")
	// simply write string to browser
	io.WriteString(w, s)

}

// RootHandler handles generating the root page
func RootHandler(w http.ResponseWriter, r *http.Request) {
	err := templates.ExecuteTemplate(w, "index.html", &Page{})
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func main() {
	router := mux.NewRouter()
	router.HandleFunc("/p/{pasteId}", PasteHandler).Methods("GET")
	router.HandleFunc("/raw/{pasteId}", RawHandler).Methods("GET")
	router.HandleFunc("/p/{pasteId}/{lang}", PasteHandler).Methods("GET")
	router.HandleFunc("/clone/{pasteId}", CloneHandler).Methods("GET")
	router.HandleFunc("/download/{pasteId}", DownloadHandler).Methods("GET")
	router.HandleFunc("/p", SaveHandler).Methods("POST")
	router.HandleFunc("/p/{output}", SaveHandler).Methods("POST")
	router.HandleFunc("/p/{pasteId}/{delKey}", DelHandler).Methods("DELETE")
	router.HandleFunc("/", RootHandler)
	err := http.ListenAndServe(PORT, router)
	if err != nil {
		log.Fatal(err)
	}

}
<|endoftext|>


<commit_before>/* Copyright (c) 2014, Daniel Martí <mvdan@mvdan.cc> */
/* See LICENSE for licensing information */

package main

import (
	"encoding/hex"
	"errors"
	"flag"
	"fmt"
	"html/template"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"regexp"
	"strconv"
	"strings"
	"time"
)

const (
	// Length of the random hexadecimal ids assigned to pastes. At least 4.
	idSize = 8
	// Number of times to try getting a random paste id
	randTries = 10
	// Name of the HTTP form field when uploading a paste
	fieldName = "paste"
	// Content-Type when serving pastes
	contentType = "text/plain; charset=utf-8"
	// Report usage stats how often
	statsReport = 1 * time.Minute
	// How long to wait before retrying to delete a file
	deleteRetry = 2 * time.Minute

	// GET error messages
	invalidID = "invalid paste id"

	// Common error messages
	unknownAction = "unsupported action"
)

var (
	siteURL, listen           string
	lifeTime                  time.Duration
	maxNumber                 int
	maxSizeStr, maxStorageStr string
	maxSize, maxStorage       ByteSize
	templates                 *template.Template

	regexByteSize = regexp.MustCompile(`^([\d\.]+)\s*([KMGT]?B?)$`)
	startTime     = time.Now()

	store Store
)

func init() {
	flag.StringVar(&siteURL, "u", "http://localhost:8080", "URL of the site")
	flag.StringVar(&listen, "l", ":8080", "Host and port to listen to")
	flag.DurationVar(&lifeTime, "t", 24*time.Hour, "Lifetime of the pastes")
	flag.StringVar(&maxSizeStr, "s", "1M", "Maximum size of pastes")
	flag.IntVar(&maxNumber, "m", 0, "Maximum number of pastes to store at once")
	flag.StringVar(&maxStorageStr, "M", "1G", "Maximum storage size to use at once")
}

type ByteSize int64

const (
	_ ByteSize = 1 << (10 * iota)
	kbyte
	mbyte
	gbyte
	tbyte
)

func parseByteSize(str string) (ByteSize, error) {
	if !regexByteSize.MatchString(str) {
		return 0, errors.New("Could not parse size in bytes")
	}
	parts := regexByteSize.FindStringSubmatch(str)
	size, _ := strconv.ParseFloat(string(parts[1]), 64)
	switch string(parts[2]) {
	case "KB", "K":
		size *= float64(kbyte)
	case "MB", "M":
		size *= float64(mbyte)
	case "GB", "G":
		size *= float64(gbyte)
	case "TB", "T":
		size *= float64(tbyte)
	}
	return ByteSize(size), nil
}

func (b ByteSize) String() string {
	switch {
	case b >= tbyte:
		return fmt.Sprintf("%.2fGB", float64(b)/float64(tbyte))
	case b >= gbyte:
		return fmt.Sprintf("%.2fGB", float64(b)/float64(gbyte))
	case b >= mbyte:
		return fmt.Sprintf("%.2fMB", float64(b)/float64(mbyte))
	case b >= kbyte:
		return fmt.Sprintf("%.2fKB", float64(b)/float64(kbyte))
	}
	return fmt.Sprintf("%dB", b)
}

type ID [idSize / 2]byte

func IDFromString(hexID string) (id ID, err error) {
	if len(hexID) != idSize {
		return id, errors.New("Invalid id at " + hexID)
	}
	b, err := hex.DecodeString(hexID)
	if err != nil || len(b) != idSize/2 {
		return id, errors.New("Invalid id at " + hexID)
	}
	copy(id[:], b)
	return id, nil
}

func (id ID) String() string {
	return hex.EncodeToString(id[:])
}

type Header struct {
	Etag, Expires string
	ModTime       time.Time
	Size          ByteSize
}

type Content interface {
	io.Reader
	io.ReaderAt
	io.Seeker
	io.Closer
}

func describeLimits() string {
	var limits []string
	if maxSize > 0 {
		limits = append(limits, fmt.Sprintf("Maximum size per paste is %s.", maxSize))
	}
	if lifeTime > 0 {
		limits = append(limits, fmt.Sprintf("Pastes will be deleted after %s.", lifeTime))
	}
	if len(limits) > 0 {
		return strings.Join(limits, " ") + "\n\n"
	}
	return ""
}

func getContentFromForm(r *http.Request) (content []byte, err error) {
	if value := r.FormValue(fieldName); value != "" {
		return []byte(value), nil
	}
	if f, _, err := r.FormFile(fieldName); err == nil {
		defer f.Close()
		content, err = ioutil.ReadAll(f)
		if err == nil {
			return content, nil
		}
	}
	return content, err
}

func setupPasteDeletion(id ID) {
	if lifeTime == 0 {
		return
	}
	timer := time.NewTimer(lifeTime)
	go func() {
		for {
			<-timer.C
			err := store.Delete(id)
			if err == nil {
				break
			}
			timer.Reset(deleteRetry)
		}
	}()
}

func handler(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case "GET":
		switch r.URL.Path {
		case "/":
			templates.ExecuteTemplate(w, "index.html",
				struct{ SiteURL, LimitDesc, FieldName string }{
					siteURL, describeLimits(), fieldName})
			return
		case "/form":
			templates.ExecuteTemplate(w, "form.html",
				struct{ SiteURL, LimitDesc, FieldName string }{
					siteURL, describeLimits(), fieldName})
			return
		}
		id, err := IDFromString(r.URL.Path[1:])
		if err != nil {
			http.Error(w, invalidID, http.StatusBadRequest)
			return
		}
		content, header, err := store.Get(id)
		if err == ErrPasteNotFound {
			http.Error(w, err.Error(), http.StatusNotFound)
			return
		} else if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		defer content.Close()
		w.Header().Set("Etag", header.Etag)
		if lifeTime > 0 {
			w.Header().Set("Expires", header.Expires)
		}
		w.Header().Set("Content-Type", contentType)
		http.ServeContent(w, r, "", header.ModTime, content)

	case "POST":
		r.Body = http.MaxBytesReader(w, r.Body, int64(maxSize))
		content, err := getContentFromForm(r)
		if err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
		}
		id, err := store.Put(content)
		if err == ErrReachedMax {
			http.Error(w, err.Error(), http.StatusServiceUnavailable)
			return
		} else if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		setupPasteDeletion(id)
		fmt.Fprintf(w, "%s/%s\n", siteURL, id)

	default:
		http.Error(w, unknownAction, http.StatusBadRequest)
		return
	}
}

func main() {
	var err error
	flag.Parse()
	if maxSize, err = parseByteSize(maxSizeStr); err != nil {
		log.Fatalf("Invalid max size '%s': %s", maxSizeStr, err)
	}
	if maxStorage, err = parseByteSize(maxStorageStr); err != nil {
		log.Fatalf("Invalid max storage '%s': %s", maxStorageStr, err)
	}
	templates = template.Must(template.ParseFiles("index.html", "form.html"))

	log.Printf("siteURL    = %s", siteURL)
	log.Printf("listen     = %s", listen)
	log.Printf("lifeTime   = %s", lifeTime)
	log.Printf("maxSize    = %s", maxSize)
	log.Printf("maxNumber  = %d", maxNumber)
	log.Printf("maxStorage = %s", maxStorage)

	args := flag.Args()
	storageType := "fs"
	if len(args) > 0 {
		storageType = args[0]
		args = args[1:]
	}
	switch storageType {
	case "fs":
		if len(args) > 1 {
			log.Fatalf("Too many arguments given for %s", storageType)
		}
		pasteDir := "pastes"
		if len(args) > 0 {
			pasteDir = args[0]
		}
		log.Printf("Starting up file store in the directory '%s'", pasteDir)
		store, err = newFileStore(pasteDir, maxNumber, maxStorage, lifeTime)
	case "mem":
		if len(args) > 0 {
			log.Fatalf("Too many arguments given for %s", storageType)
		}
		log.Printf("Starting up in-memory store")
		store, err = newMemStore(maxNumber, maxStorage, lifeTime)
	default:
		log.Fatalf("Unknown paste store type '%s'", storageType)
	}
	if err != nil {
		log.Fatalf("Could not start %s paste store: %s", storageType, err)
	}

	log.Println(store.Report())
	ticker := time.NewTicker(statsReport)
	go func() {
		for _ = range ticker.C {
			log.Println(store.Report())
		}
	}()
	http.HandleFunc("/", handler)
	log.Println("Up and running!")
	log.Fatal(http.ListenAndServe(listen, nil))
}
<commit_msg>Report unknown runtime errors in the logs<commit_after>/* Copyright (c) 2014, Daniel Martí <mvdan@mvdan.cc> */
/* See LICENSE for licensing information */

package main

import (
	"encoding/hex"
	"errors"
	"flag"
	"fmt"
	"html/template"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"regexp"
	"strconv"
	"strings"
	"time"
)

const (
	// Length of the random hexadecimal ids assigned to pastes. At least 4.
	idSize = 8
	// Number of times to try getting a random paste id
	randTries = 10
	// Name of the HTTP form field when uploading a paste
	fieldName = "paste"
	// Content-Type when serving pastes
	contentType = "text/plain; charset=utf-8"
	// Report usage stats how often
	statsReport = 1 * time.Minute
	// How long to wait before retrying to delete a file
	deleteRetry = 2 * time.Minute

	// GET error messages
	invalidID = "invalid paste id"

	// Common error messages
	unknownAction = "unsupported action"
)

var (
	siteURL, listen           string
	lifeTime                  time.Duration
	maxNumber                 int
	maxSizeStr, maxStorageStr string
	maxSize, maxStorage       ByteSize
	templates                 *template.Template

	regexByteSize = regexp.MustCompile(`^([\d\.]+)\s*([KMGT]?B?)$`)
	startTime     = time.Now()

	store Store
)

func init() {
	flag.StringVar(&siteURL, "u", "http://localhost:8080", "URL of the site")
	flag.StringVar(&listen, "l", ":8080", "Host and port to listen to")
	flag.DurationVar(&lifeTime, "t", 24*time.Hour, "Lifetime of the pastes")
	flag.StringVar(&maxSizeStr, "s", "1M", "Maximum size of pastes")
	flag.IntVar(&maxNumber, "m", 0, "Maximum number of pastes to store at once")
	flag.StringVar(&maxStorageStr, "M", "1G", "Maximum storage size to use at once")
}

type ByteSize int64

const (
	_ ByteSize = 1 << (10 * iota)
	kbyte
	mbyte
	gbyte
	tbyte
)

func parseByteSize(str string) (ByteSize, error) {
	if !regexByteSize.MatchString(str) {
		return 0, errors.New("Could not parse size in bytes")
	}
	parts := regexByteSize.FindStringSubmatch(str)
	size, _ := strconv.ParseFloat(string(parts[1]), 64)
	switch string(parts[2]) {
	case "KB", "K":
		size *= float64(kbyte)
	case "MB", "M":
		size *= float64(mbyte)
	case "GB", "G":
		size *= float64(gbyte)
	case "TB", "T":
		size *= float64(tbyte)
	}
	return ByteSize(size), nil
}

func (b ByteSize) String() string {
	switch {
	case b >= tbyte:
		return fmt.Sprintf("%.2fGB", float64(b)/float64(tbyte))
	case b >= gbyte:
		return fmt.Sprintf("%.2fGB", float64(b)/float64(gbyte))
	case b >= mbyte:
		return fmt.Sprintf("%.2fMB", float64(b)/float64(mbyte))
	case b >= kbyte:
		return fmt.Sprintf("%.2fKB", float64(b)/float64(kbyte))
	}
	return fmt.Sprintf("%dB", b)
}

type ID [idSize / 2]byte

func IDFromString(hexID string) (id ID, err error) {
	if len(hexID) != idSize {
		return id, errors.New("Invalid id at " + hexID)
	}
	b, err := hex.DecodeString(hexID)
	if err != nil || len(b) != idSize/2 {
		return id, errors.New("Invalid id at " + hexID)
	}
	copy(id[:], b)
	return id, nil
}

func (id ID) String() string {
	return hex.EncodeToString(id[:])
}

type Header struct {
	Etag, Expires string
	ModTime       time.Time
	Size          ByteSize
}

type Content interface {
	io.Reader
	io.ReaderAt
	io.Seeker
	io.Closer
}

func describeLimits() string {
	var limits []string
	if maxSize > 0 {
		limits = append(limits, fmt.Sprintf("Maximum size per paste is %s.", maxSize))
	}
	if lifeTime > 0 {
		limits = append(limits, fmt.Sprintf("Pastes will be deleted after %s.", lifeTime))
	}
	if len(limits) > 0 {
		return strings.Join(limits, " ") + "\n\n"
	}
	return ""
}

func getContentFromForm(r *http.Request) (content []byte, err error) {
	if value := r.FormValue(fieldName); value != "" {
		return []byte(value), nil
	}
	if f, _, err := r.FormFile(fieldName); err == nil {
		defer f.Close()
		content, err = ioutil.ReadAll(f)
		if err == nil {
			return content, nil
		}
	}
	return content, err
}

func setupPasteDeletion(id ID) {
	if lifeTime == 0 {
		return
	}
	timer := time.NewTimer(lifeTime)
	go func() {
		for {
			<-timer.C
			err := store.Delete(id)
			if err == nil {
				break
			}
			timer.Reset(deleteRetry)
		}
	}()
}

func handler(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case "GET":
		switch r.URL.Path {
		case "/":
			templates.ExecuteTemplate(w, "index.html",
				struct{ SiteURL, LimitDesc, FieldName string }{
					siteURL, describeLimits(), fieldName})
			return
		case "/form":
			templates.ExecuteTemplate(w, "form.html",
				struct{ SiteURL, LimitDesc, FieldName string }{
					siteURL, describeLimits(), fieldName})
			return
		}
		id, err := IDFromString(r.URL.Path[1:])
		if err != nil {
			http.Error(w, invalidID, http.StatusBadRequest)
			return
		}
		content, header, err := store.Get(id)
		if err == ErrPasteNotFound {
			http.Error(w, err.Error(), http.StatusNotFound)
			return
		} else if err != nil {
			log.Printf("Unknown store.Get() error: %s", err)
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		defer content.Close()
		w.Header().Set("Etag", header.Etag)
		if lifeTime > 0 {
			w.Header().Set("Expires", header.Expires)
		}
		w.Header().Set("Content-Type", contentType)
		http.ServeContent(w, r, "", header.ModTime, content)

	case "POST":
		r.Body = http.MaxBytesReader(w, r.Body, int64(maxSize))
		content, err := getContentFromForm(r)
		if err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
		}
		id, err := store.Put(content)
		if err == ErrReachedMax {
			http.Error(w, err.Error(), http.StatusServiceUnavailable)
			return
		} else if err != nil {
			log.Printf("Unknown store.Put() error: %s", err)
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		setupPasteDeletion(id)
		fmt.Fprintf(w, "%s/%s\n", siteURL, id)

	default:
		http.Error(w, unknownAction, http.StatusBadRequest)
		return
	}
}

func main() {
	var err error
	flag.Parse()
	if maxSize, err = parseByteSize(maxSizeStr); err != nil {
		log.Fatalf("Invalid max size '%s': %s", maxSizeStr, err)
	}
	if maxStorage, err = parseByteSize(maxStorageStr); err != nil {
		log.Fatalf("Invalid max storage '%s': %s", maxStorageStr, err)
	}
	templates = template.Must(template.ParseFiles("index.html", "form.html"))

	log.Printf("siteURL    = %s", siteURL)
	log.Printf("listen     = %s", listen)
	log.Printf("lifeTime   = %s", lifeTime)
	log.Printf("maxSize    = %s", maxSize)
	log.Printf("maxNumber  = %d", maxNumber)
	log.Printf("maxStorage = %s", maxStorage)

	args := flag.Args()
	storageType := "fs"
	if len(args) > 0 {
		storageType = args[0]
		args = args[1:]
	}
	switch storageType {
	case "fs":
		if len(args) > 1 {
			log.Fatalf("Too many arguments given for %s", storageType)
		}
		pasteDir := "pastes"
		if len(args) > 0 {
			pasteDir = args[0]
		}
		log.Printf("Starting up file store in the directory '%s'", pasteDir)
		store, err = newFileStore(pasteDir, maxNumber, maxStorage, lifeTime)
	case "mem":
		if len(args) > 0 {
			log.Fatalf("Too many arguments given for %s", storageType)
		}
		log.Printf("Starting up in-memory store")
		store, err = newMemStore(maxNumber, maxStorage, lifeTime)
	default:
		log.Fatalf("Unknown paste store type '%s'", storageType)
	}
	if err != nil {
		log.Fatalf("Could not start %s paste store: %s", storageType, err)
	}

	log.Println(store.Report())
	ticker := time.NewTicker(statsReport)
	go func() {
		for _ = range ticker.C {
			log.Println(store.Report())
		}
	}()
	http.HandleFunc("/", handler)
	log.Println("Up and running!")
	log.Fatal(http.ListenAndServe(listen, nil))
}
<|endoftext|>