<commit_before>package mountlib

import (
	"io"
	"log"
	"os"
	"runtime"
	"strings"
	"time"

	"github.com/ncw/rclone/cmd"
	"github.com/ncw/rclone/fs"
	"github.com/ncw/rclone/fs/config"
	"github.com/ncw/rclone/fs/config/flags"
	"github.com/ncw/rclone/vfs"
	"github.com/ncw/rclone/vfs/vfsflags"
	"github.com/pkg/errors"
	"github.com/spf13/cobra"
)

// Options set by command line flags
var (
	DebugFUSE                        = false
	AllowNonEmpty                    = false
	AllowRoot                        = false
	AllowOther                       = false
	DefaultPermissions               = false
	WritebackCache                   = false
	Daemon                           = false
	MaxReadAhead       fs.SizeSuffix = 128 * 1024
	ExtraOptions       []string
	ExtraFlags         []string
	AttrTimeout        = 1 * time.Second // how long the kernel caches attribute for
	VolumeName         string
	NoAppleDouble      = true        // use noappledouble by default
	NoAppleXattr       = false       // do not use noapplexattr by default
	DaemonTimeout      time.Duration // OSXFUSE only
)

// Check is folder is empty
func checkMountEmpty(mountpoint string) error {
	fp, fpErr := os.Open(mountpoint)

	if fpErr != nil {
		return errors.Wrap(fpErr, "Can not open: "+mountpoint)
	}
	defer fs.CheckClose(fp, &fpErr)

	_, fpErr = fp.Readdirnames(1)

	// directory is not empty
	if fpErr != io.EOF {
		var e error
		var errorMsg = "Directory is not empty: " + mountpoint + " If you want to mount it anyway use: --allow-non-empty option"
		if fpErr == nil {
			e = errors.New(errorMsg)
		} else {
			e = errors.Wrap(fpErr, errorMsg)
		}
		return e
	}
	return nil
}

// NewMountCommand makes a mount command with the given name and Mount function
func NewMountCommand(commandName string, Mount func(f fs.Fs, mountpoint string) error) *cobra.Command {
	var commandDefintion = &cobra.Command{
		Use:   commandName + " remote:path /path/to/mountpoint",
		Short: `Mount the remote as file system on a mountpoint.`,
		Long: `
rclone ` + commandName + ` allows Linux, FreeBSD, macOS and Windows to
mount any of Rclone's cloud storage systems as a file system with
FUSE.

First set up your remote using ` + "`rclone config`" + `.  Check it works with ` + "`rclone ls`" + ` etc.

Start the mount like this

    rclone ` + commandName + ` remote:path/to/files /path/to/local/mount

Or on Windows like this where X: is an unused drive letter

    rclone ` + commandName + ` remote:path/to/files X:

When the program ends, either via Ctrl+C or receiving a SIGINT or SIGTERM signal,
the mount is automatically stopped.

The umount operation can fail, for example when the mountpoint is busy.
When that happens, it is the user's responsibility to stop the mount manually with

    # Linux
    fusermount -u /path/to/local/mount
    # OS X
    umount /path/to/local/mount

### Installing on Windows

To run rclone ` + commandName + ` on Windows, you will need to
download and install [WinFsp](http://www.secfs.net/winfsp/).

WinFsp is an [open source](https://github.com/billziss-gh/winfsp)
Windows File System Proxy which makes it easy to write user space file
systems for Windows.  It provides a FUSE emulation layer which rclone
uses combination with
[cgofuse](https://github.com/billziss-gh/cgofuse).  Both of these
packages are by Bill Zissimopoulos who was very helpful during the
implementation of rclone ` + commandName + ` for Windows.

#### Windows caveats

Note that drives created as Administrator are not visible by other
accounts (including the account that was elevated as
Administrator). So if you start a Windows drive from an Administrative
Command Prompt and then try to access the same drive from Explorer
(which does not run as Administrator), you will not be able to see the
new drive.

The easiest way around this is to start the drive from a normal
command prompt. It is also possible to start a drive from the SYSTEM
account (using [the WinFsp.Launcher
infrastructure](https://github.com/billziss-gh/winfsp/wiki/WinFsp-Service-Architecture))
which creates drives accessible for everyone on the system or
alternatively using [the nssm service manager](https://nssm.cc/usage).

### Limitations

Without the use of "--vfs-cache-mode" this can only write files
sequentially, it can only seek when reading.  This means that many
applications won't work with their files on an rclone mount without
"--vfs-cache-mode writes" or "--vfs-cache-mode full".  See the [File
Caching](#file-caching) section for more info.

The bucket based remotes (eg Swift, S3, Google Compute Storage, B2,
Hubic) won't work from the root - you will need to specify a bucket,
or a path within the bucket.  So ` + "`swift:`" + ` won't work whereas
` + "`swift:bucket`" + ` will as will ` + "`swift:bucket/path`" + `.
None of these support the concept of directories, so empty
directories will have a tendency to disappear once they fall out of
the directory cache.

Only supported on Linux, FreeBSD, OS X and Windows at the moment.

### rclone ` + commandName + ` vs rclone sync/copy

File systems expect things to be 100% reliable, whereas cloud storage
systems are a long way from 100% reliable. The rclone sync/copy
commands cope with this with lots of retries.  However rclone ` + commandName + `
can't use retries in the same way without making local copies of the
uploads. Look at the [file caching](#file-caching)
for solutions to make ` + commandName + ` more reliable.

### Attribute caching

You can use the flag --attr-timeout to set the time the kernel caches
the attributes (size, modification time etc) for directory entries.

The default is "1s" which caches files just long enough to avoid
too many callbacks to rclone from the kernel.

In theory 0s should be the correct value for filesystems which can
change outside the control of the kernel. However this causes quite a
few problems such as
[rclone using too much memory](https://github.com/ncw/rclone/issues/2157),
[rclone not serving files to samba](https://forum.rclone.org/t/rclone-1-39-vs-1-40-mount-issue/5112)
and [excessive time listing directories](https://github.com/ncw/rclone/issues/2095#issuecomment-371141147).

The kernel can cache the info about a file for the time given by
"--attr-timeout". You may see corruption if the remote file changes
length during this window.  It will show up as either a truncated file
or a file with garbage on the end.  With "--attr-timeout 1s" this is
very unlikely but not impossible.  The higher you set "--attr-timeout"
the more likely it is.  The default setting of "1s" is the lowest
setting which mitigates the problems above.

If you set it higher ('10s' or '1m' say) then the kernel will call
back to rclone less often making it more efficient, however there is
more chance of the corruption issue above.

If files don't change on the remote outside of the control of rclone
then there is no chance of corruption.

This is the same as setting the attr_timeout option in mount.fuse.

### Filters

Note that all the rclone filters can be used to select a subset of the
files to be visible in the mount.

### systemd

When running rclone ` + commandName + ` as a systemd service, it is possible
to use Type=notify. In this case the service will enter the started state
after the mountpoint has been successfully set up.
Units having the rclone ` + commandName + ` service specified as a requirement
will see all files and folders immediately in this mode.

### chunked reading ###

--vfs-read-chunk-size will enable reading the source objects in parts.
This can reduce the used download quota for some remotes by requesting only chunks
from the remote that are actually read at the cost of an increased number of requests.

When --vfs-read-chunk-size-limit is also specified and greater than --vfs-read-chunk-size,
the chunk size for each open file will get doubled for each chunk read, until the
specified value is reached. A value of -1 will disable the limit and the chunk size will
grow indefinitely.

With --vfs-read-chunk-size 100M and --vfs-read-chunk-size-limit 0 the following
parts will be downloaded: 0-100M, 100M-200M, 200M-300M, 300M-400M and so on.
When --vfs-read-chunk-size-limit 500M is specified, the result would be
0-100M, 100M-300M, 300M-700M, 700M-1200M, 1200M-1700M and so on.

Chunked reading will only work with --vfs-cache-mode < full, as the file will always
be copied to the vfs cache before opening with --vfs-cache-mode full.
` + vfs.Help,
		Run: func(command *cobra.Command, args []string) {
			cmd.CheckArgs(2, 2, command, args)

			if Daemon {
				config.PassConfigKeyForDaemonization = true
			}

			fdst := cmd.NewFsDir(args)

			// Show stats if the user has specifically requested them
			if cmd.ShowStats() {
				defer cmd.StartStats()()
			}

			// Skip checkMountEmpty if --allow-non-empty flag is used or if
			// the Operating System is Windows
			if !AllowNonEmpty && runtime.GOOS != "windows" {
				err := checkMountEmpty(args[1])
				if err != nil {
					log.Fatalf("Fatal error: %v", err)
				}
			}

			// Work out the volume name, removing special
			// characters from it if necessary
			if VolumeName == "" {
				VolumeName = fdst.Name() + ":" + fdst.Root()
			}
			VolumeName = strings.Replace(VolumeName, ":", " ", -1)
			VolumeName = strings.Replace(VolumeName, "/", " ", -1)
			VolumeName = strings.TrimSpace(VolumeName)

			// Start background task if --background is specified
			if Daemon {
				daemonized := startBackgroundMode()
				if daemonized {
					return
				}
			}

			err := Mount(fdst, args[1])
			if err != nil {
				log.Fatalf("Fatal error: %v", err)
			}
		},
	}

	// Register the command
	cmd.Root.AddCommand(commandDefintion)

	// Add flags
	flagSet := commandDefintion.Flags()
	flags.BoolVarP(flagSet, &DebugFUSE, "debug-fuse", "", DebugFUSE, "Debug the FUSE internals - needs -v.")
	// mount options
	flags.BoolVarP(flagSet, &AllowNonEmpty, "allow-non-empty", "", AllowNonEmpty, "Allow mounting over a non-empty directory.")
	flags.BoolVarP(flagSet, &AllowRoot, "allow-root", "", AllowRoot, "Allow access to root user.")
	flags.BoolVarP(flagSet, &AllowOther, "allow-other", "", AllowOther, "Allow access to other users.")
	flags.BoolVarP(flagSet, &DefaultPermissions, "default-permissions", "", DefaultPermissions, "Makes kernel enforce access control based on the file mode.")
	flags.BoolVarP(flagSet, &WritebackCache, "write-back-cache", "", WritebackCache, "Makes kernel buffer writes before sending them to rclone. Without this, writethrough caching is used.")
	flags.FVarP(flagSet, &MaxReadAhead, "max-read-ahead", "", "The number of bytes that can be prefetched for sequential reads.")
	flags.DurationVarP(flagSet, &AttrTimeout, "attr-timeout", "", AttrTimeout, "Time for which file/directory attributes are cached.")
	flags.StringArrayVarP(flagSet, &ExtraOptions, "option", "o", []string{}, "Option for libfuse/WinFsp. Repeat if required.")
	flags.StringArrayVarP(flagSet, &ExtraFlags, "fuse-flag", "", []string{}, "Flags or arguments to be passed direct to libfuse/WinFsp. Repeat if required.")
	flags.BoolVarP(flagSet, &Daemon, "daemon", "", Daemon, "Run mount as a daemon (background mode).")
	flags.StringVarP(flagSet, &VolumeName, "volname", "", VolumeName, "Set the volume name (not supported by all OSes).")
	flags.DurationVarP(flagSet, &DaemonTimeout, "daemon-timeout", "", DaemonTimeout, "Time limit for rclone to respond to kernel (not supported by all OSes).")

	if runtime.GOOS == "darwin" {
		flags.BoolVarP(flagSet, &NoAppleDouble, "noappledouble", "", NoAppleDouble, "Sets the OSXFUSE option noappledouble.")
		flags.BoolVarP(flagSet, &NoAppleXattr, "noapplexattr", "", NoAppleXattr, "Sets the OSXFUSE option noapplexattr.")
	}

	// Add in the generic flags
	vfsflags.AddFlags(flagSet)

	return commandDefintion
}

// ClipBlocks clips the blocks pointed to to the OS max
func ClipBlocks(b *uint64) {
	var max uint64
	switch runtime.GOOS {
	case "windows":
		max = (1 << 43) - 1
	case "darwin":
		// OSX FUSE only supports 32 bit number of blocks
		// https://github.com/osxfuse/osxfuse/issues/396
		max = (1 << 32) - 1
	default:
		// no clipping
		return
	}
	if *b > max {
		*b = max
	}
}
<commit_msg>mount: check that mountpoint and local directory to mount don't overlap<commit_after>package mountlib

import (
	"io"
	"log"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"github.com/ncw/rclone/cmd"
	"github.com/ncw/rclone/fs"
	"github.com/ncw/rclone/fs/config"
	"github.com/ncw/rclone/fs/config/flags"
	"github.com/ncw/rclone/vfs"
	"github.com/ncw/rclone/vfs/vfsflags"
	"github.com/pkg/errors"
	"github.com/spf13/cobra"
)

// Options set by command line flags
var (
	DebugFUSE                        = false
	AllowNonEmpty                    = false
	AllowRoot                        = false
	AllowOther                       = false
	DefaultPermissions               = false
	WritebackCache                   = false
	Daemon                           = false
	MaxReadAhead       fs.SizeSuffix = 128 * 1024
	ExtraOptions       []string
	ExtraFlags         []string
	AttrTimeout        = 1 * time.Second // how long the kernel caches attribute for
	VolumeName         string
	NoAppleDouble      = true        // use noappledouble by default
	NoAppleXattr       = false       // do not use noapplexattr by default
	DaemonTimeout      time.Duration // OSXFUSE only
)

// Check is folder is empty
func checkMountEmpty(mountpoint string) error {
	fp, fpErr := os.Open(mountpoint)

	if fpErr != nil {
		return errors.Wrap(fpErr, "Can not open: "+mountpoint)
	}
	defer fs.CheckClose(fp, &fpErr)

	_, fpErr = fp.Readdirnames(1)

	// directory is not empty
	if fpErr != io.EOF {
		var e error
		var errorMsg = "Directory is not empty: " + mountpoint + " If you want to mount it anyway use: --allow-non-empty option"
		if fpErr == nil {
			e = errors.New(errorMsg)
		} else {
			e = errors.Wrap(fpErr, errorMsg)
		}
		return e
	}
	return nil
}

// Check the root doesn't overlap the mountpoint
func checkMountpointOverlap(root, mountpoint string) error {
	abs := func(x string) string {
		if absX, err := filepath.EvalSymlinks(x); err == nil {
			x = absX
		}
		if absX, err := filepath.Abs(x); err == nil {
			x = absX
		}
		x = filepath.ToSlash(x)
		if !strings.HasSuffix(x, "/") {
			x += "/"
		}
		return x
	}
	rootAbs, mountpointAbs := abs(root), abs(mountpoint)
	if strings.HasPrefix(rootAbs, mountpointAbs) || strings.HasPrefix(mountpointAbs, rootAbs) {
		return errors.Errorf("mount point %q and directory to be mounted %q mustn't overlap", mountpoint, root)
	}
	return nil
}

// NewMountCommand makes a mount command with the given name and Mount function
func NewMountCommand(commandName string, Mount func(f fs.Fs, mountpoint string) error) *cobra.Command {
	var commandDefintion = &cobra.Command{
		Use:   commandName + " remote:path /path/to/mountpoint",
		Short: `Mount the remote as file system on a mountpoint.`,
		Long: `
rclone ` + commandName + ` allows Linux, FreeBSD, macOS and Windows to
mount any of Rclone's cloud storage systems as a file system with
FUSE.

First set up your remote using ` + "`rclone config`" + `.  Check it works with ` + "`rclone ls`" + ` etc.

Start the mount like this

    rclone ` + commandName + ` remote:path/to/files /path/to/local/mount

Or on Windows like this where X: is an unused drive letter

    rclone ` + commandName + ` remote:path/to/files X:

When the program ends, either via Ctrl+C or receiving a SIGINT or SIGTERM signal,
the mount is automatically stopped.

The umount operation can fail, for example when the mountpoint is busy.
When that happens, it is the user's responsibility to stop the mount manually with

    # Linux
    fusermount -u /path/to/local/mount
    # OS X
    umount /path/to/local/mount

### Installing on Windows

To run rclone ` + commandName + ` on Windows, you will need to
download and install [WinFsp](http://www.secfs.net/winfsp/).

WinFsp is an [open source](https://github.com/billziss-gh/winfsp)
Windows File System Proxy which makes it easy to write user space file
systems for Windows.  It provides a FUSE emulation layer which rclone
uses combination with
[cgofuse](https://github.com/billziss-gh/cgofuse).  Both of these
packages are by Bill Zissimopoulos who was very helpful during the
implementation of rclone ` + commandName + ` for Windows.

#### Windows caveats

Note that drives created as Administrator are not visible by other
accounts (including the account that was elevated as
Administrator). So if you start a Windows drive from an Administrative
Command Prompt and then try to access the same drive from Explorer
(which does not run as Administrator), you will not be able to see the
new drive.

The easiest way around this is to start the drive from a normal
command prompt. It is also possible to start a drive from the SYSTEM
account (using [the WinFsp.Launcher
infrastructure](https://github.com/billziss-gh/winfsp/wiki/WinFsp-Service-Architecture))
which creates drives accessible for everyone on the system or
alternatively using [the nssm service manager](https://nssm.cc/usage).

### Limitations

Without the use of "--vfs-cache-mode" this can only write files
sequentially, it can only seek when reading.  This means that many
applications won't work with their files on an rclone mount without
"--vfs-cache-mode writes" or "--vfs-cache-mode full".  See the [File
Caching](#file-caching) section for more info.

The bucket based remotes (eg Swift, S3, Google Compute Storage, B2,
Hubic) won't work from the root - you will need to specify a bucket,
or a path within the bucket.  So ` + "`swift:`" + ` won't work whereas
` + "`swift:bucket`" + ` will as will ` + "`swift:bucket/path`" + `.
None of these support the concept of directories, so empty
directories will have a tendency to disappear once they fall out of
the directory cache.

Only supported on Linux, FreeBSD, OS X and Windows at the moment.

### rclone ` + commandName + ` vs rclone sync/copy

File systems expect things to be 100% reliable, whereas cloud storage
systems are a long way from 100% reliable. The rclone sync/copy
commands cope with this with lots of retries.  However rclone ` + commandName + `
can't use retries in the same way without making local copies of the
uploads. Look at the [file caching](#file-caching)
for solutions to make ` + commandName + ` more reliable.

### Attribute caching

You can use the flag --attr-timeout to set the time the kernel caches
the attributes (size, modification time etc) for directory entries.

The default is "1s" which caches files just long enough to avoid
too many callbacks to rclone from the kernel.

In theory 0s should be the correct value for filesystems which can
change outside the control of the kernel. However this causes quite a
few problems such as
[rclone using too much memory](https://github.com/ncw/rclone/issues/2157),
[rclone not serving files to samba](https://forum.rclone.org/t/rclone-1-39-vs-1-40-mount-issue/5112)
and [excessive time listing directories](https://github.com/ncw/rclone/issues/2095#issuecomment-371141147).

The kernel can cache the info about a file for the time given by
"--attr-timeout". You may see corruption if the remote file changes
length during this window.  It will show up as either a truncated file
or a file with garbage on the end.  With "--attr-timeout 1s" this is
very unlikely but not impossible.  The higher you set "--attr-timeout"
the more likely it is.  The default setting of "1s" is the lowest
setting which mitigates the problems above.

If you set it higher ('10s' or '1m' say) then the kernel will call
back to rclone less often making it more efficient, however there is
more chance of the corruption issue above.

If files don't change on the remote outside of the control of rclone
then there is no chance of corruption.

This is the same as setting the attr_timeout option in mount.fuse.

### Filters

Note that all the rclone filters can be used to select a subset of the
files to be visible in the mount.

### systemd

When running rclone ` + commandName + ` as a systemd service, it is possible
to use Type=notify. In this case the service will enter the started state
after the mountpoint has been successfully set up.
Units having the rclone ` + commandName + ` service specified as a requirement
will see all files and folders immediately in this mode.

### chunked reading ###

--vfs-read-chunk-size will enable reading the source objects in parts.
This can reduce the used download quota for some remotes by requesting only chunks
from the remote that are actually read at the cost of an increased number of requests.

When --vfs-read-chunk-size-limit is also specified and greater than --vfs-read-chunk-size,
the chunk size for each open file will get doubled for each chunk read, until the
specified value is reached. A value of -1 will disable the limit and the chunk size will
grow indefinitely.

With --vfs-read-chunk-size 100M and --vfs-read-chunk-size-limit 0 the following
parts will be downloaded: 0-100M, 100M-200M, 200M-300M, 300M-400M and so on.
When --vfs-read-chunk-size-limit 500M is specified, the result would be
0-100M, 100M-300M, 300M-700M, 700M-1200M, 1200M-1700M and so on.

Chunked reading will only work with --vfs-cache-mode < full, as the file will always
be copied to the vfs cache before opening with --vfs-cache-mode full.
` + vfs.Help,
		Run: func(command *cobra.Command, args []string) {
			cmd.CheckArgs(2, 2, command, args)

			if Daemon {
				config.PassConfigKeyForDaemonization = true
			}

			mountpoint := args[1]
			fdst := cmd.NewFsDir(args)
			if fdst.Name() == "" || fdst.Name() == "local" {
				err := checkMountpointOverlap(fdst.Root(), mountpoint)
				if err != nil {
					log.Fatalf("Fatal error: %v", err)
				}
			}

			// Show stats if the user has specifically requested them
			if cmd.ShowStats() {
				defer cmd.StartStats()()
			}

			// Skip checkMountEmpty if --allow-non-empty flag is used or if
			// the Operating System is Windows
			if !AllowNonEmpty && runtime.GOOS != "windows" {
				err := checkMountEmpty(mountpoint)
				if err != nil {
					log.Fatalf("Fatal error: %v", err)
				}
			}

			// Work out the volume name, removing special
			// characters from it if necessary
			if VolumeName == "" {
				VolumeName = fdst.Name() + ":" + fdst.Root()
			}
			VolumeName = strings.Replace(VolumeName, ":", " ", -1)
			VolumeName = strings.Replace(VolumeName, "/", " ", -1)
			VolumeName = strings.TrimSpace(VolumeName)

			// Start background task if --background is specified
			if Daemon {
				daemonized := startBackgroundMode()
				if daemonized {
					return
				}
			}

			err := Mount(fdst, mountpoint)
			if err != nil {
				log.Fatalf("Fatal error: %v", err)
			}
		},
	}

	// Register the command
	cmd.Root.AddCommand(commandDefintion)

	// Add flags
	flagSet := commandDefintion.Flags()
	flags.BoolVarP(flagSet, &DebugFUSE, "debug-fuse", "", DebugFUSE, "Debug the FUSE internals - needs -v.")
	// mount options
	flags.BoolVarP(flagSet, &AllowNonEmpty, "allow-non-empty", "", AllowNonEmpty, "Allow mounting over a non-empty directory.")
	flags.BoolVarP(flagSet, &AllowRoot, "allow-root", "", AllowRoot, "Allow access to root user.")
	flags.BoolVarP(flagSet, &AllowOther, "allow-other", "", AllowOther, "Allow access to other users.")
	flags.BoolVarP(flagSet, &DefaultPermissions, "default-permissions", "", DefaultPermissions, "Makes kernel enforce access control based on the file mode.")
	flags.BoolVarP(flagSet, &WritebackCache, "write-back-cache", "", WritebackCache, "Makes kernel buffer writes before sending them to rclone. Without this, writethrough caching is used.")
	flags.FVarP(flagSet, &MaxReadAhead, "max-read-ahead", "", "The number of bytes that can be prefetched for sequential reads.")
	flags.DurationVarP(flagSet, &AttrTimeout, "attr-timeout", "", AttrTimeout, "Time for which file/directory attributes are cached.")
	flags.StringArrayVarP(flagSet, &ExtraOptions, "option", "o", []string{}, "Option for libfuse/WinFsp. Repeat if required.")
	flags.StringArrayVarP(flagSet, &ExtraFlags, "fuse-flag", "", []string{}, "Flags or arguments to be passed direct to libfuse/WinFsp. Repeat if required.")
	flags.BoolVarP(flagSet, &Daemon, "daemon", "", Daemon, "Run mount as a daemon (background mode).")
	flags.StringVarP(flagSet, &VolumeName, "volname", "", VolumeName, "Set the volume name (not supported by all OSes).")
	flags.DurationVarP(flagSet, &DaemonTimeout, "daemon-timeout", "", DaemonTimeout, "Time limit for rclone to respond to kernel (not supported by all OSes).")

	if runtime.GOOS == "darwin" {
		flags.BoolVarP(flagSet, &NoAppleDouble, "noappledouble", "", NoAppleDouble, "Sets the OSXFUSE option noappledouble.")
		flags.BoolVarP(flagSet, &NoAppleXattr, "noapplexattr", "", NoAppleXattr, "Sets the OSXFUSE option noapplexattr.")
	}

	// Add in the generic flags
	vfsflags.AddFlags(flagSet)

	return commandDefintion
}

// ClipBlocks clips the blocks pointed to to the OS max
func ClipBlocks(b *uint64) {
	var max uint64
	switch runtime.GOOS {
	case "windows":
		max = (1 << 43) - 1
	case "darwin":
		// OSX FUSE only supports 32 bit number of blocks
		// https://github.com/osxfuse/osxfuse/issues/396
		max = (1 << 32) - 1
	default:
		// no clipping
		return
	}
	if *b > max {
		*b = max
	}
}
<|endoftext|>


<commit_before>package mountlib

import (
	"io"
	"log"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"github.com/pkg/errors"
	"github.com/rclone/rclone/cmd"
	"github.com/rclone/rclone/fs"
	"github.com/rclone/rclone/fs/config"
	"github.com/rclone/rclone/fs/config/flags"
	"github.com/rclone/rclone/vfs"
	"github.com/rclone/rclone/vfs/vfsflags"
	"github.com/spf13/cobra"
)

// Options set by command line flags
var (
	DebugFUSE                        = false
	AllowNonEmpty                    = false
	AllowRoot                        = false
	AllowOther                       = false
	DefaultPermissions               = false
	WritebackCache                   = false
	Daemon                           = false
	MaxReadAhead       fs.SizeSuffix = 128 * 1024
	ExtraOptions       []string
	ExtraFlags         []string
	AttrTimeout        = 1 * time.Second // how long the kernel caches attribute for
	VolumeName         string
	NoAppleDouble      = true        // use noappledouble by default
	NoAppleXattr       = false       // do not use noapplexattr by default
	DaemonTimeout      time.Duration // OSXFUSE only
)

func init() {
	// DaemonTimeout defaults to non zero for macOS and freebsd
	if runtime.GOOS == "darwin" || runtime.GOOS == "freebsd" {
		DaemonTimeout = 15 * time.Minute
	}
}

// Check is folder is empty
func checkMountEmpty(mountpoint string) error {
	fp, fpErr := os.Open(mountpoint)

	if fpErr != nil {
		return errors.Wrap(fpErr, "Can not open: "+mountpoint)
	}
	defer fs.CheckClose(fp, &fpErr)

	_, fpErr = fp.Readdirnames(1)

	// directory is not empty
	if fpErr != io.EOF {
		var e error
		var errorMsg = "Directory is not empty: " + mountpoint + " If you want to mount it anyway use: --allow-non-empty option"
		if fpErr == nil {
			e = errors.New(errorMsg)
		} else {
			e = errors.Wrap(fpErr, errorMsg)
		}
		return e
	}
	return nil
}

// Check the root doesn't overlap the mountpoint
func checkMountpointOverlap(root, mountpoint string) error {
	abs := func(x string) string {
		if absX, err := filepath.EvalSymlinks(x); err == nil {
			x = absX
		}
		if absX, err := filepath.Abs(x); err == nil {
			x = absX
		}
		x = filepath.ToSlash(x)
		if !strings.HasSuffix(x, "/") {
			x += "/"
		}
		return x
	}
	rootAbs, mountpointAbs := abs(root), abs(mountpoint)
	if strings.HasPrefix(rootAbs, mountpointAbs) || strings.HasPrefix(mountpointAbs, rootAbs) {
		return errors.Errorf("mount point %q and directory to be mounted %q mustn't overlap", mountpoint, root)
	}
	return nil
}

// NewMountCommand makes a mount command with the given name and Mount function
func NewMountCommand(commandName string, Mount func(f fs.Fs, mountpoint string) error) *cobra.Command {
	var commandDefintion = &cobra.Command{
		Use:   commandName + " remote:path /path/to/mountpoint",
		Short: `Mount the remote as file system on a mountpoint.`,
		Long: `
rclone ` + commandName + ` allows Linux, FreeBSD, macOS and Windows to
mount any of Rclone's cloud storage systems as a file system with
FUSE.

First set up your remote using ` + "`rclone config`" + `.  Check it works with ` + "`rclone ls`" + ` etc.

Start the mount like this

    rclone ` + commandName + ` remote:path/to/files /path/to/local/mount

Or on Windows like this where X: is an unused drive letter

    rclone ` + commandName + ` remote:path/to/files X:

When the program ends, either via Ctrl+C or receiving a SIGINT or SIGTERM signal,
the mount is automatically stopped.

The umount operation can fail, for example when the mountpoint is busy.
When that happens, it is the user's responsibility to stop the mount manually with

    # Linux
    fusermount -u /path/to/local/mount
    # OS X
    umount /path/to/local/mount

### Installing on Windows

To run rclone ` + commandName + ` on Windows, you will need to
download and install [WinFsp](http://www.secfs.net/winfsp/).

WinFsp is an [open source](https://github.com/billziss-gh/winfsp)
Windows File System Proxy which makes it easy to write user space file
systems for Windows.  It provides a FUSE emulation layer which rclone
uses combination with
[cgofuse](https://github.com/billziss-gh/cgofuse).  Both of these
packages are by Bill Zissimopoulos who was very helpful during the
implementation of rclone ` + commandName + ` for Windows.

#### Windows caveats

Note that drives created as Administrator are not visible by other
accounts (including the account that was elevated as
Administrator). So if you start a Windows drive from an Administrative
Command Prompt and then try to access the same drive from Explorer
(which does not run as Administrator), you will not be able to see the
new drive.

The easiest way around this is to start the drive from a normal
command prompt. It is also possible to start a drive from the SYSTEM
account (using [the WinFsp.Launcher
infrastructure](https://github.com/billziss-gh/winfsp/wiki/WinFsp-Service-Architecture))
which creates drives accessible for everyone on the system or
alternatively using [the nssm service manager](https://nssm.cc/usage).

### Limitations

Without the use of "--vfs-cache-mode" this can only write files
sequentially, it can only seek when reading.  This means that many
applications won't work with their files on an rclone mount without
"--vfs-cache-mode writes" or "--vfs-cache-mode full".  See the [File
Caching](#file-caching) section for more info.

The bucket based remotes (eg Swift, S3, Google Compute Storage, B2,
Hubic) won't work from the root - you will need to specify a bucket,
or a path within the bucket.  So ` + "`swift:`" + ` won't work whereas
` + "`swift:bucket`" + ` will as will ` + "`swift:bucket/path`" + `.
None of these support the concept of directories, so empty
directories will have a tendency to disappear once they fall out of
the directory cache.

Only supported on Linux, FreeBSD, OS X and Windows at the moment.

### rclone ` + commandName + ` vs rclone sync/copy

File systems expect things to be 100% reliable, whereas cloud storage
systems are a long way from 100% reliable. The rclone sync/copy
commands cope with this with lots of retries.  However rclone ` + commandName + `
can't use retries in the same way without making local copies of the
uploads. Look at the [file caching](#file-caching)
for solutions to make ` + commandName + ` more reliable.

### Attribute caching

You can use the flag --attr-timeout to set the time the kernel caches
the attributes (size, modification time etc) for directory entries.

The default is "1s" which caches files just long enough to avoid
too many callbacks to rclone from the kernel.

In theory 0s should be the correct value for filesystems which can
change outside the control of the kernel. However this causes quite a
few problems such as
[rclone using too much memory](https://github.com/rclone/rclone/issues/2157),
[rclone not serving files to samba](https://forum.rclone.org/t/rclone-1-39-vs-1-40-mount-issue/5112)
and [excessive time listing directories](https://github.com/rclone/rclone/issues/2095#issuecomment-371141147).

The kernel can cache the info about a file for the time given by
"--attr-timeout". You may see corruption if the remote file changes
length during this window.  It will show up as either a truncated file
or a file with garbage on the end.  With "--attr-timeout 1s" this is
very unlikely but not impossible.  The higher you set "--attr-timeout"
the more likely it is.  The default setting of "1s" is the lowest
setting which mitigates the problems above.

If you set it higher ('10s' or '1m' say) then the kernel will call
back to rclone less often making it more efficient, however there is
more chance of the corruption issue above.

If files don't change on the remote outside of the control of rclone
then there is no chance of corruption.

This is the same as setting the attr_timeout option in mount.fuse.

### Filters

Note that all the rclone filters can be used to select a subset of the
files to be visible in the mount.

### systemd

When running rclone ` + commandName + ` as a systemd service, it is possible
to use Type=notify. In this case the service will enter the started state
after the mountpoint has been successfully set up.
Units having the rclone ` + commandName + ` service specified as a requirement
will see all files and folders immediately in this mode.

### chunked reading ###

--vfs-read-chunk-size will enable reading the source objects in parts.
This can reduce the used download quota for some remotes by requesting only chunks
from the remote that are actually read at the cost of an increased number of requests.

When --vfs-read-chunk-size-limit is also specified and greater than --vfs-read-chunk-size,
the chunk size for each open file will get doubled for each chunk read, until the
specified value is reached. A value of -1 will disable the limit and the chunk size will
grow indefinitely.

With --vfs-read-chunk-size 100M and --vfs-read-chunk-size-limit 0 the following
parts will be downloaded: 0-100M, 100M-200M, 200M-300M, 300M-400M and so on.
When --vfs-read-chunk-size-limit 500M is specified, the result would be
0-100M, 100M-300M, 300M-700M, 700M-1200M, 1200M-1700M and so on.

Chunked reading will only work with --vfs-cache-mode < full, as the file will always
be copied to the vfs cache before opening with --vfs-cache-mode full.
` + vfs.Help,
		Run: func(command *cobra.Command, args []string) {
			cmd.CheckArgs(2, 2, command, args)

			if Daemon {
				config.PassConfigKeyForDaemonization = true
			}

			mountpoint := args[1]
			fdst := cmd.NewFsDir(args)
			if fdst.Name() == "" || fdst.Name() == "local" {
				err := checkMountpointOverlap(fdst.Root(), mountpoint)
				if err != nil {
					log.Fatalf("Fatal error: %v", err)
				}
			}

			// Show stats if the user has specifically requested them
			if cmd.ShowStats() {
				defer cmd.StartStats()()
			}

			// Skip checkMountEmpty if --allow-non-empty flag is used or if
			// the Operating System is Windows
			if !AllowNonEmpty && runtime.GOOS != "windows" {
				err := checkMountEmpty(mountpoint)
				if err != nil {
					log.Fatalf("Fatal error: %v", err)
				}
			}

			// Work out the volume name, removing special
			// characters from it if necessary
			if VolumeName == "" {
				VolumeName = fdst.Name() + ":" + fdst.Root()
			}
			VolumeName = strings.Replace(VolumeName, ":", " ", -1)
			VolumeName = strings.Replace(VolumeName, "/", " ", -1)
			VolumeName = strings.TrimSpace(VolumeName)

			// Start background task if --background is specified
			if Daemon {
				daemonized := startBackgroundMode()
				if daemonized {
					return
				}
			}

			err := Mount(fdst, mountpoint)
			if err != nil {
				log.Fatalf("Fatal error: %v", err)
			}
		},
	}

	// Register the command
	cmd.Root.AddCommand(commandDefintion)

	// Add flags
	flagSet := commandDefintion.Flags()
	flags.BoolVarP(flagSet, &DebugFUSE, "debug-fuse", "", DebugFUSE, "Debug the FUSE internals - needs -v.")
	// mount options
	flags.BoolVarP(flagSet, &AllowNonEmpty, "allow-non-empty", "", AllowNonEmpty, "Allow mounting over a non-empty directory.")
	flags.BoolVarP(flagSet, &AllowRoot, "allow-root", "", AllowRoot, "Allow access to root user.")
	flags.BoolVarP(flagSet, &AllowOther, "allow-other", "", AllowOther, "Allow access to other users.")
	flags.BoolVarP(flagSet, &DefaultPermissions, "default-permissions", "", DefaultPermissions, "Makes kernel enforce access control based on the file mode.")
	flags.BoolVarP(flagSet, &WritebackCache, "write-back-cache", "", WritebackCache, "Makes kernel buffer writes before sending them to rclone. Without this, writethrough caching is used.")
	flags.FVarP(flagSet, &MaxReadAhead, "max-read-ahead", "", "The number of bytes that can be prefetched for sequential reads.")
	flags.DurationVarP(flagSet, &AttrTimeout, "attr-timeout", "", AttrTimeout, "Time for which file/directory attributes are cached.")
	flags.StringArrayVarP(flagSet, &ExtraOptions, "option", "o", []string{}, "Option for libfuse/WinFsp. Repeat if required.")
	flags.StringArrayVarP(flagSet, &ExtraFlags, "fuse-flag", "", []string{}, "Flags or arguments to be passed direct to libfuse/WinFsp. Repeat if required.")
	flags.BoolVarP(flagSet, &Daemon, "daemon", "", Daemon, "Run mount as a daemon (background mode).")
	flags.StringVarP(flagSet, &VolumeName, "volname", "", VolumeName, "Set the volume name (not supported by all OSes).")
	flags.DurationVarP(flagSet, &DaemonTimeout, "daemon-timeout", "", DaemonTimeout, "Time limit for rclone to respond to kernel (not supported by all OSes).")

	if runtime.GOOS == "darwin" {
		flags.BoolVarP(flagSet, &NoAppleDouble, "noappledouble", "", NoAppleDouble, "Sets the OSXFUSE option noappledouble.")
		flags.BoolVarP(flagSet, &NoAppleXattr, "noapplexattr", "", NoAppleXattr, "Sets the OSXFUSE option noapplexattr.")
	}

	// Add in the generic flags
	vfsflags.AddFlags(flagSet)

	return commandDefintion
}

// ClipBlocks clips the blocks pointed to to the OS max
func ClipBlocks(b *uint64) {
	var max uint64
	switch runtime.GOOS {
	case "windows":
		if runtime.GOARCH == "386" {
			max = (1 << 32) - 1
		} else {
			max = (1 << 43) - 1
		}
	case "darwin":
		// OSX FUSE only supports 32 bit number of blocks
		// https://github.com/osxfuse/osxfuse/issues/396
		max = (1 << 32) - 1
	default:
		// no clipping
		return
	}
	if *b > max {
		*b = max
	}
}
<commit_msg>mount: update docs to show mounting from root OK for bucket based #3421<commit_after>package mountlib

import (
	"io"
	"log"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"github.com/pkg/errors"
	"github.com/rclone/rclone/cmd"
	"github.com/rclone/rclone/fs"
	"github.com/rclone/rclone/fs/config"
	"github.com/rclone/rclone/fs/config/flags"
	"github.com/rclone/rclone/vfs"
	"github.com/rclone/rclone/vfs/vfsflags"
	"github.com/spf13/cobra"
)

// Options set by command line flags
var (
	DebugFUSE                        = false
	AllowNonEmpty                    = false
	AllowRoot                        = false
	AllowOther                       = false
	DefaultPermissions               = false
	WritebackCache                   = false
	Daemon                           = false
	MaxReadAhead       fs.SizeSuffix = 128 * 1024
	ExtraOptions       []string
	ExtraFlags         []string
	AttrTimeout        = 1 * time.Second // how long the kernel caches attribute for
	VolumeName         string
	NoAppleDouble      = true        // use noappledouble by default
	NoAppleXattr       = false       // do not use noapplexattr by default
	DaemonTimeout      time.Duration // OSXFUSE only
)

func init() {
	// DaemonTimeout defaults to non zero for macOS and freebsd
	if runtime.GOOS == "darwin" || runtime.GOOS == "freebsd" {
		DaemonTimeout = 15 * time.Minute
	}
}

// Check is folder is empty
func checkMountEmpty(mountpoint string) error {
	fp, fpErr := os.Open(mountpoint)

	if fpErr != nil {
		return errors.Wrap(fpErr, "Can not open: "+mountpoint)
	}
	defer fs.CheckClose(fp, &fpErr)

	_, fpErr = fp.Readdirnames(1)

	// directory is not empty
	if fpErr != io.EOF {
		var e error
		var errorMsg = "Directory is not empty: " + mountpoint + " If you want to mount it anyway use: --allow-non-empty option"
		if fpErr == nil {
			e = errors.New(errorMsg)
		} else {
			e = errors.Wrap(fpErr, errorMsg)
		}
		return e
	}
	return nil
}

// Check the root doesn't overlap the mountpoint
func checkMountpointOverlap(root, mountpoint string) error {
	abs := func(x string) string {
		if absX, err := filepath.EvalSymlinks(x); err == nil {
			x = absX
		}
		if absX, err := filepath.Abs(x); err == nil {
			x = absX
		}
		x = filepath.ToSlash(x)
		if !strings.HasSuffix(x, "/") {
			x += "/"
		}
		return x
	}
	rootAbs, mountpointAbs := abs(root), abs(mountpoint)
	if strings.HasPrefix(rootAbs, mountpointAbs) || strings.HasPrefix(mountpointAbs, rootAbs) {
		return errors.Errorf("mount point %q and directory to be mounted %q mustn't overlap", mountpoint, root)
	}
	return nil
}

// NewMountCommand makes a mount command with the given name and Mount function
func NewMountCommand(commandName string, Mount func(f fs.Fs, mountpoint string) error) *cobra.Command {
	var commandDefintion = &cobra.Command{
		Use:   commandName + " remote:path /path/to/mountpoint",
		Short: `Mount the remote as file system on a mountpoint.`,
		Long: `
rclone ` + commandName + ` allows Linux, FreeBSD, macOS and Windows to
mount any of Rclone's cloud storage systems as a file system with
FUSE.

First set up your remote using ` + "`rclone config`" + `.  Check it works with ` + "`rclone ls`" + ` etc.

Start the mount like this

    rclone ` + commandName + ` remote:path/to/files /path/to/local/mount

Or on Windows like this where X: is an unused drive letter

    rclone ` + commandName + ` remote:path/to/files X:

When the program ends, either via Ctrl+C or receiving a SIGINT or SIGTERM signal,
the mount is automatically stopped.

The umount operation can fail, for example when the mountpoint is busy.
When that happens, it is the user's responsibility to stop the mount manually with

    # Linux
    fusermount -u /path/to/local/mount
    # OS X
    umount /path/to/local/mount

### Installing on Windows

To run rclone ` + commandName + ` on Windows, you will need to
download and install [WinFsp](http://www.secfs.net/winfsp/).

WinFsp is an [open source](https://github.com/billziss-gh/winfsp)
Windows File System Proxy which makes it easy to write user space file
systems for Windows.  It provides a FUSE emulation layer which rclone
uses combination with
[cgofuse](https://github.com/billziss-gh/cgofuse).  Both of these
packages are by Bill Zissimopoulos who was very helpful during the
implementation of rclone ` + commandName + ` for Windows.

#### Windows caveats

Note that drives created as Administrator are not visible by other
accounts (including the account that was elevated as
Administrator). So if you start a Windows drive from an Administrative
Command Prompt and then try to access the same drive from Explorer
(which does not run as Administrator), you will not be able to see the
new drive.

The easiest way around this is to start the drive from a normal
command prompt. It is also possible to start a drive from the SYSTEM
account (using [the WinFsp.Launcher
infrastructure](https://github.com/billziss-gh/winfsp/wiki/WinFsp-Service-Architecture))
which creates drives accessible for everyone on the system or
alternatively using [the nssm service manager](https://nssm.cc/usage).

### Limitations

Without the use of "--vfs-cache-mode" this can only write files
sequentially, it can only seek when reading.  This means that many
applications won't work with their files on an rclone mount without
"--vfs-cache-mode writes" or "--vfs-cache-mode full".  See the [File
Caching](#file-caching) section for more info.

The bucket based remotes (eg Swift, S3, Google Compute Storage, B2,
Hubic) do not support the concept of empty directories, so empty
directories will have a tendency to disappear once they fall out of
the directory cache.

Only supported on Linux, FreeBSD, OS X and Windows at the moment.

### rclone ` + commandName + ` vs rclone sync/copy

File systems expect things to be 100% reliable, whereas cloud storage
systems are a long way from 100% reliable. The rclone sync/copy
commands cope with this with lots of retries.  However rclone ` + commandName + `
can't use retries in the same way without making local copies of the
uploads. Look at the [file caching](#file-caching)
for solutions to make ` + commandName + ` more reliable.

### Attribute caching

You can use the flag --attr-timeout to set the time the kernel caches
the attributes (size, modification time etc) for directory entries.

The default is "1s" which caches files just long enough to avoid
too many callbacks to rclone from the kernel.

In theory 0s should be the correct value for filesystems which can
change outside the control of the kernel. However this causes quite a
few problems such as
[rclone using too much memory](https://github.com/rclone/rclone/issues/2157),
[rclone not serving files to samba](https://forum.rclone.org/t/rclone-1-39-vs-1-40-mount-issue/5112)
and [excessive time listing directories](https://github.com/rclone/rclone/issues/2095#issuecomment-371141147).

The kernel can cache the info about a file for the time given by
"--attr-timeout". You may see corruption if the remote file changes
length during this window.  It will show up as either a truncated file
or a file with garbage on the end.  With "--attr-timeout 1s" this is
very unlikely but not impossible.  The higher you set "--attr-timeout"
the more likely it is.  The default setting of "1s" is the lowest
setting which mitigates the problems above.

If you set it higher ('10s' or '1m' say) then the kernel will call
back to rclone less often making it more efficient, however there is
more chance of the corruption issue above.

If files don't change on the remote outside of the control of rclone
then there is no chance of corruption.

This is the same as setting the attr_timeout option in mount.fuse.

### Filters

Note that all the rclone filters can be used to select a subset of the
files to be visible in the mount.

### systemd

When running rclone ` + commandName + ` as a systemd service, it is possible
to use Type=notify. In this case the service will enter the started state
after the mountpoint has been successfully set up.
Units having the rclone ` + commandName + ` service specified as a requirement
will see all files and folders immediately in this mode.

### chunked reading ###

--vfs-read-chunk-size will enable reading the source objects in parts.
This can reduce the used download quota for some remotes by requesting only chunks
from the remote that are actually read at the cost of an increased number of requests.

When --vfs-read-chunk-size-limit is also specified and greater than --vfs-read-chunk-size,
the chunk size for each open file will get doubled for each chunk read, until the
specified value is reached. A value of -1 will disable the limit and the chunk size will
grow indefinitely.

With --vfs-read-chunk-size 100M and --vfs-read-chunk-size-limit 0 the following
parts will be downloaded: 0-100M, 100M-200M, 200M-300M, 300M-400M and so on.
When --vfs-read-chunk-size-limit 500M is specified, the result would be
0-100M, 100M-300M, 300M-700M, 700M-1200M, 1200M-1700M and so on.

Chunked reading will only work with --vfs-cache-mode < full, as the file will always
be copied to the vfs cache before opening with --vfs-cache-mode full.
` + vfs.Help,
		Run: func(command *cobra.Command, args []string) {
			cmd.CheckArgs(2, 2, command, args)

			if Daemon {
				config.PassConfigKeyForDaemonization = true
			}

			mountpoint := args[1]
			fdst := cmd.NewFsDir(args)
			if fdst.Name() == "" || fdst.Name() == "local" {
				err := checkMountpointOverlap(fdst.Root(), mountpoint)
				if err != nil {
					log.Fatalf("Fatal error: %v", err)
				}
			}

			// Show stats if the user has specifically requested them
			if cmd.ShowStats() {
				defer cmd.StartStats()()
			}

			// Skip checkMountEmpty if --allow-non-empty flag is used or if
			// the Operating System is Windows
			if !AllowNonEmpty && runtime.GOOS != "windows" {
				err := checkMountEmpty(mountpoint)
				if err != nil {
					log.Fatalf("Fatal error: %v", err)
				}
			}

			// Work out the volume name, removing special
			// characters from it if necessary
			if VolumeName == "" {
				VolumeName = fdst.Name() + ":" + fdst.Root()
			}
			VolumeName = strings.Replace(VolumeName, ":", " ", -1)
			VolumeName = strings.Replace(VolumeName, "/", " ", -1)
			VolumeName = strings.TrimSpace(VolumeName)

			// Start background task if --background is specified
			if Daemon {
				daemonized := startBackgroundMode()
				if daemonized {
					return
				}
			}

			err := Mount(fdst, mountpoint)
			if err != nil {
				log.Fatalf("Fatal error: %v", err)
			}
		},
	}

	// Register the command
	cmd.Root.AddCommand(commandDefintion)

	// Add flags
	flagSet := commandDefintion.Flags()
	flags.BoolVarP(flagSet, &DebugFUSE, "debug-fuse", "", DebugFUSE, "Debug the FUSE internals - needs -v.")
	// mount options
	flags.BoolVarP(flagSet, &AllowNonEmpty, "allow-non-empty", "", AllowNonEmpty, "Allow mounting over a non-empty directory.")
	flags.BoolVarP(flagSet, &AllowRoot, "allow-root", "", AllowRoot, "Allow access to root user.")
	flags.BoolVarP(flagSet, &AllowOther, "allow-other", "", AllowOther, "Allow access to other users.")
	flags.BoolVarP(flagSet, &DefaultPermissions, "default-permissions", "", DefaultPermissions, "Makes kernel enforce access control based on the file mode.")
	flags.BoolVarP(flagSet, &WritebackCache, "write-back-cache", "", WritebackCache, "Makes kernel buffer writes before sending them to rclone. Without this, writethrough caching is used.")
	flags.FVarP(flagSet, &MaxReadAhead, "max-read-ahead", "", "The number of bytes that can be prefetched for sequential reads.")
	flags.DurationVarP(flagSet, &AttrTimeout, "attr-timeout", "", AttrTimeout, "Time for which file/directory attributes are cached.")
	flags.StringArrayVarP(flagSet, &ExtraOptions, "option", "o", []string{}, "Option for libfuse/WinFsp. Repeat if required.")
	flags.StringArrayVarP(flagSet, &ExtraFlags, "fuse-flag", "", []string{}, "Flags or arguments to be passed direct to libfuse/WinFsp. Repeat if required.")
	flags.BoolVarP(flagSet, &Daemon, "daemon", "", Daemon, "Run mount as a daemon (background mode).")
	flags.StringVarP(flagSet, &VolumeName, "volname", "", VolumeName, "Set the volume name (not supported by all OSes).")
	flags.DurationVarP(flagSet, &DaemonTimeout, "daemon-timeout", "", DaemonTimeout, "Time limit for rclone to respond to kernel (not supported by all OSes).")

	if runtime.GOOS == "darwin" {
		flags.BoolVarP(flagSet, &NoAppleDouble, "noappledouble", "", NoAppleDouble, "Sets the OSXFUSE option noappledouble.")
		flags.BoolVarP(flagSet, &NoAppleXattr, "noapplexattr", "", NoAppleXattr, "Sets the OSXFUSE option noapplexattr.")
	}

	// Add in the generic flags
	vfsflags.AddFlags(flagSet)

	return commandDefintion
}

// ClipBlocks clips the blocks pointed to to the OS max
func ClipBlocks(b *uint64) {
	var max uint64
	switch runtime.GOOS {
	case "windows":
		if runtime.GOARCH == "386" {
			max = (1 << 32) - 1
		} else {
			max = (1 << 43) - 1
		}
	case "darwin":
		// OSX FUSE only supports 32 bit number of blocks
		// https://github.com/osxfuse/osxfuse/issues/396
		max = (1 << 32) - 1
	default:
		// no clipping
		return
	}
	if *b > max {
		*b = max
	}
}
<|endoftext|>


<commit_before>package main // import "github.com/prometheus/prombench/cmd/prombench"

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/pkg/errors"
	"github.com/prometheus/prombench/provider/gke"
	"gopkg.in/alecthomas/kingpin.v2"
)

func main() {

	app := kingpin.New(filepath.Base(os.Args[0]), "The Prometheus benchmarking tool")
	app.HelpFlag.Short('h')

	g := gke.New()
	k8sGKE := app.Command("gke", `Google container engine provider - https://cloud.google.com/kubernetes-engine/`).
		Action(g.NewGKEClient).
		Action(g.ConfigParse)
	k8sGKE.Flag("auth", "json authentication file for the project - https://cloud.google.com/iam/docs/creating-managing-service-account-keys. If not set the tool will use the GOOGLE_APPLICATION_CREDENTIALS env variable (export GOOGLE_APPLICATION_CREDENTIALS=service-account.json)").
		Required().
		PlaceHolder("service-account.json").
		Short('a').
		ExistingFileVar(&g.AuthFile)
	k8sGKE.Flag("config", "GKE cluster-config yaml file").
		PlaceHolder("cluster.yaml").
		Short('c').
		Default("config/cluster.yaml").
		ExistingFileVar(&g.ConfigFile)
	k8sGKE.Flag("file", "yaml file used to apply or delete k8s resources. If directory is given, all the yaml files from are read recursively from it.").
		PlaceHolder("resources.yaml").
		Short('f').
		Default("manifests").
		ExistingFilesOrDirsVar(&g.ResourceFiles)
	k8sGKE.Flag("vars", "When provided it will substitute the token holders in the yaml file. Follows the standard golang template formating - {{ .hashStable }}.").
		Short('v').
		StringMapVar(&g.ResourceVars)

	// cluster operations
	k8sGKECluster := k8sGKE.Command("cluster", "Create or delete GKE k8s clusters")
	k8sGKECluster.Command("create", "gke cluster create -a service-account.json -c config/cluster.yaml").
		Action(g.ClusterCreate)
	k8sGKECluster.Command("delete", "gke cluster delete -a service-account.json -c config/cluster.yaml").
		Action(g.ClusterDelete)

	// node-pool operations
	k8sGKENodePool := k8sGKE.Command("nodepool", "Scale up or down a k8s clusters using node-pools")
	k8sGKENodePool.Command("create", "gke nodepool create -a service-account.json -c config/cluster.yaml").
		Action(g.NodePoolCreate)
	k8sGKENodePool.Command("delete", "gke nodepool delete -a service-account.json -c config/cluster.yaml").
		Action(g.NodePoolDelete)

	k8sGKEResource := k8sGKE.Command("resource", "Create,update and delete different k8s resources - deployments, services, config maps etc.").
		Action(g.NewResourceClient)
	k8sGKEResource.Command("apply", "gke resource apply -a service-account.json -c config/cluster.yaml -f manifests -v hashStable:COMMIT1 -v hashTesting:COMMIT2").
		Action(g.ResourceApply)
	k8sGKEResource.Command("delete", "gke resource delete -a service-account.json -c config/cluster.yaml -f manifests -v hashStable:COMMIT1 -v hashTesting:COMMIT2").
		Action(g.ResourceDelete)

	if _, err := app.Parse(os.Args[1:]); err != nil {
		fmt.Fprintln(os.Stderr, errors.Wrapf(err, "Error parsing commandline arguments"))
		app.Usage(os.Args[1:])
		os.Exit(2)
	}

}
<commit_msg>when the flag is required we can't use GOOGLE_APPLICATION_CREDENTIALS=service-account.json<commit_after>package main // import "github.com/prometheus/prombench/cmd/prombench"

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/pkg/errors"
	"github.com/prometheus/prombench/provider/gke"
	"gopkg.in/alecthomas/kingpin.v2"
)

func main() {

	app := kingpin.New(filepath.Base(os.Args[0]), "The Prometheus benchmarking tool")
	app.HelpFlag.Short('h')

	g := gke.New()
	k8sGKE := app.Command("gke", `Google container engine provider - https://cloud.google.com/kubernetes-engine/`).
		Action(g.NewGKEClient).
		Action(g.ConfigParse)
	k8sGKE.Flag("auth", "json authentication file for the project - https://cloud.google.com/iam/docs/creating-managing-service-account-keys. If not set the tool will use the GOOGLE_APPLICATION_CREDENTIALS env variable (export GOOGLE_APPLICATION_CREDENTIALS=service-account.json)").
		PlaceHolder("service-account.json").
		Short('a').
		ExistingFileVar(&g.AuthFile)
	k8sGKE.Flag("config", "GKE cluster-config yaml file").
		PlaceHolder("cluster.yaml").
		Short('c').
		Default("config/cluster.yaml").
		ExistingFileVar(&g.ConfigFile)
	k8sGKE.Flag("file", "yaml file used to apply or delete k8s resources. If directory is given, all the yaml files from are read recursively from it.").
		PlaceHolder("resources.yaml").
		Short('f').
		Default("manifests").
		ExistingFilesOrDirsVar(&g.ResourceFiles)
	k8sGKE.Flag("vars", "When provided it will substitute the token holders in the yaml file. Follows the standard golang template formating - {{ .hashStable }}.").
		Short('v').
		StringMapVar(&g.ResourceVars)

	// cluster operations
	k8sGKECluster := k8sGKE.Command("cluster", "Create or delete GKE k8s clusters")
	k8sGKECluster.Command("create", "gke cluster create -a service-account.json -c config/cluster.yaml").
		Action(g.ClusterCreate)
	k8sGKECluster.Command("delete", "gke cluster delete -a service-account.json -c config/cluster.yaml").
		Action(g.ClusterDelete)

	// node-pool operations
	k8sGKENodePool := k8sGKE.Command("nodepool", "Scale up or down a k8s clusters using node-pools")
	k8sGKENodePool.Command("create", "gke nodepool create -a service-account.json -c config/cluster.yaml").
		Action(g.NodePoolCreate)
	k8sGKENodePool.Command("delete", "gke nodepool delete -a service-account.json -c config/cluster.yaml").
		Action(g.NodePoolDelete)

	k8sGKEResource := k8sGKE.Command("resource", "Create,update and delete different k8s resources - deployments, services, config maps etc.").
		Action(g.NewResourceClient)
	k8sGKEResource.Command("apply", "gke resource apply -a service-account.json -c config/cluster.yaml -f manifests -v hashStable:COMMIT1 -v hashTesting:COMMIT2").
		Action(g.ResourceApply)
	k8sGKEResource.Command("delete", "gke resource delete -a service-account.json -c config/cluster.yaml -f manifests -v hashStable:COMMIT1 -v hashTesting:COMMIT2").
		Action(g.ResourceDelete)

	if _, err := app.Parse(os.Args[1:]); err != nil {
		fmt.Fprintln(os.Stderr, errors.Wrapf(err, "Error parsing commandline arguments"))
		app.Usage(os.Args[1:])
		os.Exit(2)
	}

}
<|endoftext|>


<commit_before>package main

import (
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/codegangsta/cli"
	"github.com/cryptix/goPshdlRest/api"
	"github.com/visionmedia/go-debug"
	"gopkg.in/fsnotify.v0"
)

const (
	appName  = "pshdlSync"
	widFname = ".wid"
)

var dbg = debug.Debug(appName)

func main() {
	app := cli.NewApp()
	app.Name = appName
	app.Usage = "sync a remote PSHDL workspace with your filesystem"
	app.Flags = []cli.Flag{
		cli.StringFlag{Name: "workspace,w", Usage: "specifiy the workspace to connect to"},
	}
	app.Action = run

	app.Run(os.Args)
}

func run(c *cli.Context) {
	var (
		err    error
		client *pshdlApi.Client
		wp     *pshdlApi.Workspace
	)

	widStat, widStatErr := os.Stat(widFname)

	if os.IsNotExist(widStatErr) {
		// TODO: pshdlApi.CreateWorkspace()
		wp, _, err = client.Workspace.Create()
		check(err)
		log.Println("Workspace Created:", wp.ID)

		err = ioutil.WriteFile(widFname, []byte(wp.ID), os.ModePerm-7)
		check(err)
	}

	if widStat.Mode().IsRegular() {
		wid, err := ioutil.ReadFile(widFname)
		check(err)

		// TODO: pshdlApi.OpenWorkspace()
		client = pshdlApi.NewClientWithID(nil, string(wid[:16]))
	}

	wp, _, err = client.Workspace.GetInfo()
	check(err)
	log.Printf("Workspace Opened:%s - PID:%d", wp.ID, os.Getpid())
	log.Println("Files:")
	recs := make([]pshdlApi.Record, len(wp.Files))
	for i, f := range wp.Files {
		log.Printf("* %s\nInfos:%s\n", f.Record.RelPath, f.ModuleInfos)
		recs[i] = f.Record
	}

	// todo check if files allready there
	err = client.Workspace.DownloadRecords(recs)
	check(err)
	log.Println("Download of PSHDL-Code complete.")

	//todo push containing files
	log.Println("Starting to watch..")

	watcher, err := fsnotify.NewWatcher()
	check(err)
	defer watcher.Close()

	done := make(chan struct{})

	// Process events
	go func() {
		for err := range watcher.Errors {
			check(err)
			close(done)
		}
	}()

	go func() {
		for ev := range watcher.Events {
			dbg("watcher event: %s", ev)

			if strings.HasSuffix(ev.Name, ".pshdl") {
				switch {

				case ev.Op&fsnotify.Write == fsnotify.Write:
					log.Println("write to ", ev.Name, ", uploading...")
					file, err := os.Open(ev.Name)
					check(err)

					err = client.Workspace.UploadFile(filepath.Base(ev.Name), file)
					if err != nil {
						log.Fatalf("UploadFile error: %s\n", err)
						return
					}
					file.Close()

					_, err = client.Compiler.Validate()
					if err != nil {
						log.Printf("Validate error: %s\n", err)
					}

				case ev.Op&fsnotify.Remove == fsnotify.Remove:
					log.Println(ev.Name, "deleted, skipping...")
					// client.Workspace.Delete(ev.Name)
				}
			}
		}
		close(done)
	}()

	cwd, err := os.Getwd()
	check(err)

	err = watcher.Add(cwd)
	check(err)

	<-done

	os.Exit(0)
}

func check(err error) {
	if err != nil {
		_, file, line, _ := runtime.Caller(1)
		log.Printf("Fatal from <%s:%d>\n", file, line)
		log.Fatal("Error:", err)
	}
}
<commit_msg>sublime recreates a file on save...<commit_after>package main

import (
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/codegangsta/cli"
	"github.com/cryptix/goPshdlRest/api"
	"github.com/visionmedia/go-debug"
	"gopkg.in/fsnotify.v0"
)

const (
	appName  = "pshdlSync"
	widFname = ".wid"
)

var dbg = debug.Debug(appName)

func main() {
	app := cli.NewApp()
	app.Name = appName
	app.Usage = "sync a remote PSHDL workspace with your filesystem"
	app.Flags = []cli.Flag{
		cli.StringFlag{Name: "workspace,w", Usage: "specifiy the workspace to connect to"},
	}
	app.Action = run

	app.Run(os.Args)
}

func run(c *cli.Context) {
	var (
		err    error
		client *pshdlApi.Client
		wp     *pshdlApi.Workspace
	)

	widStat, widStatErr := os.Stat(widFname)

	if os.IsNotExist(widStatErr) {
		// TODO: pshdlApi.CreateWorkspace()
		wp, _, err = client.Workspace.Create()
		check(err)
		log.Println("Workspace Created:", wp.ID)

		err = ioutil.WriteFile(widFname, []byte(wp.ID), os.ModePerm-7)
		check(err)
	}

	if widStat.Mode().IsRegular() {
		wid, err := ioutil.ReadFile(widFname)
		check(err)

		// TODO: pshdlApi.OpenWorkspace()
		client = pshdlApi.NewClientWithID(nil, string(wid[:16]))
	}

	wp, _, err = client.Workspace.GetInfo()
	check(err)
	log.Printf("Workspace Opened:%s - PID:%d", wp.ID, os.Getpid())
	log.Println("Files:")
	recs := make([]pshdlApi.Record, len(wp.Files))
	for i, f := range wp.Files {
		log.Printf("* %s\nInfos:%s\n", f.Record.RelPath, f.ModuleInfos)
		recs[i] = f.Record
	}

	// todo check if files allready there
	err = client.Workspace.DownloadRecords(recs)
	check(err)
	log.Println("Download of PSHDL-Code complete.")

	//todo push containing files
	log.Println("Starting to watch..")

	watcher, err := fsnotify.NewWatcher()
	check(err)
	defer watcher.Close()

	done := make(chan struct{})

	// Process events
	go func() {
		for err := range watcher.Errors {
			check(err)
			close(done)
		}
	}()

	go func() {
		for ev := range watcher.Events {
			dbg("watcher event: %s", ev)

			if strings.HasSuffix(ev.Name, ".pshdl") {
				switch {
				case ev.Op&fsnotify.Create == fsnotify.Create:
					fallthrough
				case ev.Op&fsnotify.Write == fsnotify.Write:
					log.Println("write to ", ev.Name, ", uploading...")
					file, err := os.Open(ev.Name)
					check(err)

					err = client.Workspace.UploadFile(filepath.Base(ev.Name), file)
					if err != nil {
						log.Fatalf("UploadFile error: %s\n", err)
						return
					}
					file.Close()

					_, err = client.Compiler.Validate()
					if err != nil {
						log.Printf("Validate error: %s\n", err)
					}

				case ev.Op&fsnotify.Remove == fsnotify.Remove:
					log.Println(ev.Name, "deleted, skipping...")
					// client.Workspace.Delete(ev.Name)
				}
			}
		}
		close(done)
	}()

	cwd, err := os.Getwd()
	check(err)

	err = watcher.Add(cwd)
	check(err)

	<-done

	os.Exit(0)
}

func check(err error) {
	if err != nil {
		_, file, line, _ := runtime.Caller(1)
		log.Printf("Fatal from <%s:%d>\n", file, line)
		log.Fatal("Error:", err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"os/signal"
	"sync"
	"syscall"

	"github.com/restic/restic/internal/debug"
)

var cleanupHandlers struct {
	sync.Mutex
	list []func() error
	done bool
	ch   chan os.Signal
}

var stderr = os.Stderr

func init() {
	cleanupHandlers.ch = make(chan os.Signal)
	go CleanupHandler(cleanupHandlers.ch)
	InstallSignalHandler()
}

// InstallSignalHandler listens for SIGINT and SIGPIPE, and triggers the cleanup handlers.
func InstallSignalHandler() {
	signal.Notify(cleanupHandlers.ch, syscall.SIGINT)
	signal.Notify(cleanupHandlers.ch, syscall.SIGPIPE)
}

// SuspendSignalHandler removes the signal handler for SIGINT and SIGPIPE.
func SuspendSignalHandler() {
	signal.Reset(syscall.SIGINT)
	signal.Reset(syscall.SIGPIPE)
}

// AddCleanupHandler adds the function f to the list of cleanup handlers so
// that it is executed when all the cleanup handlers are run, e.g. when SIGINT
// is received.
func AddCleanupHandler(f func() error) {
	cleanupHandlers.Lock()
	defer cleanupHandlers.Unlock()

	// reset the done flag for integration tests
	cleanupHandlers.done = false

	cleanupHandlers.list = append(cleanupHandlers.list, f)
}

// RunCleanupHandlers runs all registered cleanup handlers
func RunCleanupHandlers() {
	cleanupHandlers.Lock()
	defer cleanupHandlers.Unlock()

	if cleanupHandlers.done {
		return
	}
	cleanupHandlers.done = true

	for _, f := range cleanupHandlers.list {
		err := f()
		if err != nil {
			fmt.Fprintf(stderr, "error in cleanup handler: %v\n", err)
		}
	}
	cleanupHandlers.list = nil
}

// CleanupHandler handles the SIGINT and SIGPIPE signals.
func CleanupHandler(c <-chan os.Signal) {
	for s := range c {
		debug.Log("signal %v received, cleaning up", s)
		fmt.Fprintf(stderr, "%ssignal %v received, cleaning up\n", ClearLine(), s)

		code := 0
		if s != syscall.SIGINT {
			code = 1
		}

		Exit(code)
	}
}

// Exit runs the cleanup handlers and then terminates the process with the
// given exit code.
func Exit(code int) {
	RunCleanupHandlers()
	os.Exit(code)
}
<commit_msg>Ignore SIGPIPE<commit_after>package main

import (
	"fmt"
	"os"
	"os/signal"
	"sync"
	"syscall"

	"github.com/restic/restic/internal/debug"
)

var cleanupHandlers struct {
	sync.Mutex
	list []func() error
	done bool
	ch   chan os.Signal
}

var stderr = os.Stderr

func init() {
	cleanupHandlers.ch = make(chan os.Signal)
	go CleanupHandler(cleanupHandlers.ch)
	InstallSignalHandler()
}

// InstallSignalHandler listens for SIGINT, and triggers the cleanup handlers.
func InstallSignalHandler() {
	signal.Notify(cleanupHandlers.ch, syscall.SIGINT)
}

// SuspendSignalHandler removes the signal handler for SIGINT.
func SuspendSignalHandler() {
	signal.Reset(syscall.SIGINT)
}

// AddCleanupHandler adds the function f to the list of cleanup handlers so
// that it is executed when all the cleanup handlers are run, e.g. when SIGINT
// is received.
func AddCleanupHandler(f func() error) {
	cleanupHandlers.Lock()
	defer cleanupHandlers.Unlock()

	// reset the done flag for integration tests
	cleanupHandlers.done = false

	cleanupHandlers.list = append(cleanupHandlers.list, f)
}

// RunCleanupHandlers runs all registered cleanup handlers
func RunCleanupHandlers() {
	cleanupHandlers.Lock()
	defer cleanupHandlers.Unlock()

	if cleanupHandlers.done {
		return
	}
	cleanupHandlers.done = true

	for _, f := range cleanupHandlers.list {
		err := f()
		if err != nil {
			fmt.Fprintf(stderr, "error in cleanup handler: %v\n", err)
		}
	}
	cleanupHandlers.list = nil
}

// CleanupHandler handles the SIGINT signals.
func CleanupHandler(c <-chan os.Signal) {
	for s := range c {
		debug.Log("signal %v received, cleaning up", s)
		fmt.Fprintf(stderr, "%ssignal %v received, cleaning up\n", ClearLine(), s)

		code := 0
		if s != syscall.SIGINT {
			code = 1
		}

		Exit(code)
	}
}

// Exit runs the cleanup handlers and then terminates the process with the
// given exit code.
func Exit(code int) {
	RunCleanupHandlers()
	os.Exit(code)
}
<|endoftext|>


<commit_before>/*
 * Copyright © 2015-2018 Aeneas Rekkas <aeneas+oss@aeneas.io>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * @author		Aeneas Rekkas <aeneas+oss@aeneas.io>
 * @copyright 	2015-2018 Aeneas Rekkas <aeneas+oss@aeneas.io>
 * @license 	Apache-2.0
 */

package server

import (
	"crypto/tls"
	"fmt"
	"net/http"
	"net/url"
	"os"

	"strconv"
	"strings"

	"github.com/gorilla/context"
	"github.com/julienschmidt/httprouter"
	"github.com/meatballhat/negroni-logrus"
	"github.com/ory/graceful"
	"github.com/ory/herodot"
	"github.com/ory/hydra/client"
	"github.com/ory/hydra/config"
	"github.com/ory/hydra/consent"
	"github.com/ory/hydra/jwk"
	"github.com/ory/hydra/oauth2"
	"github.com/ory/hydra/pkg"
	"github.com/pkg/errors"
	"github.com/rs/cors"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
	"github.com/urfave/negroni"
)

var _ = &consent.Handler{}

func parseCorsOptions() cors.Options {
	allowCredentials, _ := strconv.ParseBool(viper.GetString("CORS_ALLOWED_CREDENTIALS"))
	debug, _ := strconv.ParseBool(viper.GetString("CORS_DEBUG"))
	maxAge, _ := strconv.Atoi(viper.GetString("CORS_MAX_AGE"))
	return cors.Options{
		AllowedOrigins:   strings.Split(viper.GetString("CORS_ALLOWED_ORIGINS"), ","),
		AllowedMethods:   strings.Split(viper.GetString("CORS_ALLOWED_METHODS"), ","),
		AllowedHeaders:   strings.Split(viper.GetString("CORS_ALLOWED_HEADERS"), ","),
		ExposedHeaders:   strings.Split(viper.GetString("CORS_EXPOSED_HEADERS"), ","),
		AllowCredentials: allowCredentials,
		MaxAge:           maxAge,
		Debug:            debug,
	}
}

func RunHost(c *config.Config) func(cmd *cobra.Command, args []string) {
	return func(cmd *cobra.Command, args []string) {
		fmt.Println(banner)

		router := httprouter.New()
		logger := c.GetLogger()
		serverHandler := &Handler{
			Config: c,
			H:      herodot.NewJSONWriter(logger),
		}
		serverHandler.registerRoutes(router)
		c.ForceHTTP, _ = cmd.Flags().GetBool("dangerous-force-http")

		if !c.ForceHTTP {
			if c.Issuer == "" {
				logger.Fatalln("Issuer must be explicitly specified unless --dangerous-force-http is passed. To find out more, use `hydra help host`.")
			}
			issuer, err := url.Parse(c.Issuer)
			pkg.Must(err, "Could not parse issuer URL: %s", err)
			if issuer.Scheme != "https" {
				logger.Fatalln("Issuer must use HTTPS unless --dangerous-force-http is passed. To find out more, use `hydra help host`.")
			}
		}

		n := negroni.New()

		if ok, _ := cmd.Flags().GetBool("disable-telemetry"); !ok && os.Getenv("DISABLE_TELEMETRY") != "1" {
			metrics := c.GetMetrics()
			go metrics.RegisterSegment()
			go metrics.CommitMemoryStatistics()
			n.Use(metrics)
		}

		n.Use(negronilogrus.NewMiddlewareFromLogger(logger, c.Issuer))
		n.UseFunc(serverHandler.rejectInsecureRequests)
		n.UseHandler(router)
		corsHandler := cors.New(parseCorsOptions()).Handler(n)

		var srv = graceful.WithDefaults(&http.Server{
			Addr:    c.GetAddress(),
			Handler: context.ClearHandler(corsHandler),
			TLSConfig: &tls.Config{
				Certificates: []tls.Certificate{getOrCreateTLSCertificate(cmd, c)},
			},
		})

		err := graceful.Graceful(func() error {
			var err error
			logger.Infof("Setting up http server on %s", c.GetAddress())
			if c.ForceHTTP {
				logger.Warnln("HTTPS disabled. Never do this in production.")
				err = srv.ListenAndServe()
			} else if c.AllowTLSTermination != "" {
				logger.Infoln("TLS termination enabled, disabling https.")
				err = srv.ListenAndServe()
			} else {
				err = srv.ListenAndServeTLS("", "")
			}

			return err
		}, srv.Shutdown)
		logger.WithError(err).Fatal("Could not gracefully run server")
	}
}

type Handler struct {
	Clients *client.Handler
	Keys    *jwk.Handler
	OAuth2  *oauth2.Handler
	Consent *consent.Handler
	Config  *config.Config
	H       herodot.Writer
}

func (h *Handler) registerRoutes(router *httprouter.Router) {
	c := h.Config
	ctx := c.Context()

	// Set up dependencies
	injectJWKManager(c)
	clientsManager := newClientManager(c)
	injectConsentManager(c,clientsManager )

	injectFositeStore(c, clientsManager)
	oauth2Provider, idTokenKeyID := newOAuth2Provider(c)

	// Set up handlers
	h.Clients = newClientHandler(c, router, clientsManager)
	h.Keys = newJWKHandler(c, router)
	h.Consent = newConsentHandler(c, router)
	h.OAuth2 = newOAuth2Handler(c, router, ctx.ConsentManager, oauth2Provider, idTokenKeyID)
	_ = newHealthHandler(c, router)
}

func (h *Handler) rejectInsecureRequests(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc) {
	if r.TLS != nil || h.Config.ForceHTTP {
		next.ServeHTTP(rw, r)
		return
	}

	if err := h.Config.DoesRequestSatisfyTermination(r); err == nil {
		next.ServeHTTP(rw, r)
		return
	} else {
		h.Config.GetLogger().WithError(err).Warnln("Could not serve http connection")
	}

	h.H.WriteErrorCode(rw, r, http.StatusBadGateway, errors.New("Can not serve request over insecure http"))
}
<commit_msg>all: Runs gofmt<commit_after>/*
 * Copyright © 2015-2018 Aeneas Rekkas <aeneas+oss@aeneas.io>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * @author		Aeneas Rekkas <aeneas+oss@aeneas.io>
 * @copyright 	2015-2018 Aeneas Rekkas <aeneas+oss@aeneas.io>
 * @license 	Apache-2.0
 */

package server

import (
	"crypto/tls"
	"fmt"
	"net/http"
	"net/url"
	"os"

	"strconv"
	"strings"

	"github.com/gorilla/context"
	"github.com/julienschmidt/httprouter"
	"github.com/meatballhat/negroni-logrus"
	"github.com/ory/graceful"
	"github.com/ory/herodot"
	"github.com/ory/hydra/client"
	"github.com/ory/hydra/config"
	"github.com/ory/hydra/consent"
	"github.com/ory/hydra/jwk"
	"github.com/ory/hydra/oauth2"
	"github.com/ory/hydra/pkg"
	"github.com/pkg/errors"
	"github.com/rs/cors"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
	"github.com/urfave/negroni"
)

var _ = &consent.Handler{}

func parseCorsOptions() cors.Options {
	allowCredentials, _ := strconv.ParseBool(viper.GetString("CORS_ALLOWED_CREDENTIALS"))
	debug, _ := strconv.ParseBool(viper.GetString("CORS_DEBUG"))
	maxAge, _ := strconv.Atoi(viper.GetString("CORS_MAX_AGE"))
	return cors.Options{
		AllowedOrigins:   strings.Split(viper.GetString("CORS_ALLOWED_ORIGINS"), ","),
		AllowedMethods:   strings.Split(viper.GetString("CORS_ALLOWED_METHODS"), ","),
		AllowedHeaders:   strings.Split(viper.GetString("CORS_ALLOWED_HEADERS"), ","),
		ExposedHeaders:   strings.Split(viper.GetString("CORS_EXPOSED_HEADERS"), ","),
		AllowCredentials: allowCredentials,
		MaxAge:           maxAge,
		Debug:            debug,
	}
}

func RunHost(c *config.Config) func(cmd *cobra.Command, args []string) {
	return func(cmd *cobra.Command, args []string) {
		fmt.Println(banner)

		router := httprouter.New()
		logger := c.GetLogger()
		serverHandler := &Handler{
			Config: c,
			H:      herodot.NewJSONWriter(logger),
		}
		serverHandler.registerRoutes(router)
		c.ForceHTTP, _ = cmd.Flags().GetBool("dangerous-force-http")

		if !c.ForceHTTP {
			if c.Issuer == "" {
				logger.Fatalln("Issuer must be explicitly specified unless --dangerous-force-http is passed. To find out more, use `hydra help host`.")
			}
			issuer, err := url.Parse(c.Issuer)
			pkg.Must(err, "Could not parse issuer URL: %s", err)
			if issuer.Scheme != "https" {
				logger.Fatalln("Issuer must use HTTPS unless --dangerous-force-http is passed. To find out more, use `hydra help host`.")
			}
		}

		n := negroni.New()

		if ok, _ := cmd.Flags().GetBool("disable-telemetry"); !ok && os.Getenv("DISABLE_TELEMETRY") != "1" {
			metrics := c.GetMetrics()
			go metrics.RegisterSegment()
			go metrics.CommitMemoryStatistics()
			n.Use(metrics)
		}

		n.Use(negronilogrus.NewMiddlewareFromLogger(logger, c.Issuer))
		n.UseFunc(serverHandler.rejectInsecureRequests)
		n.UseHandler(router)
		corsHandler := cors.New(parseCorsOptions()).Handler(n)

		var srv = graceful.WithDefaults(&http.Server{
			Addr:    c.GetAddress(),
			Handler: context.ClearHandler(corsHandler),
			TLSConfig: &tls.Config{
				Certificates: []tls.Certificate{getOrCreateTLSCertificate(cmd, c)},
			},
		})

		err := graceful.Graceful(func() error {
			var err error
			logger.Infof("Setting up http server on %s", c.GetAddress())
			if c.ForceHTTP {
				logger.Warnln("HTTPS disabled. Never do this in production.")
				err = srv.ListenAndServe()
			} else if c.AllowTLSTermination != "" {
				logger.Infoln("TLS termination enabled, disabling https.")
				err = srv.ListenAndServe()
			} else {
				err = srv.ListenAndServeTLS("", "")
			}

			return err
		}, srv.Shutdown)
		logger.WithError(err).Fatal("Could not gracefully run server")
	}
}

type Handler struct {
	Clients *client.Handler
	Keys    *jwk.Handler
	OAuth2  *oauth2.Handler
	Consent *consent.Handler
	Config  *config.Config
	H       herodot.Writer
}

func (h *Handler) registerRoutes(router *httprouter.Router) {
	c := h.Config
	ctx := c.Context()

	// Set up dependencies
	injectJWKManager(c)
	clientsManager := newClientManager(c)
	injectConsentManager(c, clientsManager)

	injectFositeStore(c, clientsManager)
	oauth2Provider, idTokenKeyID := newOAuth2Provider(c)

	// Set up handlers
	h.Clients = newClientHandler(c, router, clientsManager)
	h.Keys = newJWKHandler(c, router)
	h.Consent = newConsentHandler(c, router)
	h.OAuth2 = newOAuth2Handler(c, router, ctx.ConsentManager, oauth2Provider, idTokenKeyID)
	_ = newHealthHandler(c, router)
}

func (h *Handler) rejectInsecureRequests(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc) {
	if r.TLS != nil || h.Config.ForceHTTP {
		next.ServeHTTP(rw, r)
		return
	}

	if err := h.Config.DoesRequestSatisfyTermination(r); err == nil {
		next.ServeHTTP(rw, r)
		return
	} else {
		h.Config.GetLogger().WithError(err).Warnln("Could not serve http connection")
	}

	h.H.WriteErrorCode(rw, r, http.StatusBadGateway, errors.New("Can not serve request over insecure http"))
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"mime"
	"net/http"

	"github.com/chop-dbhi/sql-agent"

	_ "github.com/denisenkom/go-mssqldb"
	_ "github.com/go-sql-driver/mysql"
	_ "github.com/lib/pq"
	_ "github.com/mattn/go-oci8"
	_ "github.com/mattn/go-sqlite3"
	_ "github.com/snowflakedb/gosnowflake"
)

var usage = `SQL Agent - HTTP interface

This is an HTTP interface for the SQL Agent.

Run:

	sql-agent [-host=<host>] [-port=<port>]

Example:

	POST /
	Content-Type application/json

	{
		"driver": "postgres",
		"connection": {
			"host": "pghost.org",
			"port": 5432,
		},
		"sql": "SELECT * FROM users WHERE zipcode = :zipcode",
		"parameters": {
			"zipcode": 19104
		}
	}
`

const StatusUnprocessableEntity = 422

var (
	defaultMimetype = "application/json"

	mimetypeFormats = map[string]string{
		"*/*":                  "json",
		"text/csv":             "csv",
		"application/json":     "json",
		"application/x-ldjson": "ldjson",
	}
)

// parseMimetype parses a mimetype from the Accept header.
func parseMimetype(mimetype string) string {
	mimetype, params, err := mime.ParseMediaType(mimetype)

	if err != nil {
		return ""
	}

	// No Accept header passed.
	if mimetype == "" {
		return defaultMimetype
	}

	switch mimetype {
	case "application/json":
		if params["boundary"] == "NL" {
			return "application/x-ldjson"
		}
	default:
		if _, ok := mimetypeFormats[mimetype]; !ok {
			return ""
		}
	}

	return mimetype
}

func init() {
	flag.Usage = func() {
		fmt.Println(usage)
		flag.PrintDefaults()
	}

	log.SetFlags(log.LstdFlags | log.Lshortfile)
}

func main() {
	var (
		host string
		port int
	)

	flag.StringVar(&host, "host", "localhost", "Host of the agent.")
	flag.IntVar(&port, "port", 5000, "Port of the agent.")

	flag.Parse()

	addr := fmt.Sprintf("%s:%d", host, port)
	log.Printf("* Listening on %s...\n", addr)

	http.HandleFunc("/", handleRequest)

	err := http.ListenAndServe(addr, nil)
	sqlagent.Shutdown()
	log.Fatal(err)
}

type Payload struct {
	Driver     string
	Connection map[string]interface{}
	SQL        string
	Params     map[string]interface{}
}

func handleRequest(w http.ResponseWriter, r *http.Request) {
	// Alive.
	if r.Method == "GET" || r.Method == "HEAD" {
		return
	}

	// Requires POST otherwise.
	if r.Method != "POST" {
		w.WriteHeader(http.StatusMethodNotAllowed)
		return
	}

	// Validate the Accept header and parse it to ensure it is
	// supported.
	mimetype := r.Header.Get("Accept")

	if mimetype = parseMimetype(mimetype); mimetype == "" {
		w.WriteHeader(http.StatusNotAcceptable)
		return
	}

	var payload Payload

	// Decode the body.
	if err := json.NewDecoder(r.Body).Decode(&payload); err != nil {
		w.WriteHeader(StatusUnprocessableEntity)
		w.Write([]byte(fmt.Sprintf("could not decode JSON: %s", err)))
		return
	}

	if _, ok := sqlagent.Drivers[payload.Driver]; !ok {
		w.WriteHeader(StatusUnprocessableEntity)
		w.Write([]byte(fmt.Sprintf("unknown driver: %v", payload.Driver)))
		return
	}

	db, err := sqlagent.PersistentConnect(payload.Driver, payload.Connection)
	if err != nil {
		w.WriteHeader(http.StatusServiceUnavailable)
		w.Write([]byte(fmt.Sprintf("problem connecting to database: %s", err)))
		return
	}

	iter, err := sqlagent.Execute(db, payload.SQL, payload.Params)

	if err != nil {
		w.WriteHeader(http.StatusServiceUnavailable)
		w.Write([]byte(fmt.Sprintf("error executing query: %s", err)))
		return
	}

	defer iter.Close()

	w.Header().Set("content-type", mimetype)

	switch mimetypeFormats[mimetype] {
	case "csv":
		err = sqlagent.EncodeCSV(w, iter)
	case "json":
		err = sqlagent.EncodeJSON(w, iter)
	case "ldjson":
		err = sqlagent.EncodeLDJSON(w, iter)
	}

	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		w.Write([]byte(fmt.Sprintf("error encoding data: %s", err)))
		return
	}
}
<commit_msg>Support ?ping param for connection testing<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"mime"
	"net/http"

	"github.com/chop-dbhi/sql-agent"

	_ "github.com/denisenkom/go-mssqldb"
	_ "github.com/go-sql-driver/mysql"
	_ "github.com/lib/pq"
	_ "github.com/mattn/go-oci8"
	_ "github.com/mattn/go-sqlite3"
	_ "github.com/snowflakedb/gosnowflake"
)

var usage = `SQL Agent - HTTP interface

This is an HTTP interface for the SQL Agent.

Run:

	sql-agent [-host=<host>] [-port=<port>]

Example:

	POST /
	Content-Type application/json

	{
		"driver": "postgres",
		"connection": {
			"host": "pghost.org",
			"port": 5432,
		},
		"sql": "SELECT * FROM users WHERE zipcode = :zipcode",
		"parameters": {
			"zipcode": 19104
		}
	}
`

const StatusUnprocessableEntity = 422

var (
	defaultMimetype = "application/json"

	mimetypeFormats = map[string]string{
		"*/*":                  "json",
		"text/csv":             "csv",
		"application/json":     "json",
		"application/x-ldjson": "ldjson",
	}
)

// parseMimetype parses a mimetype from the Accept header.
func parseMimetype(mimetype string) string {
	mimetype, params, err := mime.ParseMediaType(mimetype)

	if err != nil {
		return ""
	}

	// No Accept header passed.
	if mimetype == "" {
		return defaultMimetype
	}

	switch mimetype {
	case "application/json":
		if params["boundary"] == "NL" {
			return "application/x-ldjson"
		}
	default:
		if _, ok := mimetypeFormats[mimetype]; !ok {
			return ""
		}
	}

	return mimetype
}

func init() {
	flag.Usage = func() {
		fmt.Println(usage)
		flag.PrintDefaults()
	}

	log.SetFlags(log.LstdFlags | log.Lshortfile)
}

func main() {
	var (
		host string
		port int
	)

	flag.StringVar(&host, "host", "localhost", "Host of the agent.")
	flag.IntVar(&port, "port", 5000, "Port of the agent.")

	flag.Parse()

	addr := fmt.Sprintf("%s:%d", host, port)
	log.Printf("* Listening on %s...\n", addr)

	http.HandleFunc("/", handleRequest)

	err := http.ListenAndServe(addr, nil)
	sqlagent.Shutdown()
	log.Fatal(err)
}

type Payload struct {
	Driver     string
	Connection map[string]interface{}
	SQL        string
	Params     map[string]interface{}
}

func handleRequest(w http.ResponseWriter, r *http.Request) {
	// Alive.
	if r.Method == "GET" || r.Method == "HEAD" {
		return
	}

	// Requires POST otherwise.
	if r.Method != "POST" {
		w.WriteHeader(http.StatusMethodNotAllowed)
		return
	}

	// Check if ping param is set. This will only test the connection
	// and does not require a query.
	_, pingOnly := r.URL.Query()["ping"]

	mimetype := r.Header.Get("Accept")

	// Validate the Accept header and parse it to ensure it is
	// supported.
	if !pingOnly {
		if mimetype = parseMimetype(mimetype); mimetype == "" {
			w.WriteHeader(http.StatusNotAcceptable)
			return
		}
	}

	var payload Payload

	// Decode the body.
	if err := json.NewDecoder(r.Body).Decode(&payload); err != nil {
		w.WriteHeader(StatusUnprocessableEntity)
		w.Write([]byte(fmt.Sprintf("could not decode JSON: %s", err)))
		return
	}

	if _, ok := sqlagent.Drivers[payload.Driver]; !ok {
		w.WriteHeader(StatusUnprocessableEntity)
		w.Write([]byte(fmt.Sprintf("unknown driver: %v", payload.Driver)))
		return
	}

	db, err := sqlagent.PersistentConnect(payload.Driver, payload.Connection)
	if err != nil {
		w.WriteHeader(http.StatusServiceUnavailable)
		w.Write([]byte(fmt.Sprintf("problem connecting to database: %s", err)))
		return
	}

	if pingOnly {
		if err := db.Ping(); err != nil {
			w.WriteHeader(http.StatusServiceUnavailable)
			w.Write([]byte(fmt.Sprintf("problem pinging the database: %s", err)))
		}
		return
	}

	iter, err := sqlagent.Execute(db, payload.SQL, payload.Params)

	if err != nil {
		w.WriteHeader(http.StatusServiceUnavailable)
		w.Write([]byte(fmt.Sprintf("error executing query: %s", err)))
		return
	}

	defer iter.Close()

	w.Header().Set("content-type", mimetype)

	switch mimetypeFormats[mimetype] {
	case "csv":
		err = sqlagent.EncodeCSV(w, iter)
	case "json":
		err = sqlagent.EncodeJSON(w, iter)
	case "ldjson":
		err = sqlagent.EncodeLDJSON(w, iter)
	}

	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		w.Write([]byte(fmt.Sprintf("error encoding data: %s", err)))
		return
	}
}
<|endoftext|>