<commit_before>// Copyright 2014 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//+build linux

package main

// this implements /init of stage1/nspawn+systemd

import (
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"syscall"

	"github.com/coreos/rkt/Godeps/_workspace/src/github.com/appc/spec/schema/types"
	"github.com/coreos/rkt/Godeps/_workspace/src/github.com/godbus/dbus"
	"github.com/coreos/rkt/Godeps/_workspace/src/github.com/godbus/dbus/introspect"

	"github.com/coreos/rkt/common"
	"github.com/coreos/rkt/networking"
	"github.com/coreos/rkt/pkg/sys"
)

const (
	// Path to systemd-nspawn binary within the stage1 rootfs
	nspawnBin = "/usr/bin/systemd-nspawn"
	// Path to the interpreter within the stage1 rootfs
	interpBin = "/usr/lib/ld-linux-x86-64.so.2"
	// Path to the localtime file/symlink in host
	localtimePath = "/etc/localtime"
)

// mirrorLocalZoneInfo tries to reproduce the /etc/localtime target in stage1/ to satisfy systemd-nspawn
func mirrorLocalZoneInfo(root string) {
	zif, err := os.Readlink(localtimePath)
	if err != nil {
		return
	}

	// On some systems /etc/localtime is a relative symlink, make it absolute
	if !filepath.IsAbs(zif) {
		zif = filepath.Join(filepath.Dir(localtimePath), zif)
		zif = filepath.Clean(zif)
	}

	src, err := os.Open(zif)
	if err != nil {
		return
	}
	defer src.Close()

	destp := filepath.Join(common.Stage1RootfsPath(root), zif)

	if err = os.MkdirAll(filepath.Dir(destp), 0755); err != nil {
		return
	}

	dest, err := os.OpenFile(destp, os.O_CREATE|os.O_WRONLY, 0644)
	if err != nil {
		return
	}
	defer dest.Close()

	_, _ = io.Copy(dest, src)
}

var (
	debug       bool
	privNet     bool
	interactive bool
)

func init() {
	flag.BoolVar(&debug, "debug", false, "Run in debug mode")
	flag.BoolVar(&privNet, "private-net", false, "Setup private network")
	flag.BoolVar(&interactive, "interactive", false, "The pod is interactive")

	// this ensures that main runs only on main thread (thread group leader).
	// since namespace ops (unshare, setns) are done for a single thread, we
	// must ensure that the goroutine does not jump from OS thread to thread
	runtime.LockOSThread()
}

// machinedRegister checks if nspawn should register the pod to machined
func machinedRegister() bool {
	// machined has a D-Bus interface following versioning guidelines, see:
	// http://www.freedesktop.org/wiki/Software/systemd/machined/
	// Therefore we can just check if the D-Bus method we need exists and we
	// don't need to check the signature.
	var found int

	conn, err := dbus.SystemBus()
	if err != nil {
		return false
	}
	node, err := introspect.Call(conn.Object("org.freedesktop.machine1", "/org/freedesktop/machine1"))
	if err != nil {
		return false
	}
	for _, iface := range node.Interfaces {
		if iface.Name != "org.freedesktop.machine1.Manager" {
			continue
		}
		// machined v215 supports methods "RegisterMachine" and "CreateMachine" called by nspawn v215.
		// machined v216+ (since commit 5aa4bb) additionally supports methods "CreateMachineWithNetwork"
		// and "RegisterMachineWithNetwork", called by nspawn v216+.
		// TODO(alban): write checks for both versions in order to register on machined v215?
		for _, method := range iface.Methods {
			if method.Name == "CreateMachineWithNetwork" || method.Name == "RegisterMachineWithNetwork" {
				found++
			}
		}
		break
	}
	return found == 2
}

// getArgsEnv returns the nspawn args and env according to the usr used
func getArgsEnv(p *Pod, debug bool) ([]string, []string, error) {
	args := []string{}
	env := os.Environ()

	flavor, err := os.Readlink(filepath.Join(common.Stage1RootfsPath(p.Root), "flavor"))
	if err != nil {
		return nil, nil, fmt.Errorf("unable to determine stage1 flavor: %v", err)
	}

	switch flavor {
	case "coreos":
		// when running the coreos-derived stage1 with unpatched systemd-nspawn we need some ld-linux hackery
		args = append(args, filepath.Join(common.Stage1RootfsPath(p.Root), interpBin))
		args = append(args, filepath.Join(common.Stage1RootfsPath(p.Root), nspawnBin))
		args = append(args, "--boot") // Launch systemd in the pod

		// Note: the coreos flavor uses systemd-nspawn v215 but machinedRegister()
		// checks for the nspawn registration method used since v216. So we will
		// not register when the host has systemd v215.
		if machinedRegister() {
			args = append(args, fmt.Sprintf("--register=true"))
		} else {
			args = append(args, fmt.Sprintf("--register=false"))
		}

		env = append(env, "LD_PRELOAD="+filepath.Join(common.Stage1RootfsPath(p.Root), "fakesdboot.so"))
		env = append(env, "LD_LIBRARY_PATH="+filepath.Join(common.Stage1RootfsPath(p.Root), "usr/lib"))

	case "src":
		args = append(args, filepath.Join(common.Stage1RootfsPath(p.Root), nspawnBin))
		args = append(args, "--boot") // Launch systemd in the pod
		out, err := os.Getwd()
		if err != nil {
			return nil, nil, err
		}
		lfd, err := common.GetRktLockFD()
		if err != nil {
			return nil, nil, err
		}
		args = append(args, fmt.Sprintf("--pid-file=%v", filepath.Join(out, "pid")))
		args = append(args, fmt.Sprintf("--keep-fd=%v", lfd))
		if machinedRegister() {
			args = append(args, fmt.Sprintf("--register=true"))
		} else {
			args = append(args, fmt.Sprintf("--register=false"))
		}
	default:
		return nil, nil, fmt.Errorf("unrecognized stage1 flavor: %q", flavor)
	}

	if !debug {
		args = append(args, "--quiet") // silence most nspawn output (log_warning is currently not covered by this)
	}

	nsargs, err := p.PodToNspawnArgs()
	if err != nil {
		return nil, nil, fmt.Errorf("Failed to generate nspawn args: %v", err)
	}
	args = append(args, nsargs...)

	// Arguments to systemd
	args = append(args, "--")
	args = append(args, "--default-standard-output=tty") // redirect all service logs straight to tty
	if !debug {
		args = append(args, "--log-target=null") // silence systemd output inside pod
		args = append(args, "--show-status=0")   // silence systemd initialization status output
	}

	return args, env, nil
}

func withClearedCloExec(lfd int, f func() error) error {
	err := sys.CloseOnExec(lfd, false)
	if err != nil {
		return err
	}
	defer sys.CloseOnExec(lfd, true)

	return f()
}

func forwardedPorts(pod *Pod) ([]networking.ForwardedPort, error) {
	fps := []networking.ForwardedPort{}

	for _, ep := range pod.Manifest.Ports {
		n := ""
		fp := networking.ForwardedPort{}

		for an, a := range pod.Apps {
			for _, p := range a.App.Ports {
				if p.Name == ep.Name {
					if n == "" {
						fp.Protocol = p.Protocol
						fp.HostPort = ep.HostPort
						fp.PodPort = p.Port
						n = an
					} else {
						return nil, fmt.Errorf("Ambiguous exposed port in PodManifest: %q and %q both define port %q", n, an, p.Name)
					}
				}
			}
		}

		if n == "" {
			return nil, fmt.Errorf("Port name %q is not defined by any apps", ep.Name)
		}

		fps = append(fps, fp)
	}

	// TODO(eyakubovich): validate that there're no conflicts

	return fps, nil
}

func stage1() int {
	uuid, err := types.NewUUID(flag.Arg(0))
	if err != nil {
		fmt.Fprintln(os.Stderr, "UUID is missing or malformed")
		return 1
	}

	root := "."
	p, err := LoadPod(root, uuid)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Failed to load pod: %v\n", err)
		return 1
	}

	// set close-on-exec flag on RKT_LOCK_FD so it gets correctly closed when invoking
	// network plugins
	lfd, err := common.GetRktLockFD()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Failed to get rkt lock fd: %v\n", err)
		return 1
	}

	if err := sys.CloseOnExec(lfd, true); err != nil {
		fmt.Fprintf(os.Stderr, "Failed to set FD_CLOEXEC on rkt lock: %v\n", err)
		return 1
	}

	mirrorLocalZoneInfo(p.Root)

	if privNet {
		fps, err := forwardedPorts(p)
		if err != nil {
			fmt.Fprintln(os.Stderr, err.Error())
			return 6
		}

		n, err := networking.Setup(root, p.UUID, fps)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Failed to setup network: %v\n", err)
			return 6
		}
		defer n.Teardown()

		if err = n.Save(); err != nil {
			fmt.Fprintf(os.Stderr, "Failed to save networking state %v\n", err)
			return 6
		}

		p.MetadataServiceURL = common.MetadataServicePublicURL(n.GetDefaultHostIP())

		if err = registerPod(p, n.GetDefaultIP()); err != nil {
			fmt.Fprintf(os.Stderr, "Failed to register pod: %v\n", err)
			return 6
		}
		defer unregisterPod(p)
	}

	if err = p.PodToSystemd(interactive); err != nil {
		fmt.Fprintf(os.Stderr, "Failed to configure systemd: %v\n", err)
		return 2
	}

	args, env, err := getArgsEnv(p, debug)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Failed to get execution parameters: %v\n", err)
		return 3
	}

	var execFn func() error

	if privNet {
		cmd := exec.Cmd{
			Path:   args[0],
			Args:   args,
			Stdin:  os.Stdin,
			Stdout: os.Stdout,
			Stderr: os.Stderr,
			Env:    env,
		}
		execFn = cmd.Run
	} else {
		execFn = func() error {
			return syscall.Exec(args[0], args, env)
		}
	}

	err = withClearedCloExec(lfd, execFn)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Failed to execute nspawn: %v\n", err)
		return 5
	}

	return 0
}

func main() {
	flag.Parse()

	if !debug {
		log.SetOutput(ioutil.Discard)
	}

	// move code into stage1() helper so defered fns get run
	os.Exit(stage1())
}
<commit_msg>stage1/init: Use pod manifest to get forwarded ports<commit_after>// Copyright 2014 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//+build linux

package main

// this implements /init of stage1/nspawn+systemd

import (
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"syscall"

	"github.com/coreos/rkt/Godeps/_workspace/src/github.com/appc/spec/schema/types"
	"github.com/coreos/rkt/Godeps/_workspace/src/github.com/godbus/dbus"
	"github.com/coreos/rkt/Godeps/_workspace/src/github.com/godbus/dbus/introspect"

	"github.com/coreos/rkt/common"
	"github.com/coreos/rkt/networking"
	"github.com/coreos/rkt/pkg/sys"
)

const (
	// Path to systemd-nspawn binary within the stage1 rootfs
	nspawnBin = "/usr/bin/systemd-nspawn"
	// Path to the interpreter within the stage1 rootfs
	interpBin = "/usr/lib/ld-linux-x86-64.so.2"
	// Path to the localtime file/symlink in host
	localtimePath = "/etc/localtime"
)

// mirrorLocalZoneInfo tries to reproduce the /etc/localtime target in stage1/ to satisfy systemd-nspawn
func mirrorLocalZoneInfo(root string) {
	zif, err := os.Readlink(localtimePath)
	if err != nil {
		return
	}

	// On some systems /etc/localtime is a relative symlink, make it absolute
	if !filepath.IsAbs(zif) {
		zif = filepath.Join(filepath.Dir(localtimePath), zif)
		zif = filepath.Clean(zif)
	}

	src, err := os.Open(zif)
	if err != nil {
		return
	}
	defer src.Close()

	destp := filepath.Join(common.Stage1RootfsPath(root), zif)

	if err = os.MkdirAll(filepath.Dir(destp), 0755); err != nil {
		return
	}

	dest, err := os.OpenFile(destp, os.O_CREATE|os.O_WRONLY, 0644)
	if err != nil {
		return
	}
	defer dest.Close()

	_, _ = io.Copy(dest, src)
}

var (
	debug       bool
	privNet     bool
	interactive bool
)

func init() {
	flag.BoolVar(&debug, "debug", false, "Run in debug mode")
	flag.BoolVar(&privNet, "private-net", false, "Setup private network")
	flag.BoolVar(&interactive, "interactive", false, "The pod is interactive")

	// this ensures that main runs only on main thread (thread group leader).
	// since namespace ops (unshare, setns) are done for a single thread, we
	// must ensure that the goroutine does not jump from OS thread to thread
	runtime.LockOSThread()
}

// machinedRegister checks if nspawn should register the pod to machined
func machinedRegister() bool {
	// machined has a D-Bus interface following versioning guidelines, see:
	// http://www.freedesktop.org/wiki/Software/systemd/machined/
	// Therefore we can just check if the D-Bus method we need exists and we
	// don't need to check the signature.
	var found int

	conn, err := dbus.SystemBus()
	if err != nil {
		return false
	}
	node, err := introspect.Call(conn.Object("org.freedesktop.machine1", "/org/freedesktop/machine1"))
	if err != nil {
		return false
	}
	for _, iface := range node.Interfaces {
		if iface.Name != "org.freedesktop.machine1.Manager" {
			continue
		}
		// machined v215 supports methods "RegisterMachine" and "CreateMachine" called by nspawn v215.
		// machined v216+ (since commit 5aa4bb) additionally supports methods "CreateMachineWithNetwork"
		// and "RegisterMachineWithNetwork", called by nspawn v216+.
		// TODO(alban): write checks for both versions in order to register on machined v215?
		for _, method := range iface.Methods {
			if method.Name == "CreateMachineWithNetwork" || method.Name == "RegisterMachineWithNetwork" {
				found++
			}
		}
		break
	}
	return found == 2
}

// getArgsEnv returns the nspawn args and env according to the usr used
func getArgsEnv(p *Pod, debug bool) ([]string, []string, error) {
	args := []string{}
	env := os.Environ()

	flavor, err := os.Readlink(filepath.Join(common.Stage1RootfsPath(p.Root), "flavor"))
	if err != nil {
		return nil, nil, fmt.Errorf("unable to determine stage1 flavor: %v", err)
	}

	switch flavor {
	case "coreos":
		// when running the coreos-derived stage1 with unpatched systemd-nspawn we need some ld-linux hackery
		args = append(args, filepath.Join(common.Stage1RootfsPath(p.Root), interpBin))
		args = append(args, filepath.Join(common.Stage1RootfsPath(p.Root), nspawnBin))
		args = append(args, "--boot") // Launch systemd in the pod

		// Note: the coreos flavor uses systemd-nspawn v215 but machinedRegister()
		// checks for the nspawn registration method used since v216. So we will
		// not register when the host has systemd v215.
		if machinedRegister() {
			args = append(args, fmt.Sprintf("--register=true"))
		} else {
			args = append(args, fmt.Sprintf("--register=false"))
		}

		env = append(env, "LD_PRELOAD="+filepath.Join(common.Stage1RootfsPath(p.Root), "fakesdboot.so"))
		env = append(env, "LD_LIBRARY_PATH="+filepath.Join(common.Stage1RootfsPath(p.Root), "usr/lib"))

	case "src":
		args = append(args, filepath.Join(common.Stage1RootfsPath(p.Root), nspawnBin))
		args = append(args, "--boot") // Launch systemd in the pod
		out, err := os.Getwd()
		if err != nil {
			return nil, nil, err
		}
		lfd, err := common.GetRktLockFD()
		if err != nil {
			return nil, nil, err
		}
		args = append(args, fmt.Sprintf("--pid-file=%v", filepath.Join(out, "pid")))
		args = append(args, fmt.Sprintf("--keep-fd=%v", lfd))
		if machinedRegister() {
			args = append(args, fmt.Sprintf("--register=true"))
		} else {
			args = append(args, fmt.Sprintf("--register=false"))
		}
	default:
		return nil, nil, fmt.Errorf("unrecognized stage1 flavor: %q", flavor)
	}

	if !debug {
		args = append(args, "--quiet") // silence most nspawn output (log_warning is currently not covered by this)
	}

	nsargs, err := p.PodToNspawnArgs()
	if err != nil {
		return nil, nil, fmt.Errorf("Failed to generate nspawn args: %v", err)
	}
	args = append(args, nsargs...)

	// Arguments to systemd
	args = append(args, "--")
	args = append(args, "--default-standard-output=tty") // redirect all service logs straight to tty
	if !debug {
		args = append(args, "--log-target=null") // silence systemd output inside pod
		args = append(args, "--show-status=0")   // silence systemd initialization status output
	}

	return args, env, nil
}

func withClearedCloExec(lfd int, f func() error) error {
	err := sys.CloseOnExec(lfd, false)
	if err != nil {
		return err
	}
	defer sys.CloseOnExec(lfd, true)

	return f()
}

func forwardedPorts(pod *Pod) ([]networking.ForwardedPort, error) {
	fps := []networking.ForwardedPort{}

	for _, ep := range pod.Manifest.Ports {
		n := ""
		fp := networking.ForwardedPort{}

		for _, a := range pod.Manifest.Apps {
			for _, p := range a.App.Ports {
				if p.Name == ep.Name {
					if n == "" {
						fp.Protocol = p.Protocol
						fp.HostPort = ep.HostPort
						fp.PodPort = p.Port
						n = a.Name.String()
					} else {
						return nil, fmt.Errorf("Ambiguous exposed port in PodManifest: %q and %q both define port %q", n, a.Name, p.Name)
					}
				}
			}
		}

		if n == "" {
			return nil, fmt.Errorf("Port name %q is not defined by any apps", ep.Name)
		}

		fps = append(fps, fp)
	}

	// TODO(eyakubovich): validate that there're no conflicts

	return fps, nil
}

func stage1() int {
	uuid, err := types.NewUUID(flag.Arg(0))
	if err != nil {
		fmt.Fprintln(os.Stderr, "UUID is missing or malformed")
		return 1
	}

	root := "."
	p, err := LoadPod(root, uuid)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Failed to load pod: %v\n", err)
		return 1
	}

	// set close-on-exec flag on RKT_LOCK_FD so it gets correctly closed when invoking
	// network plugins
	lfd, err := common.GetRktLockFD()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Failed to get rkt lock fd: %v\n", err)
		return 1
	}

	if err := sys.CloseOnExec(lfd, true); err != nil {
		fmt.Fprintf(os.Stderr, "Failed to set FD_CLOEXEC on rkt lock: %v\n", err)
		return 1
	}

	mirrorLocalZoneInfo(p.Root)

	if privNet {
		fps, err := forwardedPorts(p)
		if err != nil {
			fmt.Fprintln(os.Stderr, err.Error())
			return 6
		}

		n, err := networking.Setup(root, p.UUID, fps)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Failed to setup network: %v\n", err)
			return 6
		}
		defer n.Teardown()

		if err = n.Save(); err != nil {
			fmt.Fprintf(os.Stderr, "Failed to save networking state %v\n", err)
			return 6
		}

		p.MetadataServiceURL = common.MetadataServicePublicURL(n.GetDefaultHostIP())

		if err = registerPod(p, n.GetDefaultIP()); err != nil {
			fmt.Fprintf(os.Stderr, "Failed to register pod: %v\n", err)
			return 6
		}
		defer unregisterPod(p)
	}

	if err = p.PodToSystemd(interactive); err != nil {
		fmt.Fprintf(os.Stderr, "Failed to configure systemd: %v\n", err)
		return 2
	}

	args, env, err := getArgsEnv(p, debug)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Failed to get execution parameters: %v\n", err)
		return 3
	}

	var execFn func() error

	if privNet {
		cmd := exec.Cmd{
			Path:   args[0],
			Args:   args,
			Stdin:  os.Stdin,
			Stdout: os.Stdout,
			Stderr: os.Stderr,
			Env:    env,
		}
		execFn = cmd.Run
	} else {
		execFn = func() error {
			return syscall.Exec(args[0], args, env)
		}
	}

	err = withClearedCloExec(lfd, execFn)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Failed to execute nspawn: %v\n", err)
		return 5
	}

	return 0
}

func main() {
	flag.Parse()

	if !debug {
		log.SetOutput(ioutil.Discard)
	}

	// move code into stage1() helper so defered fns get run
	os.Exit(stage1())
}
<|endoftext|>


<commit_before>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package backups

import (
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"

	"github.com/juju/errors"
	"github.com/juju/utils/set"

	"github.com/juju/juju/mongo"
	coreutils "github.com/juju/juju/utils"
)

// db is a surrogate for the proverbial DB layer abstraction that we
// wish we had for juju state.  To that end, the package holds the DB
// implementation-specific details and functionality needed for backups.
// Currently that means mongo-specific details.  However, as a stand-in
// for a future DB layer abstraction, the db package does not expose any
// low-level details publicly.  Thus the backups implementation remains
// oblivious to the underlying DB implementation.

var runCommand = coreutils.RunCommand

// DBConnInfo is a simplification of authentication.MongoInfo, focused
// on the needs of juju state backups.  To ensure that the info is valid
// for use in backups, use the Check() method to get the contained
// values.
type DBConnInfo struct {
	// Address is the DB system's host address.
	Address string
	// Username is used when connecting to the DB system.
	Username string
	// Password is used when connecting to the DB system.
	Password string
}

// Validate checks the DB connection info.  If it isn't valid for use in
// juju state backups, it returns an error.  Make sure that the ConnInfo
// values do not change between the time you call this method and when
// you actually need the values.
func (ci *DBConnInfo) Validate() error {
	if ci.Address == "" {
		return errors.New("missing address")
	}
	if ci.Username == "" {
		return errors.New("missing username")
	}
	if ci.Password == "" {
		return errors.New("missing password")
	}
	return nil
}

// DBInfo wraps all the DB-specific information backups needs to dump
// and restore the database.
type DBInfo struct {
	DBConnInfo
	// Targets is a list of databases to dump.
	Targets set.Strings
}

const dumpName = "mongodump"

// DBDumper is any type that dumps something to a dump dir.
type DBDumper interface {
	// Dump something to dumpDir.
	Dump(dumpDir string) error
}

var getMongodumpPath = func() (string, error) {
	mongod, err := mongo.Path()
	if err != nil {
		return "", errors.Annotate(err, "failed to get mongod path")
	}
	mongoDumpPath := filepath.Join(filepath.Dir(mongod), dumpName)

	if _, err := os.Stat(mongoDumpPath); err == nil {
		// It already exists so no need to continue.
		return mongoDumpPath, nil
	}

	path, err := exec.LookPath(dumpName)
	if err != nil {
		return "", errors.Trace(err)
	}
	return path, nil
}

type mongoDumper struct {
	DBInfo
	// binPath is the path to the dump executable.
	binPath string
}

// NewDBDumper returns a new value with a Dump method for dumping the
// juju state database.
func NewDBDumper(info DBInfo) (DBDumper, error) {
	err := info.Validate()
	if err != nil {
		return nil, errors.Trace(err)
	}

	mongodumpPath, err := getMongodumpPath()
	if err != nil {
		return nil, errors.Annotate(err, "mongodump not available")
	}

	dumper := mongoDumper{
		DBInfo:  info,
		binPath: mongodumpPath,
	}
	return &dumper, nil
}

func (md *mongoDumper) options(dumpDir string) []string {
	options := []string{
		"--ssl",
		"--authenticationDatabase", "admin",
		"--host", md.Address,
		"--username", md.Username,
		"--password", md.Password,
		"--out", dumpDir,
		"--oplog",
	}
	return options
}

func (md *mongoDumper) dump(dumpDir string) error {
	options := md.options(dumpDir)
	if err := runCommand(md.binPath, options...); err != nil {
		return errors.Annotate(err, "error dumping databases")
	}
	return nil
}

// Dump dumps the juju state-related databases.  To do this we dump all
// databases and then remove any ignored databases from the dump results.
func (md *mongoDumper) Dump(baseDumpDir string) error {
	if err := md.dump(baseDumpDir); err != nil {
		return errors.Trace(err)
	}

	found, err := listDatabases(baseDumpDir)
	if err != nil {
		return errors.Trace(err)
	}

	// Strip the ignored database from the dump dir.
	ignored := found.Difference(md.Targets)
	err = stripIgnored(ignored, baseDumpDir)
	return errors.Trace(err)
}

// stripIgnored removes the ignored DBs from the mongo dump files.
// This involves deleting DB-specific directories.
func stripIgnored(ignored set.Strings, dumpDir string) error {
	for _, dbName := range ignored.Values() {
		if dbName != "backups" {
			// We allow all ignored databases except "backups" to be
			// included in the archive file.  Restore will be
			// responsible for deleting those databases after
			// restoring them.
			continue
		}
		dirname := filepath.Join(dumpDir, dbName)
		if err := os.RemoveAll(dirname); err != nil {
			return errors.Trace(err)
		}
	}

	return nil
}

// listDatabases returns the name of each sub-directory of the dump
// directory.  Each corresponds to a database dump generated by
// mongodump.  Note that, while mongodump is unlikely to change behavior
// in this regard, this is not a documented guaranteed behavior.
func listDatabases(dumpDir string) (set.Strings, error) {
	list, err := ioutil.ReadDir(dumpDir)
	if err != nil {
		return set.Strings{}, errors.Trace(err)
	}

	databases := make(set.Strings)
	for _, info := range list {
		if !info.IsDir() {
			// Notably, oplog.bson is thus excluded here.
			continue
		}
		databases.Add(info.Name())
	}
	return databases, nil
}
<commit_msg>Drop an import alias.<commit_after>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package backups

import (
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"

	"github.com/juju/errors"
	"github.com/juju/utils/set"

	"github.com/juju/juju/mongo"
	"github.com/juju/juju/utils"
)

// db is a surrogate for the proverbial DB layer abstraction that we
// wish we had for juju state.  To that end, the package holds the DB
// implementation-specific details and functionality needed for backups.
// Currently that means mongo-specific details.  However, as a stand-in
// for a future DB layer abstraction, the db package does not expose any
// low-level details publicly.  Thus the backups implementation remains
// oblivious to the underlying DB implementation.

var runCommand = utils.RunCommand

// DBConnInfo is a simplification of authentication.MongoInfo, focused
// on the needs of juju state backups.  To ensure that the info is valid
// for use in backups, use the Check() method to get the contained
// values.
type DBConnInfo struct {
	// Address is the DB system's host address.
	Address string
	// Username is used when connecting to the DB system.
	Username string
	// Password is used when connecting to the DB system.
	Password string
}

// Validate checks the DB connection info.  If it isn't valid for use in
// juju state backups, it returns an error.  Make sure that the ConnInfo
// values do not change between the time you call this method and when
// you actually need the values.
func (ci *DBConnInfo) Validate() error {
	if ci.Address == "" {
		return errors.New("missing address")
	}
	if ci.Username == "" {
		return errors.New("missing username")
	}
	if ci.Password == "" {
		return errors.New("missing password")
	}
	return nil
}

// DBInfo wraps all the DB-specific information backups needs to dump
// and restore the database.
type DBInfo struct {
	DBConnInfo
	// Targets is a list of databases to dump.
	Targets set.Strings
}

const dumpName = "mongodump"

// DBDumper is any type that dumps something to a dump dir.
type DBDumper interface {
	// Dump something to dumpDir.
	Dump(dumpDir string) error
}

var getMongodumpPath = func() (string, error) {
	mongod, err := mongo.Path()
	if err != nil {
		return "", errors.Annotate(err, "failed to get mongod path")
	}
	mongoDumpPath := filepath.Join(filepath.Dir(mongod), dumpName)

	if _, err := os.Stat(mongoDumpPath); err == nil {
		// It already exists so no need to continue.
		return mongoDumpPath, nil
	}

	path, err := exec.LookPath(dumpName)
	if err != nil {
		return "", errors.Trace(err)
	}
	return path, nil
}

type mongoDumper struct {
	DBInfo
	// binPath is the path to the dump executable.
	binPath string
}

// NewDBDumper returns a new value with a Dump method for dumping the
// juju state database.
func NewDBDumper(info DBInfo) (DBDumper, error) {
	err := info.Validate()
	if err != nil {
		return nil, errors.Trace(err)
	}

	mongodumpPath, err := getMongodumpPath()
	if err != nil {
		return nil, errors.Annotate(err, "mongodump not available")
	}

	dumper := mongoDumper{
		DBInfo:  info,
		binPath: mongodumpPath,
	}
	return &dumper, nil
}

func (md *mongoDumper) options(dumpDir string) []string {
	options := []string{
		"--ssl",
		"--authenticationDatabase", "admin",
		"--host", md.Address,
		"--username", md.Username,
		"--password", md.Password,
		"--out", dumpDir,
		"--oplog",
	}
	return options
}

func (md *mongoDumper) dump(dumpDir string) error {
	options := md.options(dumpDir)
	if err := runCommand(md.binPath, options...); err != nil {
		return errors.Annotate(err, "error dumping databases")
	}
	return nil
}

// Dump dumps the juju state-related databases.  To do this we dump all
// databases and then remove any ignored databases from the dump results.
func (md *mongoDumper) Dump(baseDumpDir string) error {
	if err := md.dump(baseDumpDir); err != nil {
		return errors.Trace(err)
	}

	found, err := listDatabases(baseDumpDir)
	if err != nil {
		return errors.Trace(err)
	}

	// Strip the ignored database from the dump dir.
	ignored := found.Difference(md.Targets)
	err = stripIgnored(ignored, baseDumpDir)
	return errors.Trace(err)
}

// stripIgnored removes the ignored DBs from the mongo dump files.
// This involves deleting DB-specific directories.
func stripIgnored(ignored set.Strings, dumpDir string) error {
	for _, dbName := range ignored.Values() {
		if dbName != "backups" {
			// We allow all ignored databases except "backups" to be
			// included in the archive file.  Restore will be
			// responsible for deleting those databases after
			// restoring them.
			continue
		}
		dirname := filepath.Join(dumpDir, dbName)
		if err := os.RemoveAll(dirname); err != nil {
			return errors.Trace(err)
		}
	}

	return nil
}

// listDatabases returns the name of each sub-directory of the dump
// directory.  Each corresponds to a database dump generated by
// mongodump.  Note that, while mongodump is unlikely to change behavior
// in this regard, this is not a documented guaranteed behavior.
func listDatabases(dumpDir string) (set.Strings, error) {
	list, err := ioutil.ReadDir(dumpDir)
	if err != nil {
		return set.Strings{}, errors.Trace(err)
	}

	databases := make(set.Strings)
	for _, info := range list {
		if !info.IsDir() {
			// Notably, oplog.bson is thus excluded here.
			continue
		}
		databases.Add(info.Name())
	}
	return databases, nil
}
<|endoftext|>


<commit_before>package egoscale

import (
	"encoding/json"
	"net/url"
	"fmt"
)

func (exo *Client) CreateEgressRule(rule SecurityGroupRule) (*AuthorizeSecurityGroupEgressResponse, error) {

	params := url.Values{}
	params.Set("securitygroupid", rule.SecurityGroupId)
	params.Set("cidrlist", rule.Cidr)
	params.Set("protocol", rule.Protocol)

	if (rule.Protocol == "icmp") {
		params.Set("icmpcode", fmt.Sprintf("%d", rule.IcmpCode))
		params.Set("icmptype", fmt.Sprintf("%d", rule.IcmpType))
	} else {
		params.Set("startport", fmt.Sprintf("%d", rule.Port))
		params.Set("endport", fmt.Sprintf("%d", rule.Port))
	}

	resp, err := exo.Request("authorizeSecurityGroupEgress", params)
	if err != nil {
		return nil, err
	}

	var r AuthorizeSecurityGroupEgressResponse
	if err := json.Unmarshal(resp, &r); err != nil {
		return nil, err
	}

	return &r, nil
}

func (exo *Client) CreateIngressRule(rule SecurityGroupRule) (*AuthorizeSecurityGroupIngressResponse, error) {

	params := url.Values{}
	params.Set("securitygroupid", rule.SecurityGroupId)
	params.Set("cidrlist", rule.Cidr)
	params.Set("protocol", rule.Protocol)

	if (rule.Protocol == "icmp") {
		params.Set("icmpcode", fmt.Sprintf("%d", rule.IcmpCode))
		params.Set("icmptype", fmt.Sprintf("%d", rule.IcmpType))
	} else {
		params.Set("startport", fmt.Sprintf("%d", rule.Port))
		params.Set("endport", fmt.Sprintf("%d", rule.Port))
	}

	resp, err := exo.Request("authorizeSecurityGroupIngress", params)

	if err != nil {
		return nil, err
	}

	var r AuthorizeSecurityGroupIngressResponse
	if err := json.Unmarshal(resp, &r); err != nil {
		return nil, err
	}

	return &r, nil
}

func (exo *Client) CreateSecurityGroupWithRules(name string, ingress []SecurityGroupRule, egress []SecurityGroupRule) (*CreateSecurityGroupResponse, error) {

	params := url.Values{}
	params.Set("name", name)

	resp, err := exo.Request("createSecurityGroup", params)

	var r CreateSecurityGroupResponseWrapper
	if err := json.Unmarshal(resp, &r); err != nil {
		return nil, err
	}

	if err != nil {
		return nil, err
	}

	sgid := r.Wrapped.Id

	for _, erule := range(egress) {
		erule.SecurityGroupId = sgid
		_, err = exo.CreateEgressRule(erule)
		if (err != nil) {
			return nil, err
		}
	}

	for _, inrule := range(ingress) {
		inrule.SecurityGroupId = sgid
		fmt.Printf("group ID: %s\n", inrule.SecurityGroupId)
		_, err = exo.CreateIngressRule(inrule)
		if (err != nil) {
			return nil, err
		}
	}

	return &r.Wrapped, nil
}
<commit_msg>uppercase protocols<commit_after>package egoscale

import (
	"encoding/json"
	"net/url"
	"fmt"
)

func (exo *Client) CreateEgressRule(rule SecurityGroupRule) (*AuthorizeSecurityGroupEgressResponse, error) {

	params := url.Values{}
	params.Set("securitygroupid", rule.SecurityGroupId)
	params.Set("cidrlist", rule.Cidr)
	params.Set("protocol", rule.Protocol)

	if (rule.Protocol == "ICMP") {
		params.Set("icmpcode", fmt.Sprintf("%d", rule.IcmpCode))
		params.Set("icmptype", fmt.Sprintf("%d", rule.IcmpType))
	} else {
		params.Set("startport", fmt.Sprintf("%d", rule.Port))
		params.Set("endport", fmt.Sprintf("%d", rule.Port))
	}

	resp, err := exo.Request("authorizeSecurityGroupEgress", params)
	if err != nil {
		return nil, err
	}

	var r AuthorizeSecurityGroupEgressResponse
	if err := json.Unmarshal(resp, &r); err != nil {
		return nil, err
	}

	return &r, nil
}

func (exo *Client) CreateIngressRule(rule SecurityGroupRule) (*AuthorizeSecurityGroupIngressResponse, error) {

	params := url.Values{}
	params.Set("securitygroupid", rule.SecurityGroupId)
	params.Set("cidrlist", rule.Cidr)
	params.Set("protocol", rule.Protocol)

	if (rule.Protocol == "icmp") {
		params.Set("icmpcode", fmt.Sprintf("%d", rule.IcmpCode))
		params.Set("icmptype", fmt.Sprintf("%d", rule.IcmpType))
	} else {
		params.Set("startport", fmt.Sprintf("%d", rule.Port))
		params.Set("endport", fmt.Sprintf("%d", rule.Port))
	}

	resp, err := exo.Request("authorizeSecurityGroupIngress", params)

	if err != nil {
		return nil, err
	}

	var r AuthorizeSecurityGroupIngressResponse
	if err := json.Unmarshal(resp, &r); err != nil {
		return nil, err
	}

	return &r, nil
}

func (exo *Client) CreateSecurityGroupWithRules(name string, ingress []SecurityGroupRule, egress []SecurityGroupRule) (*CreateSecurityGroupResponse, error) {

	params := url.Values{}
	params.Set("name", name)

	resp, err := exo.Request("createSecurityGroup", params)

	var r CreateSecurityGroupResponseWrapper
	if err := json.Unmarshal(resp, &r); err != nil {
		return nil, err
	}

	if err != nil {
		return nil, err
	}

	sgid := r.Wrapped.Id

	for _, erule := range(egress) {
		erule.SecurityGroupId = sgid
		_, err = exo.CreateEgressRule(erule)
		if (err != nil) {
			return nil, err
		}
	}

	for _, inrule := range(ingress) {
		inrule.SecurityGroupId = sgid
		fmt.Printf("group ID: %s\n", inrule.SecurityGroupId)
		_, err = exo.CreateIngressRule(inrule)
		if (err != nil) {
			return nil, err
		}
	}

	return &r.Wrapped, nil
}
<|endoftext|>


<commit_before>package mailgun

import (
	"github.com/mbanzon/simplehttp"
	"strconv"
	"time"
)

type Bounce struct {
	CreatedAt string `json:"created_at"`
	Code      string `json:"code"`
	Address   string `json:"address"`
	Error     string `json:"error"`
}

type bounceEnvelope struct {
	TotalCount int      `json:"total_count"`
	Items      []Bounce `json:"items"`
}

type singleBounceEnvelope struct {
	Bounce Bounce `json:"bounce"`
}

func (i Bounce) GetCreatedAt() (t time.Time, err error) {
	t, err = time.Parse("Mon, 2 Jan 2006 15:04:05 MST", i.CreatedAt)
	return
}

func (m *mailgunImpl) GetBounces(limit, skip int) (int, []Bounce, error) {
	r := simplehttp.NewGetRequest(generateApiUrl(m, bouncesEndpoint))
	if limit != -1 {
		r.AddParameter("limit", strconv.Itoa(limit))
	}
	if skip != -1 {
		r.AddParameter("skip", strconv.Itoa(skip))
	}

	r.SetBasicAuth(basicAuthUser, m.ApiKey())

	var response bounceEnvelope
	err := r.MakeJSONRequest(&response)
	if err != nil {
		return -1, nil, err
	}

	return response.TotalCount, response.Items, nil
}

func (m *mailgunImpl) GetSingleBounce(address string) (Bounce, error) {
	r := simplehttp.NewGetRequest(generateApiUrl(m, bouncesEndpoint)+"/"+address)
	r.SetBasicAuth(basicAuthUser, m.ApiKey())

	var response singleBounceEnvelope
	err := r.MakeJSONRequest(&response)
	if err != nil {
		return Bounce{}, err
	}

	return response.Bounce, nil
}

func (m *mailgunImpl) AddBounce(address, code, error string) error {
	r := simplehttp.NewPostRequest(generateApiUrl(m, bouncesEndpoint))

	r.AddFormValue("address", address)
	if code != "" {
		r.AddFormValue("code", code)
	}
	if error != "" {
		r.AddFormValue("error", error)
	}
	r.SetBasicAuth(basicAuthUser, m.ApiKey())
	_, err := r.MakeRequest()
	return err
}

func (m *mailgunImpl) DeleteBounce(address string) error {
	r := simplehttp.NewDeleteRequest(generateApiUrl(m, bouncesEndpoint)+"/"+address)
	r.SetBasicAuth(basicAuthUser, m.ApiKey())
	_, err := r.MakeRequest()
	return err
}
<commit_msg>Gofmt.<commit_after>package mailgun

import (
	"github.com/mbanzon/simplehttp"
	"strconv"
	"time"
)

type Bounce struct {
	CreatedAt string `json:"created_at"`
	Code      string `json:"code"`
	Address   string `json:"address"`
	Error     string `json:"error"`
}

type bounceEnvelope struct {
	TotalCount int      `json:"total_count"`
	Items      []Bounce `json:"items"`
}

type singleBounceEnvelope struct {
	Bounce Bounce `json:"bounce"`
}

func (i Bounce) GetCreatedAt() (t time.Time, err error) {
	t, err = time.Parse("Mon, 2 Jan 2006 15:04:05 MST", i.CreatedAt)
	return
}

func (m *mailgunImpl) GetBounces(limit, skip int) (int, []Bounce, error) {
	r := simplehttp.NewGetRequest(generateApiUrl(m, bouncesEndpoint))
	if limit != -1 {
		r.AddParameter("limit", strconv.Itoa(limit))
	}
	if skip != -1 {
		r.AddParameter("skip", strconv.Itoa(skip))
	}

	r.SetBasicAuth(basicAuthUser, m.ApiKey())

	var response bounceEnvelope
	err := r.MakeJSONRequest(&response)
	if err != nil {
		return -1, nil, err
	}

	return response.TotalCount, response.Items, nil
}

func (m *mailgunImpl) GetSingleBounce(address string) (Bounce, error) {
	r := simplehttp.NewGetRequest(generateApiUrl(m, bouncesEndpoint) + "/" + address)
	r.SetBasicAuth(basicAuthUser, m.ApiKey())

	var response singleBounceEnvelope
	err := r.MakeJSONRequest(&response)
	if err != nil {
		return Bounce{}, err
	}

	return response.Bounce, nil
}

func (m *mailgunImpl) AddBounce(address, code, error string) error {
	r := simplehttp.NewPostRequest(generateApiUrl(m, bouncesEndpoint))

	r.AddFormValue("address", address)
	if code != "" {
		r.AddFormValue("code", code)
	}
	if error != "" {
		r.AddFormValue("error", error)
	}
	r.SetBasicAuth(basicAuthUser, m.ApiKey())
	_, err := r.MakeRequest()
	return err
}

func (m *mailgunImpl) DeleteBounce(address string) error {
	r := simplehttp.NewDeleteRequest(generateApiUrl(m, bouncesEndpoint) + "/" + address)
	r.SetBasicAuth(basicAuthUser, m.ApiKey())
	_, err := r.MakeRequest()
	return err
}
<|endoftext|>


<commit_before>package fire

import (
	"net/http"
	"time"

	"github.com/gin-gonic/gin"
	"github.com/ory-am/fosite"
	"github.com/ory-am/fosite/handler/core"
	"github.com/ory-am/fosite/handler/core/client"
	"github.com/ory-am/fosite/handler/core/implicit"
	"github.com/ory-am/fosite/handler/core/owner"
	"github.com/ory-am/fosite/handler/core/strategy"
	"github.com/ory-am/fosite/token/hmac"
	"gopkg.in/mgo.v2"
)

var callbackTemplate = []byte(`<!DOCTYPE html>
<html>
  <head>
    <title>Authorize</title>
    <script>
      window.opener.fireCallback(window.location.hash);
      window.close();
    </script>
  </head>
</html>`)

// AccessToken is the internal model used to stored access tokens. The model
// can be mounted as a fire Resource to become manageable via the API.
type AccessToken struct {
	Base          `bson:",inline" fire:"access-token:access-tokens:access_tokens"`
	Signature     string    `json:"-" valid:"required"`
	RequestedAt   time.Time `json:"requested-at" valid:"required" bson:"requested_at"`
	GrantedScopes []string  `json:"granted-scopes" valid:"required" bson:"granted_scopes"`
}

var accessTokenModel *AccessToken

func init() {
	accessTokenModel = &AccessToken{}
	Init(accessTokenModel)
}

// A Authenticator provides OAuth2 based authentication. The implementation
// currently supports the Resource Owner Credentials, Client Credentials and
// Implicit Grant flows. The flows can be enabled using their respective methods.
type Authenticator struct {
	storage *authenticatorStorage

	strategy     *strategy.HMACSHAStrategy
	handleHelper *core.HandleHelper
	fosite       *fosite.Fosite
}

// NetAuthenticator creates and returns a new Authenticator.
func NewAuthenticator(db *mgo.Database, ownerModel, clientModel Model, secret string) *Authenticator {
	// initialize models
	Init(ownerModel)
	Init(clientModel)

	// extract attributes from owner
	ownerIdentifiable := ownerModel.getBase().attributesByTag("identifiable")
	if len(ownerIdentifiable) != 1 {
		panic("expected to find exactly one 'identifiable' tag on model")
	}
	ownerVerifiable := ownerModel.getBase().attributesByTag("verifiable")
	if len(ownerVerifiable) != 1 {
		panic("expected to find exactly one 'verifiable' tag on model")
	}

	// extract attributes from client
	clientIdentifiable := clientModel.getBase().attributesByTag("identifiable")
	if len(clientIdentifiable) != 1 {
		panic("expected to find exactly one 'identifiable' tag on model")
	}
	clientVerifiable := clientModel.getBase().attributesByTag("verifiable")
	if len(clientVerifiable) != 1 {
		panic("expected to find exactly one 'verifiable' tag on model")
	}
	clientCallable := clientModel.getBase().attributesByTag("callable")
	if len(clientCallable) != 1 {
		panic("expected to find exactly one 'callable' tag on model")
	}

	// create storage
	s := &authenticatorStorage{
		db:                 db,
		ownerModel:         ownerModel,
		ownerIDAttr:        ownerIdentifiable[0],
		ownerSecretAttr:    ownerVerifiable[0],
		clientModel:        clientModel,
		clientIDAttr:       clientIdentifiable[0],
		clientSecretAttr:   clientVerifiable[0],
		clientCallableAttr: clientCallable[0],
	}

	// set the default token lifespan to one hour
	tokenLifespan := time.Hour

	// create a new token generation strategy
	strategy := &strategy.HMACSHAStrategy{
		Enigma: &hmac.HMACStrategy{
			GlobalSecret: []byte(secret),
		},
		AccessTokenLifespan:   tokenLifespan,
		AuthorizeCodeLifespan: tokenLifespan,
	}

	// instantiate a new fosite instance
	f := fosite.NewFosite(s)

	// set mandatory scope
	f.MandatoryScope = "fire"

	// this little helper is used by some of the handlers later
	handleHelper := &core.HandleHelper{
		AccessTokenStrategy: strategy,
		AccessTokenStorage:  s,
		AccessTokenLifespan: tokenLifespan,
	}

	// add a request validator for access tokens to fosite
	f.AuthorizedRequestValidators.Append(&core.CoreValidator{
		AccessTokenStrategy: strategy,
		AccessTokenStorage:  s,
	})

	return &Authenticator{
		storage:      s,
		fosite:       f,
		handleHelper: handleHelper,
		strategy:     strategy,
	}
}

// EnablePasswordGrant enables the usage of the OAuth 2.0 Resource Owner Password
// Credentials Grant.
//
// Note: This method should only be called once.
func (a *Authenticator) EnablePasswordGrant() {
	// create handler
	passwordHandler := &owner.ResourceOwnerPasswordCredentialsGrantHandler{
		HandleHelper:                                 a.handleHelper,
		ResourceOwnerPasswordCredentialsGrantStorage: a.storage,
	}

	// add handler to fosite
	a.fosite.TokenEndpointHandlers.Append(passwordHandler)
}

// EnableCredentialsGrant enables the usage of the OAuth 2.0 Client Credentials Grant.
//
// Note: This method should only be called once.
func (a *Authenticator) EnableCredentialsGrant() {
	// create handler
	credentialsHandler := &client.ClientCredentialsGrantHandler{
		HandleHelper: a.handleHelper,
	}

	// add handler to fosite
	a.fosite.TokenEndpointHandlers.Append(credentialsHandler)
}

// EnableImplicitGrant enables the usage of the OAuth 2.0 Implicit Grant.
//
// Note: This method should only be called once.
func (a *Authenticator) EnableImplicitGrant() {
	// create handler
	implicitHandler := &implicit.AuthorizeImplicitGrantTypeHandler{
		AccessTokenStrategy: a.handleHelper.AccessTokenStrategy,
		AccessTokenStorage:  a.handleHelper.AccessTokenStorage,
		AccessTokenLifespan: a.handleHelper.AccessTokenLifespan,
	}

	// add handler to fosite
	a.fosite.AuthorizeEndpointHandlers.Append(implicitHandler)
}

// HashPassword returns an Authenticator compatible hash of the password.
func (a *Authenticator) HashPassword(password string) ([]byte, error) {
	return a.fosite.Hasher.Hash([]byte(password))
}

// MustHashPassword is the same as HashPassword except that it raises and error
// when the hashing failed.
func (a *Authenticator) MustHashPassword(password string) []byte {
	bytes, err := a.HashPassword(password)
	if err != nil {
		panic(err)
	}

	return bytes
}

// Register will create all necessary routes on the passed router. If want to
// prefix the auth endpoint (e.g. /auth/) you need to pass it to Register.
//
// Note: This functions should only be called once after enabling all flows.
func (a *Authenticator) Register(prefix string, router gin.IRouter) {
	router.POST(prefix+"/token", a.tokenEndpoint)
	router.POST(prefix+"/authorize", a.authorizeEndpoint)
	router.GET(prefix+"/callback", a.callbackEndpoint)

	// TODO: Redirect to auxiliary Login form.
}

// Authorizer returns a callback that can be used to protect resources by requiring
// an access tokens with the provides scopes to be granted.
func (a *Authenticator) Authorizer() Callback {
	// TODO: Add scopes.

	return func(ctx *Context) (error, error) {
		// prepare fosite
		f := fosite.NewContext()
		session := &strategy.HMACSession{}

		// validate request
		_, err := a.fosite.ValidateRequestAuthorization(f, ctx.GinContext.Request, session, "fire")
		if err != nil {
			return err, nil
		}

		return nil, nil
	}
}

func (a *Authenticator) tokenEndpoint(ctx *gin.Context) {
	// create new context
	f := fosite.NewContext()

	// create new session
	s := &strategy.HMACSession{}

	// obtain access request
	req, err := a.fosite.NewAccessRequest(f, ctx.Request, s)
	if err != nil {
		ctx.Error(err)
		a.fosite.WriteAccessError(ctx.Writer, req, err)
		return
	}

	// grant the mandatory scope
	if req.GetScopes().Has("fire") {
		req.GrantScope("fire")
	}

	// obtain access response
	res, err := a.fosite.NewAccessResponse(f, ctx.Request, req)
	if err != nil {
		ctx.Error(err)
		a.fosite.WriteAccessError(ctx.Writer, req, err)
		return
	}

	// write response
	a.fosite.WriteAccessResponse(ctx.Writer, req, res)
}

func (a *Authenticator) authorizeEndpoint(ctx *gin.Context) {
	// create new context
	f := fosite.NewContext()

	// obtain authorize request
	req, err := a.fosite.NewAuthorizeRequest(f, ctx.Request)
	if err != nil {
		ctx.Error(err)
		a.fosite.WriteAuthorizeError(ctx.Writer, req, err)
		return
	}

	// get credentials
	username := ctx.Request.Form.Get("username")
	password := ctx.Request.Form.Get("password")

	// authenticate user
	err = a.storage.Authenticate(f, username, password)
	if err != nil {
		uri := ctx.Request.Referer() + "&error=invalid_credentials"
		ctx.Redirect(http.StatusTemporaryRedirect, uri)
		return
	}

	// create new session
	s := &strategy.HMACSession{}

	// obtain authorize response
	res, err := a.fosite.NewAuthorizeResponse(ctx, ctx.Request, req, s)
	if err != nil {
		ctx.Error(err)
		a.fosite.WriteAuthorizeError(ctx.Writer, req, err)
		return
	}

	// write response
	a.fosite.WriteAuthorizeResponse(ctx.Writer, req, res)
}

func (a *Authenticator) callbackEndpoint(ctx *gin.Context) {
	// TODO: Remove callback endpoint.

	ctx.Writer.Write(callbackTemplate)
}
<commit_msg>removed specific callback endpoint<commit_after>package fire

import (
	"net/http"
	"time"

	"github.com/gin-gonic/gin"
	"github.com/ory-am/fosite"
	"github.com/ory-am/fosite/handler/core"
	"github.com/ory-am/fosite/handler/core/client"
	"github.com/ory-am/fosite/handler/core/implicit"
	"github.com/ory-am/fosite/handler/core/owner"
	"github.com/ory-am/fosite/handler/core/strategy"
	"github.com/ory-am/fosite/token/hmac"
	"gopkg.in/mgo.v2"
)

// AccessToken is the internal model used to stored access tokens. The model
// can be mounted as a fire Resource to become manageable via the API.
type AccessToken struct {
	Base          `bson:",inline" fire:"access-token:access-tokens:access_tokens"`
	Signature     string    `json:"-" valid:"required"`
	RequestedAt   time.Time `json:"requested-at" valid:"required" bson:"requested_at"`
	GrantedScopes []string  `json:"granted-scopes" valid:"required" bson:"granted_scopes"`
}

var accessTokenModel *AccessToken

func init() {
	accessTokenModel = &AccessToken{}
	Init(accessTokenModel)
}

// A Authenticator provides OAuth2 based authentication. The implementation
// currently supports the Resource Owner Credentials, Client Credentials and
// Implicit Grant flows. The flows can be enabled using their respective methods.
type Authenticator struct {
	storage *authenticatorStorage

	strategy     *strategy.HMACSHAStrategy
	handleHelper *core.HandleHelper
	fosite       *fosite.Fosite
}

// NetAuthenticator creates and returns a new Authenticator.
func NewAuthenticator(db *mgo.Database, ownerModel, clientModel Model, secret string) *Authenticator {
	// initialize models
	Init(ownerModel)
	Init(clientModel)

	// extract attributes from owner
	ownerIdentifiable := ownerModel.getBase().attributesByTag("identifiable")
	if len(ownerIdentifiable) != 1 {
		panic("expected to find exactly one 'identifiable' tag on model")
	}
	ownerVerifiable := ownerModel.getBase().attributesByTag("verifiable")
	if len(ownerVerifiable) != 1 {
		panic("expected to find exactly one 'verifiable' tag on model")
	}

	// extract attributes from client
	clientIdentifiable := clientModel.getBase().attributesByTag("identifiable")
	if len(clientIdentifiable) != 1 {
		panic("expected to find exactly one 'identifiable' tag on model")
	}
	clientVerifiable := clientModel.getBase().attributesByTag("verifiable")
	if len(clientVerifiable) != 1 {
		panic("expected to find exactly one 'verifiable' tag on model")
	}
	clientCallable := clientModel.getBase().attributesByTag("callable")
	if len(clientCallable) != 1 {
		panic("expected to find exactly one 'callable' tag on model")
	}

	// create storage
	s := &authenticatorStorage{
		db:                 db,
		ownerModel:         ownerModel,
		ownerIDAttr:        ownerIdentifiable[0],
		ownerSecretAttr:    ownerVerifiable[0],
		clientModel:        clientModel,
		clientIDAttr:       clientIdentifiable[0],
		clientSecretAttr:   clientVerifiable[0],
		clientCallableAttr: clientCallable[0],
	}

	// set the default token lifespan to one hour
	tokenLifespan := time.Hour

	// create a new token generation strategy
	strategy := &strategy.HMACSHAStrategy{
		Enigma: &hmac.HMACStrategy{
			GlobalSecret: []byte(secret),
		},
		AccessTokenLifespan:   tokenLifespan,
		AuthorizeCodeLifespan: tokenLifespan,
	}

	// instantiate a new fosite instance
	f := fosite.NewFosite(s)

	// set mandatory scope
	f.MandatoryScope = "fire"

	// this little helper is used by some of the handlers later
	handleHelper := &core.HandleHelper{
		AccessTokenStrategy: strategy,
		AccessTokenStorage:  s,
		AccessTokenLifespan: tokenLifespan,
	}

	// add a request validator for access tokens to fosite
	f.AuthorizedRequestValidators.Append(&core.CoreValidator{
		AccessTokenStrategy: strategy,
		AccessTokenStorage:  s,
	})

	return &Authenticator{
		storage:      s,
		fosite:       f,
		handleHelper: handleHelper,
		strategy:     strategy,
	}
}

// EnablePasswordGrant enables the usage of the OAuth 2.0 Resource Owner Password
// Credentials Grant.
//
// Note: This method should only be called once.
func (a *Authenticator) EnablePasswordGrant() {
	// create handler
	passwordHandler := &owner.ResourceOwnerPasswordCredentialsGrantHandler{
		HandleHelper:                                 a.handleHelper,
		ResourceOwnerPasswordCredentialsGrantStorage: a.storage,
	}

	// add handler to fosite
	a.fosite.TokenEndpointHandlers.Append(passwordHandler)
}

// EnableCredentialsGrant enables the usage of the OAuth 2.0 Client Credentials Grant.
//
// Note: This method should only be called once.
func (a *Authenticator) EnableCredentialsGrant() {
	// create handler
	credentialsHandler := &client.ClientCredentialsGrantHandler{
		HandleHelper: a.handleHelper,
	}

	// add handler to fosite
	a.fosite.TokenEndpointHandlers.Append(credentialsHandler)
}

// EnableImplicitGrant enables the usage of the OAuth 2.0 Implicit Grant.
//
// Note: This method should only be called once.
func (a *Authenticator) EnableImplicitGrant() {
	// create handler
	implicitHandler := &implicit.AuthorizeImplicitGrantTypeHandler{
		AccessTokenStrategy: a.handleHelper.AccessTokenStrategy,
		AccessTokenStorage:  a.handleHelper.AccessTokenStorage,
		AccessTokenLifespan: a.handleHelper.AccessTokenLifespan,
	}

	// add handler to fosite
	a.fosite.AuthorizeEndpointHandlers.Append(implicitHandler)
}

// HashPassword returns an Authenticator compatible hash of the password.
func (a *Authenticator) HashPassword(password string) ([]byte, error) {
	return a.fosite.Hasher.Hash([]byte(password))
}

// MustHashPassword is the same as HashPassword except that it raises and error
// when the hashing failed.
func (a *Authenticator) MustHashPassword(password string) []byte {
	bytes, err := a.HashPassword(password)
	if err != nil {
		panic(err)
	}

	return bytes
}

// Register will create all necessary routes on the passed router. If want to
// prefix the auth endpoint (e.g. /auth/) you need to pass it to Register.
//
// Note: This functions should only be called once after enabling all flows.
func (a *Authenticator) Register(prefix string, router gin.IRouter) {
	router.POST(prefix+"/token", a.tokenEndpoint)
	router.POST(prefix+"/authorize", a.authorizeEndpoint)

	// TODO: Redirect to auxiliary Login form.
}

// Authorizer returns a callback that can be used to protect resources by requiring
// an access tokens with the provides scopes to be granted.
func (a *Authenticator) Authorizer() Callback {
	// TODO: Add scopes.

	return func(ctx *Context) (error, error) {
		// prepare fosite
		f := fosite.NewContext()
		session := &strategy.HMACSession{}

		// validate request
		_, err := a.fosite.ValidateRequestAuthorization(f, ctx.GinContext.Request, session, "fire")
		if err != nil {
			return err, nil
		}

		return nil, nil
	}
}

func (a *Authenticator) tokenEndpoint(ctx *gin.Context) {
	// create new context
	f := fosite.NewContext()

	// create new session
	s := &strategy.HMACSession{}

	// obtain access request
	req, err := a.fosite.NewAccessRequest(f, ctx.Request, s)
	if err != nil {
		ctx.Error(err)
		a.fosite.WriteAccessError(ctx.Writer, req, err)
		return
	}

	// grant the mandatory scope
	if req.GetScopes().Has("fire") {
		req.GrantScope("fire")
	}

	// obtain access response
	res, err := a.fosite.NewAccessResponse(f, ctx.Request, req)
	if err != nil {
		ctx.Error(err)
		a.fosite.WriteAccessError(ctx.Writer, req, err)
		return
	}

	// write response
	a.fosite.WriteAccessResponse(ctx.Writer, req, res)
}

func (a *Authenticator) authorizeEndpoint(ctx *gin.Context) {
	// create new context
	f := fosite.NewContext()

	// obtain authorize request
	req, err := a.fosite.NewAuthorizeRequest(f, ctx.Request)
	if err != nil {
		ctx.Error(err)
		a.fosite.WriteAuthorizeError(ctx.Writer, req, err)
		return
	}

	// get credentials
	username := ctx.Request.Form.Get("username")
	password := ctx.Request.Form.Get("password")

	// authenticate user
	err = a.storage.Authenticate(f, username, password)
	if err != nil {
		uri := ctx.Request.Referer() + "&error=invalid_credentials"
		ctx.Redirect(http.StatusTemporaryRedirect, uri)
		return
	}

	// create new session
	s := &strategy.HMACSession{}

	// obtain authorize response
	res, err := a.fosite.NewAuthorizeResponse(ctx, ctx.Request, req, s)
	if err != nil {
		ctx.Error(err)
		a.fosite.WriteAuthorizeError(ctx.Writer, req, err)
		return
	}

	// write response
	a.fosite.WriteAuthorizeResponse(ctx.Writer, req, res)
}
<|endoftext|>


<commit_before>package main

import (
	"code.google.com/p/go.net/websocket"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"strings"
)

const execname = "ditaa"

var addr string

// Send the list of files in the current directory
func listing(ws *websocket.Conn) {
	entries, err := ioutil.ReadDir(".")
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	files := make([]string, len(entries))
	for i, f := range entries {
		files[i] = f.Name()
	}
	err = websocket.JSON.Send(ws, files)
	if err != nil {
		fmt.Println(err)
	}
}

func notify(ws *websocket.Conn) {
}

func image(w http.ResponseWriter, filename string) {
	fmt.Fprintf(os.Stderr, "image: %s\n", filename)
	// TODO test if filename exists and returns a 404 if it's not the case

	tmpfile, err := ioutil.TempFile("", execname)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	tmpname := tmpfile.Name()
	defer os.Remove(tmpname)
	defer tmpfile.Close()

	cmd := exec.Command(execname, filename, "-o", tmpname)
	err = cmd.Run()
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	tmpfile.Close()
	tmpfile, err = os.Open(tmpname)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	io.Copy(w, tmpfile)
}

func page(w http.ResponseWriter, filename string) {
	fmt.Fprintf(w, `<!DOCTYPE html>

  <meta charset="utf-8">
  <title>%s</title>

%s

`, filename, filename, filename)
}

func index(w http.ResponseWriter) {
	fmt.Fprintf(w, `<!DOCTYPE html>

  <meta charset="utf-8">
  <title>ditaa-live</title>

Ditaa-live

`, addr)
}

func dispatch(w http.ResponseWriter, r *http.Request) {
	png := "/png/"
	html := "/html/"
	path := r.URL.Path
	switch {
	case path == "/":
		index(w)
	case strings.HasPrefix(path, html):
		filename := r.URL.Path[len(html):]
		page(w, filename)
	case strings.HasPrefix(path, png):
		filename := r.URL.Path[len(png):]
		image(w, filename)
	default:
		http.NotFound(w, r)
	}
}

func main() {
	flag.StringVar(&addr, "addr", "127.0.0.1:4444", "Bind to this address:port")
	flag.Parse()
	args := flag.Args()
	if len(args) > 0 {
		os.Chdir(args[0])
	}
	fmt.Printf("Listening on http://%s/\n", addr)

	http.Handle("/notify", websocket.Handler(notify))
	http.Handle("/ls", websocket.Handler(listing))
	http.HandleFunc("/", dispatch)
	err := http.ListenAndServe(addr, nil)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Can't listen on %s\n", addr)
		os.Exit(1)
	}
}
<commit_msg>Use inotify to check new version of ditaa files and reload them<commit_after>package main

import (
	"code.google.com/p/go.net/websocket"
	"exp/inotify"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"strings"
	"time"
)

const execname = "ditaa"

var addr string

func listing(ws *websocket.Conn) {
	defer ws.Close()
	entries, err := ioutil.ReadDir(".")
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	files := make([]string, len(entries))
	for i, f := range entries {
		files[i] = f.Name()
	}
	err = websocket.JSON.Send(ws, files)
	if err != nil {
		fmt.Println(err)
	}
}

func notify(ws *websocket.Conn) {
	var filename string
	defer ws.Close()
	err := websocket.JSON.Receive(ws, &filename)
	if err != nil {
		fmt.Println(err)
		return
	}

	watcher, err := inotify.NewWatcher()
	if err != nil {
		fmt.Println(err)
		return
	}
	defer watcher.Close()

	for {
		time.Sleep(10 * time.Millisecond)
		err = watcher.AddWatch(filename, inotify.IN_MODIFY)
		if err != nil {
			fmt.Println(err)
			return
		}

		select {
		case ev := <-watcher.Event:
			watcher.RemoveWatch(filename)
			err = websocket.JSON.Send(ws, ev)
			if err != nil {
				fmt.Println(err)
				return
			}
		case err = <-watcher.Error:
			fmt.Println(err)
			return
		}
	}
}

func image(w http.ResponseWriter, filename string) {
	_, err := os.Lstat(filename)
	if err != nil {
		notfound := fmt.Sprintf("Error 404: %s has not been found!\n", filename)
		http.Error(w, notfound, 404)
		return
	}

	tmpfile, err := ioutil.TempFile("", execname)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	tmpname := tmpfile.Name()
	defer os.Remove(tmpname)
	defer tmpfile.Close()

	cmd := exec.Command(execname, filename, "-o", tmpname)
	err = cmd.Run()
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	tmpfile.Close()
	tmpfile, err = os.Open(tmpname)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	io.Copy(w, tmpfile)
}

func page(w http.ResponseWriter, filename string) {
	fmt.Fprintf(w, `<!DOCTYPE html>

  <meta charset="utf-8">
  <title>%s</title>

%s

`, filename, filename, filename, addr)
}

func index(w http.ResponseWriter) {
	fmt.Fprintf(w, `<!DOCTYPE html>

  <meta charset="utf-8">
  <title>ditaa-live</title>

Ditaa-live

`, addr)
}

func dispatch(w http.ResponseWriter, r *http.Request) {
	png := "/png/"
	html := "/html/"
	path := r.URL.Path
	switch {
	case path == "/":
		index(w)
	case strings.HasPrefix(path, html):
		filename := r.URL.Path[len(html):]
		page(w, filename)
	case strings.HasPrefix(path, png):
		filename := r.URL.Path[len(png):]
		image(w, filename)
	default:
		http.NotFound(w, r)
	}
}

func main() {
	flag.StringVar(&addr, "addr", "127.0.0.1:4444", "Bind to this address:port")
	flag.Parse()
	args := flag.Args()
	if len(args) > 0 {
		os.Chdir(args[0])
	}
	fmt.Printf("Listening on http://%s/\n", addr)

	http.Handle("/notify", websocket.Handler(notify))
	http.Handle("/ls", websocket.Handler(listing))
	http.HandleFunc("/", dispatch)
	err := http.ListenAndServe(addr, nil)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Can't listen on %s\n", addr)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package filesys

import (
	"bytes"
	"context"
	"fmt"
	"sync"
	"sync/atomic"
	"time"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/operation"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/security"
)

type ContinuousDirtyPages struct {
	hasData bool
	Offset  int64
	Size    int64
	Data    []byte
	f       *File
	lock    sync.Mutex
}

func newDirtyPages(file *File) *ContinuousDirtyPages {
	return &ContinuousDirtyPages{
		Data: nil,
		f:    file,
	}
}

func (pages *ContinuousDirtyPages) releaseResource() {
	if pages.Data != nil {
		pages.f.wfs.bufPool.Put(pages.Data)
		pages.Data = nil
		atomic.AddInt32(&counter, -1)
		glog.V(3).Infof("%s/%s releasing resource %d", pages.f.dir.Path, pages.f.Name, counter)
	}
}

var counter = int32(0)

func (pages *ContinuousDirtyPages) AddPage(ctx context.Context, offset int64, data []byte) (chunks []*filer_pb.FileChunk, err error) {

	pages.lock.Lock()
	defer pages.lock.Unlock()

	var chunk *filer_pb.FileChunk

	if len(data) > int(pages.f.wfs.option.ChunkSizeLimit) {
		// this is more than what buffer can hold.
		return pages.flushAndSave(ctx, offset, data)
	}

	if pages.Data == nil {
		pages.Data = pages.f.wfs.bufPool.Get().([]byte)
		atomic.AddInt32(&counter, 1)
		glog.V(3).Infof("%s/%s acquire resource %d", pages.f.dir.Path, pages.f.Name, counter)
	}

	if offset < pages.Offset || offset >= pages.Offset+int64(len(pages.Data)) ||
		pages.Offset+int64(len(pages.Data)) < offset+int64(len(data)) {
		// if the data is out of range,
		// or buffer is full if adding new data,
		// flush current buffer and add new data

		// println("offset", offset, "size", len(data), "existing offset", pages.Offset, "size", pages.Size)

		if chunk, err = pages.saveExistingPagesToStorage(ctx); err == nil {
			if chunk != nil {
				glog.V(4).Infof("%s/%s add save [%d,%d)", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size))
				chunks = append(chunks, chunk)
			}
		} else {
			glog.V(0).Infof("%s/%s add save [%d,%d): %v", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size), err)
			return
		}
		pages.Offset = offset
		copy(pages.Data, data)
		pages.Size = int64(len(data))
		return
	}

	if offset != pages.Offset+pages.Size {
		// when this happens, debug shows the data overlapping with existing data is empty
		// the data is not just append
		if offset == pages.Offset && int(pages.Size) < len(data) {
			// glog.V(2).Infof("pages[%d,%d) pages.Data len=%v, data len=%d, pages.Size=%d", pages.Offset, pages.Offset+pages.Size, len(pages.Data), len(data), pages.Size)
			copy(pages.Data[pages.Size:], data[pages.Size:])
		} else {
			if pages.Size != 0 {
				glog.V(1).Infof("%s/%s add page: pages [%d, %d) write [%d, %d)", pages.f.dir.Path, pages.f.Name, pages.Offset, pages.Offset+pages.Size, offset, offset+int64(len(data)))
			}
			return pages.flushAndSave(ctx, offset, data)
		}
	} else {
		copy(pages.Data[offset-pages.Offset:], data)
	}

	pages.Size = max(pages.Size, offset+int64(len(data))-pages.Offset)

	return
}

func (pages *ContinuousDirtyPages) flushAndSave(ctx context.Context, offset int64, data []byte) (chunks []*filer_pb.FileChunk, err error) {

	var chunk *filer_pb.FileChunk

	// flush existing
	if chunk, err = pages.saveExistingPagesToStorage(ctx); err == nil {
		if chunk != nil {
			glog.V(4).Infof("%s/%s flush existing [%d,%d)", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size))
			chunks = append(chunks, chunk)
		}
	} else {
		glog.V(0).Infof("%s/%s failed to flush1 [%d,%d): %v", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size), err)
		return
	}
	pages.Size = 0
	pages.Offset = 0

	// flush the new page
	if chunk, err = pages.saveToStorage(ctx, data, offset); err == nil {
		if chunk != nil {
			glog.V(4).Infof("%s/%s flush big request [%d,%d)", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size))
			chunks = append(chunks, chunk)
		}
	} else {
		glog.V(0).Infof("%s/%s failed to flush2 [%d,%d): %v", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size), err)
		return
	}

	return
}

func (pages *ContinuousDirtyPages) FlushToStorage(ctx context.Context) (chunk *filer_pb.FileChunk, err error) {

	pages.lock.Lock()
	defer pages.lock.Unlock()

	if pages.Size == 0 {
		return nil, nil
	}

	if chunk, err = pages.saveExistingPagesToStorage(ctx); err == nil {
		pages.Size = 0
		pages.Offset = 0
		if chunk != nil {
			glog.V(4).Infof("%s/%s flush [%d,%d)", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size))
		}
	}
	return
}

func (pages *ContinuousDirtyPages) saveExistingPagesToStorage(ctx context.Context) (*filer_pb.FileChunk, error) {

	if pages.Size == 0 {
		return nil, nil
	}

	return pages.saveToStorage(ctx, pages.Data[:pages.Size], pages.Offset)
}

func (pages *ContinuousDirtyPages) saveToStorage(ctx context.Context, buf []byte, offset int64) (*filer_pb.FileChunk, error) {

	var fileId, host string
	var auth security.EncodedJwt

	if err := pages.f.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {

		request := &filer_pb.AssignVolumeRequest{
			Count:       1,
			Replication: pages.f.wfs.option.Replication,
			Collection:  pages.f.wfs.option.Collection,
			TtlSec:      pages.f.wfs.option.TtlSec,
			DataCenter:  pages.f.wfs.option.DataCenter,
		}

		resp, err := client.AssignVolume(ctx, request)
		if err != nil {
			glog.V(0).Infof("assign volume failure %v: %v", request, err)
			return err
		}

		fileId, host, auth = resp.FileId, resp.Url, security.EncodedJwt(resp.Auth)

		return nil
	}); err != nil {
		return nil, fmt.Errorf("filerGrpcAddress assign volume: %v", err)
	}

	fileUrl := fmt.Sprintf("http://%s/%s", host, fileId)
	bufReader := bytes.NewReader(buf)
	uploadResult, err := operation.Upload(fileUrl, pages.f.Name, bufReader, false, "application/octet-stream", nil, auth)
	if err != nil {
		glog.V(0).Infof("upload data %v to %s: %v", pages.f.Name, fileUrl, err)
		return nil, fmt.Errorf("upload data: %v", err)
	}
	if uploadResult.Error != "" {
		glog.V(0).Infof("upload failure %v to %s: %v", pages.f.Name, fileUrl, err)
		return nil, fmt.Errorf("upload result: %v", uploadResult.Error)
	}

	return &filer_pb.FileChunk{
		FileId: fileId,
		Offset: offset,
		Size:   uint64(len(buf)),
		Mtime:  time.Now().UnixNano(),
		ETag:   uploadResult.ETag,
	}, nil

}

func max(x, y int64) int64 {
	if x > y {
		return x
	}
	return y
}
<commit_msg>adjust logging<commit_after>package filesys

import (
	"bytes"
	"context"
	"fmt"
	"sync"
	"sync/atomic"
	"time"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/operation"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/security"
)

type ContinuousDirtyPages struct {
	hasData bool
	Offset  int64
	Size    int64
	Data    []byte
	f       *File
	lock    sync.Mutex
}

func newDirtyPages(file *File) *ContinuousDirtyPages {
	return &ContinuousDirtyPages{
		Data: nil,
		f:    file,
	}
}

func (pages *ContinuousDirtyPages) releaseResource() {
	if pages.Data != nil {
		pages.f.wfs.bufPool.Put(pages.Data)
		pages.Data = nil
		atomic.AddInt32(&counter, -1)
		glog.V(3).Infof("%s/%s releasing resource %d", pages.f.dir.Path, pages.f.Name, counter)
	}
}

var counter = int32(0)

func (pages *ContinuousDirtyPages) AddPage(ctx context.Context, offset int64, data []byte) (chunks []*filer_pb.FileChunk, err error) {

	pages.lock.Lock()
	defer pages.lock.Unlock()

	var chunk *filer_pb.FileChunk

	if len(data) > int(pages.f.wfs.option.ChunkSizeLimit) {
		// this is more than what buffer can hold.
		return pages.flushAndSave(ctx, offset, data)
	}

	if pages.Data == nil {
		pages.Data = pages.f.wfs.bufPool.Get().([]byte)
		atomic.AddInt32(&counter, 1)
		glog.V(3).Infof("%s/%s acquire resource %d", pages.f.dir.Path, pages.f.Name, counter)
	}

	if offset < pages.Offset || offset >= pages.Offset+int64(len(pages.Data)) ||
		pages.Offset+int64(len(pages.Data)) < offset+int64(len(data)) {
		// if the data is out of range,
		// or buffer is full if adding new data,
		// flush current buffer and add new data

		// println("offset", offset, "size", len(data), "existing offset", pages.Offset, "size", pages.Size)

		if chunk, err = pages.saveExistingPagesToStorage(ctx); err == nil {
			if chunk != nil {
				glog.V(4).Infof("%s/%s add save [%d,%d)", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size))
				chunks = append(chunks, chunk)
			}
		} else {
			glog.V(0).Infof("%s/%s add save [%d,%d): %v", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size), err)
			return
		}
		pages.Offset = offset
		copy(pages.Data, data)
		pages.Size = int64(len(data))
		return
	}

	if offset != pages.Offset+pages.Size {
		// when this happens, debug shows the data overlapping with existing data is empty
		// the data is not just append
		if offset == pages.Offset && int(pages.Size) < len(data) {
			// glog.V(2).Infof("pages[%d,%d) pages.Data len=%v, data len=%d, pages.Size=%d", pages.Offset, pages.Offset+pages.Size, len(pages.Data), len(data), pages.Size)
			copy(pages.Data[pages.Size:], data[pages.Size:])
		} else {
			if pages.Size != 0 {
				glog.V(1).Infof("%s/%s add page: pages [%d, %d) write [%d, %d)", pages.f.dir.Path, pages.f.Name, pages.Offset, pages.Offset+pages.Size, offset, offset+int64(len(data)))
			}
			return pages.flushAndSave(ctx, offset, data)
		}
	} else {
		copy(pages.Data[offset-pages.Offset:], data)
	}

	pages.Size = max(pages.Size, offset+int64(len(data))-pages.Offset)

	return
}

func (pages *ContinuousDirtyPages) flushAndSave(ctx context.Context, offset int64, data []byte) (chunks []*filer_pb.FileChunk, err error) {

	var chunk *filer_pb.FileChunk

	// flush existing
	if chunk, err = pages.saveExistingPagesToStorage(ctx); err == nil {
		if chunk != nil {
			glog.V(4).Infof("%s/%s flush existing [%d,%d) to %s", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size), chunk.FileId)
			chunks = append(chunks, chunk)
		}
	} else {
		glog.V(0).Infof("%s/%s failed to flush1 [%d,%d): %v", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size), err)
		return
	}
	pages.Size = 0
	pages.Offset = 0

	// flush the new page
	if chunk, err = pages.saveToStorage(ctx, data, offset); err == nil {
		if chunk != nil {
			glog.V(4).Infof("%s/%s flush big request [%d,%d) to %s", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size), chunk.FileId)
			chunks = append(chunks, chunk)
		}
	} else {
		glog.V(0).Infof("%s/%s failed to flush2 [%d,%d): %v", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size), err)
		return
	}

	return
}

func (pages *ContinuousDirtyPages) FlushToStorage(ctx context.Context) (chunk *filer_pb.FileChunk, err error) {

	pages.lock.Lock()
	defer pages.lock.Unlock()

	if pages.Size == 0 {
		return nil, nil
	}

	if chunk, err = pages.saveExistingPagesToStorage(ctx); err == nil {
		pages.Size = 0
		pages.Offset = 0
		if chunk != nil {
			glog.V(4).Infof("%s/%s flush [%d,%d)", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size))
		}
	}
	return
}

func (pages *ContinuousDirtyPages) saveExistingPagesToStorage(ctx context.Context) (*filer_pb.FileChunk, error) {

	if pages.Size == 0 {
		return nil, nil
	}

	return pages.saveToStorage(ctx, pages.Data[:pages.Size], pages.Offset)
}

func (pages *ContinuousDirtyPages) saveToStorage(ctx context.Context, buf []byte, offset int64) (*filer_pb.FileChunk, error) {

	var fileId, host string
	var auth security.EncodedJwt

	if err := pages.f.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {

		request := &filer_pb.AssignVolumeRequest{
			Count:       1,
			Replication: pages.f.wfs.option.Replication,
			Collection:  pages.f.wfs.option.Collection,
			TtlSec:      pages.f.wfs.option.TtlSec,
			DataCenter:  pages.f.wfs.option.DataCenter,
		}

		resp, err := client.AssignVolume(ctx, request)
		if err != nil {
			glog.V(0).Infof("assign volume failure %v: %v", request, err)
			return err
		}

		fileId, host, auth = resp.FileId, resp.Url, security.EncodedJwt(resp.Auth)

		return nil
	}); err != nil {
		return nil, fmt.Errorf("filerGrpcAddress assign volume: %v", err)
	}

	fileUrl := fmt.Sprintf("http://%s/%s", host, fileId)
	bufReader := bytes.NewReader(buf)
	uploadResult, err := operation.Upload(fileUrl, pages.f.Name, bufReader, false, "application/octet-stream", nil, auth)
	if err != nil {
		glog.V(0).Infof("upload data %v to %s: %v", pages.f.Name, fileUrl, err)
		return nil, fmt.Errorf("upload data: %v", err)
	}
	if uploadResult.Error != "" {
		glog.V(0).Infof("upload failure %v to %s: %v", pages.f.Name, fileUrl, err)
		return nil, fmt.Errorf("upload result: %v", uploadResult.Error)
	}

	return &filer_pb.FileChunk{
		FileId: fileId,
		Offset: offset,
		Size:   uint64(len(buf)),
		Mtime:  time.Now().UnixNano(),
		ETag:   uploadResult.ETag,
	}, nil

}

func max(x, y int64) int64 {
	if x > y {
		return x
	}
	return y
}
<|endoftext|>


<commit_before>package filesys

import (
	"context"
	"fmt"
	"io"
	"math"
	"net/http"
	"os"
	"sync"
	"time"

	"github.com/seaweedfs/fuse"
	"github.com/seaweedfs/fuse/fs"

	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
)

type FileHandle struct {
	// cache file has been written to
	dirtyPages     *PageWriter
	entryViewCache []filer.VisibleInterval
	reader         io.ReaderAt
	contentType    string
	handle         uint64
	sync.Mutex

	f         *File
	RequestId fuse.RequestID // unique ID for request
	NodeId    fuse.NodeID    // file or directory the request is about
	Uid       uint32         // user ID of process making request
	Gid       uint32         // group ID of process making request
	writeOnly bool
	isDeleted bool
}

func newFileHandle(file *File, uid, gid uint32) *FileHandle {
	fh := &FileHandle{
		f: file,
		// dirtyPages: newContinuousDirtyPages(file, writeOnly),
		dirtyPages: newPageWriter(file, file.wfs.option.ChunkSizeLimit),
		Uid:        uid,
		Gid:        gid,
	}
	entry := fh.f.getEntry()
	if entry != nil {
		entry.Attributes.FileSize = filer.FileSize(entry)
	}

	return fh
}

var _ = fs.Handle(&FileHandle{})

// var _ = fs.HandleReadAller(&FileHandle{})
var _ = fs.HandleReader(&FileHandle{})
var _ = fs.HandleFlusher(&FileHandle{})
var _ = fs.HandleWriter(&FileHandle{})
var _ = fs.HandleReleaser(&FileHandle{})

func (fh *FileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error {

	fh.Lock()
	defer fh.Unlock()

	glog.V(4).Infof("%s read fh %d: [%d,%d) size %d resp.Data cap=%d", fh.f.fullpath(), fh.handle, req.Offset, req.Offset+int64(req.Size), req.Size, cap(resp.Data))

	if req.Size <= 0 {
		return nil
	}

	buff := resp.Data[:cap(resp.Data)]
	if req.Size > cap(resp.Data) {
		// should not happen
		buff = make([]byte, req.Size)
	}

	totalRead, err := fh.readFromChunks(buff, req.Offset)
	if err == nil || err == io.EOF {
		maxStop := fh.readFromDirtyPages(buff, req.Offset)
		totalRead = max(maxStop-req.Offset, totalRead)
	}

	if err == io.EOF {
		err = nil
	}

	if err != nil {
		glog.Warningf("file handle read %s %d: %v", fh.f.fullpath(), totalRead, err)
		return fuse.EIO
	}

	if totalRead > int64(len(buff)) {
		glog.Warningf("%s FileHandle Read %d: [%d,%d) size %d totalRead %d", fh.f.fullpath(), fh.handle, req.Offset, req.Offset+int64(req.Size), req.Size, totalRead)
		totalRead = min(int64(len(buff)), totalRead)
	}
	if err == nil {
		resp.Data = buff[:totalRead]
	}

	return err
}

func (fh *FileHandle) readFromDirtyPages(buff []byte, startOffset int64) (maxStop int64) {
	maxStop = fh.dirtyPages.ReadDirtyDataAt(buff, startOffset)
	return
}

func (fh *FileHandle) readFromChunks(buff []byte, offset int64) (int64, error) {

	entry := fh.f.getEntry()
	if entry == nil {
		return 0, io.EOF
	}

	if entry.IsInRemoteOnly() {
		glog.V(4).Infof("download remote entry %s", fh.f.fullpath())
		newEntry, err := fh.f.downloadRemoteEntry(entry)
		if err != nil {
			glog.V(1).Infof("download remote entry %s: %v", fh.f.fullpath(), err)
			return 0, err
		}
		entry = newEntry
	}

	fileSize := int64(filer.FileSize(entry))
	fileFullPath := fh.f.fullpath()

	if fileSize == 0 {
		glog.V(1).Infof("empty fh %v", fileFullPath)
		return 0, io.EOF
	}

	if offset+int64(len(buff)) <= int64(len(entry.Content)) {
		totalRead := copy(buff, entry.Content[offset:])
		glog.V(4).Infof("file handle read cached %s [%d,%d] %d", fileFullPath, offset, offset+int64(totalRead), totalRead)
		return int64(totalRead), nil
	}

	var chunkResolveErr error
	if fh.entryViewCache == nil {
		fh.entryViewCache, chunkResolveErr = filer.NonOverlappingVisibleIntervals(fh.f.wfs.LookupFn(), entry.Chunks, 0, math.MaxInt64)
		if chunkResolveErr != nil {
			return 0, fmt.Errorf("fail to resolve chunk manifest: %v", chunkResolveErr)
		}
		fh.reader = nil
	}

	reader := fh.reader
	if reader == nil {
		chunkViews := filer.ViewFromVisibleIntervals(fh.entryViewCache, 0, math.MaxInt64)
		reader = filer.NewChunkReaderAtFromClient(fh.f.wfs.LookupFn(), chunkViews, fh.f.wfs.chunkCache, fileSize)
	}
	fh.reader = reader

	totalRead, err := reader.ReadAt(buff, offset)

	if err != nil && err != io.EOF {
		glog.Errorf("file handle read %s: %v", fileFullPath, err)
	}

	// glog.V(4).Infof("file handle read %s [%d,%d] %d : %v", fileFullPath, offset, offset+int64(totalRead), totalRead, err)

	return int64(totalRead), err
}

// Write to the file handle
func (fh *FileHandle) Write(ctx context.Context, req *fuse.WriteRequest, resp *fuse.WriteResponse) error {

	fh.dirtyPages.writerPattern.MonitorWriteAt(req.Offset, len(req.Data))

	fh.Lock()
	defer fh.Unlock()

	// write the request to volume servers
	data := req.Data
	if len(data) <= 512 && req.Offset == 0 {
		// fuse message cacheable size
		data = make([]byte, len(req.Data))
		copy(data, req.Data)
	}

	entry := fh.f.getEntry()
	if entry == nil {
		return fuse.EIO
	}

	entry.Content = nil
	entry.Attributes.FileSize = uint64(max(req.Offset+int64(len(data)), int64(entry.Attributes.FileSize)))
	// glog.V(4).Infof("%v write [%d,%d) %d", fh.f.fullpath(), req.Offset, req.Offset+int64(len(req.Data)), len(req.Data))

	fh.dirtyPages.AddPage(req.Offset, data)

	resp.Size = len(data)

	if req.Offset == 0 {
		// detect mime type
		fh.contentType = http.DetectContentType(data)
		fh.f.dirtyMetadata = true
	}

	fh.f.dirtyMetadata = true

	return nil
}

func (fh *FileHandle) Release(ctx context.Context, req *fuse.ReleaseRequest) error {

	glog.V(4).Infof("Release %v fh %d open=%d", fh.f.fullpath(), fh.handle, fh.f.isOpen)

	fh.f.wfs.handlesLock.Lock()
	fh.f.isOpen--
	fh.f.wfs.handlesLock.Unlock()

	if fh.f.isOpen <= 0 {
		fh.f.entry = nil
		fh.entryViewCache = nil
		fh.reader = nil

		fh.f.wfs.ReleaseHandle(fh.f.fullpath(), fuse.HandleID(fh.handle))
		fh.dirtyPages.Destroy()
	}

	if fh.f.isOpen < 0 {
		glog.V(0).Infof("Release reset %s open count %d => %d", fh.f.Name, fh.f.isOpen, 0)
		fh.f.isOpen = 0
		return nil
	}

	return nil
}

func (fh *FileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error {

	glog.V(4).Infof("Flush %v fh %d", fh.f.fullpath(), fh.handle)

	if fh.isDeleted {
		glog.V(4).Infof("Flush %v fh %d skip deleted", fh.f.fullpath(), fh.handle)
		return nil
	}

	fh.Lock()
	defer fh.Unlock()

	if err := fh.doFlush(ctx, req.Header); err != nil {
		glog.Errorf("Flush doFlush %s: %v", fh.f.Name, err)
		return err
	}

	return nil
}

func (fh *FileHandle) doFlush(ctx context.Context, header fuse.Header) error {
	// flush works at fh level
	// send the data to the OS
	glog.V(4).Infof("doFlush %s fh %d", fh.f.fullpath(), fh.handle)

	if err := fh.dirtyPages.FlushData(); err != nil {
		glog.Errorf("%v doFlush: %v", fh.f.fullpath(), err)
		return fuse.EIO
	}

	if !fh.f.dirtyMetadata {
		return nil
	}

	err := fh.f.wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {

		entry := fh.f.getEntry()
		if entry == nil {
			return nil
		}

		if entry.Attributes != nil {
			entry.Attributes.Mime = fh.contentType
			if entry.Attributes.Uid == 0 {
				entry.Attributes.Uid = header.Uid
			}
			if entry.Attributes.Gid == 0 {
				entry.Attributes.Gid = header.Gid
			}
			if entry.Attributes.Crtime == 0 {
				entry.Attributes.Crtime = time.Now().Unix()
			}
			entry.Attributes.Mtime = time.Now().Unix()
			entry.Attributes.FileMode = uint32(os.FileMode(entry.Attributes.FileMode) &^ fh.f.wfs.option.Umask)
			entry.Attributes.Collection, entry.Attributes.Replication = fh.dirtyPages.GetStorageOptions()
		}

		request := &filer_pb.CreateEntryRequest{
			Directory:  fh.f.dir.FullPath(),
			Entry:      entry,
			Signatures: []int32{fh.f.wfs.signature},
		}

		glog.V(4).Infof("%s set chunks: %v", fh.f.fullpath(), len(entry.Chunks))
		for i, chunk := range entry.Chunks {
			glog.V(4).Infof("%s chunks %d: %v [%d,%d)", fh.f.fullpath(), i, chunk.GetFileIdString(), chunk.Offset, chunk.Offset+int64(chunk.Size))
		}

		manifestChunks, nonManifestChunks := filer.SeparateManifestChunks(entry.Chunks)

		chunks, _ := filer.CompactFileChunks(fh.f.wfs.LookupFn(), nonManifestChunks)
		chunks, manifestErr := filer.MaybeManifestize(fh.f.wfs.saveDataAsChunk(fh.f.fullpath()), chunks)
		if manifestErr != nil {
			// not good, but should be ok
			glog.V(0).Infof("MaybeManifestize: %v", manifestErr)
		}
		entry.Chunks = append(chunks, manifestChunks...)

		fh.f.wfs.mapPbIdFromLocalToFiler(request.Entry)
		defer fh.f.wfs.mapPbIdFromFilerToLocal(request.Entry)

		if err := filer_pb.CreateEntry(client, request); err != nil {
			glog.Errorf("fh flush create %s: %v", fh.f.fullpath(), err)
			return fmt.Errorf("fh flush create %s: %v", fh.f.fullpath(), err)
		}

		fh.f.wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, request.Entry))

		return nil
	})

	if err == nil {
		fh.f.dirtyMetadata = false
	}

	if err != nil {
		glog.Errorf("%v fh %d flush: %v", fh.f.fullpath(), fh.handle, err)
		return fuse.EIO
	}

	return nil
}
<commit_msg>remove unused variables<commit_after>package filesys

import (
	"context"
	"fmt"
	"io"
	"math"
	"net/http"
	"os"
	"sync"
	"time"

	"github.com/seaweedfs/fuse"
	"github.com/seaweedfs/fuse/fs"

	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
)

type FileHandle struct {
	// cache file has been written to
	dirtyPages     *PageWriter
	entryViewCache []filer.VisibleInterval
	reader         io.ReaderAt
	contentType    string
	handle         uint64
	sync.Mutex

	f         *File
	NodeId    fuse.NodeID // file or directory the request is about
	Uid       uint32      // user ID of process making request
	Gid       uint32      // group ID of process making request
	isDeleted bool
}

func newFileHandle(file *File, uid, gid uint32) *FileHandle {
	fh := &FileHandle{
		f: file,
		// dirtyPages: newContinuousDirtyPages(file, writeOnly),
		dirtyPages: newPageWriter(file, file.wfs.option.ChunkSizeLimit),
		Uid:        uid,
		Gid:        gid,
	}
	entry := fh.f.getEntry()
	if entry != nil {
		entry.Attributes.FileSize = filer.FileSize(entry)
	}

	return fh
}

var _ = fs.Handle(&FileHandle{})

// var _ = fs.HandleReadAller(&FileHandle{})
var _ = fs.HandleReader(&FileHandle{})
var _ = fs.HandleFlusher(&FileHandle{})
var _ = fs.HandleWriter(&FileHandle{})
var _ = fs.HandleReleaser(&FileHandle{})

func (fh *FileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error {

	fh.Lock()
	defer fh.Unlock()

	glog.V(4).Infof("%s read fh %d: [%d,%d) size %d resp.Data cap=%d", fh.f.fullpath(), fh.handle, req.Offset, req.Offset+int64(req.Size), req.Size, cap(resp.Data))

	if req.Size <= 0 {
		return nil
	}

	buff := resp.Data[:cap(resp.Data)]
	if req.Size > cap(resp.Data) {
		// should not happen
		buff = make([]byte, req.Size)
	}

	totalRead, err := fh.readFromChunks(buff, req.Offset)
	if err == nil || err == io.EOF {
		maxStop := fh.readFromDirtyPages(buff, req.Offset)
		totalRead = max(maxStop-req.Offset, totalRead)
	}

	if err == io.EOF {
		err = nil
	}

	if err != nil {
		glog.Warningf("file handle read %s %d: %v", fh.f.fullpath(), totalRead, err)
		return fuse.EIO
	}

	if totalRead > int64(len(buff)) {
		glog.Warningf("%s FileHandle Read %d: [%d,%d) size %d totalRead %d", fh.f.fullpath(), fh.handle, req.Offset, req.Offset+int64(req.Size), req.Size, totalRead)
		totalRead = min(int64(len(buff)), totalRead)
	}
	if err == nil {
		resp.Data = buff[:totalRead]
	}

	return err
}

func (fh *FileHandle) readFromDirtyPages(buff []byte, startOffset int64) (maxStop int64) {
	maxStop = fh.dirtyPages.ReadDirtyDataAt(buff, startOffset)
	return
}

func (fh *FileHandle) readFromChunks(buff []byte, offset int64) (int64, error) {

	entry := fh.f.getEntry()
	if entry == nil {
		return 0, io.EOF
	}

	if entry.IsInRemoteOnly() {
		glog.V(4).Infof("download remote entry %s", fh.f.fullpath())
		newEntry, err := fh.f.downloadRemoteEntry(entry)
		if err != nil {
			glog.V(1).Infof("download remote entry %s: %v", fh.f.fullpath(), err)
			return 0, err
		}
		entry = newEntry
	}

	fileSize := int64(filer.FileSize(entry))
	fileFullPath := fh.f.fullpath()

	if fileSize == 0 {
		glog.V(1).Infof("empty fh %v", fileFullPath)
		return 0, io.EOF
	}

	if offset+int64(len(buff)) <= int64(len(entry.Content)) {
		totalRead := copy(buff, entry.Content[offset:])
		glog.V(4).Infof("file handle read cached %s [%d,%d] %d", fileFullPath, offset, offset+int64(totalRead), totalRead)
		return int64(totalRead), nil
	}

	var chunkResolveErr error
	if fh.entryViewCache == nil {
		fh.entryViewCache, chunkResolveErr = filer.NonOverlappingVisibleIntervals(fh.f.wfs.LookupFn(), entry.Chunks, 0, math.MaxInt64)
		if chunkResolveErr != nil {
			return 0, fmt.Errorf("fail to resolve chunk manifest: %v", chunkResolveErr)
		}
		fh.reader = nil
	}

	reader := fh.reader
	if reader == nil {
		chunkViews := filer.ViewFromVisibleIntervals(fh.entryViewCache, 0, math.MaxInt64)
		reader = filer.NewChunkReaderAtFromClient(fh.f.wfs.LookupFn(), chunkViews, fh.f.wfs.chunkCache, fileSize)
	}
	fh.reader = reader

	totalRead, err := reader.ReadAt(buff, offset)

	if err != nil && err != io.EOF {
		glog.Errorf("file handle read %s: %v", fileFullPath, err)
	}

	// glog.V(4).Infof("file handle read %s [%d,%d] %d : %v", fileFullPath, offset, offset+int64(totalRead), totalRead, err)

	return int64(totalRead), err
}

// Write to the file handle
func (fh *FileHandle) Write(ctx context.Context, req *fuse.WriteRequest, resp *fuse.WriteResponse) error {

	fh.dirtyPages.writerPattern.MonitorWriteAt(req.Offset, len(req.Data))

	fh.Lock()
	defer fh.Unlock()

	// write the request to volume servers
	data := req.Data
	if len(data) <= 512 && req.Offset == 0 {
		// fuse message cacheable size
		data = make([]byte, len(req.Data))
		copy(data, req.Data)
	}

	entry := fh.f.getEntry()
	if entry == nil {
		return fuse.EIO
	}

	entry.Content = nil
	entry.Attributes.FileSize = uint64(max(req.Offset+int64(len(data)), int64(entry.Attributes.FileSize)))
	// glog.V(4).Infof("%v write [%d,%d) %d", fh.f.fullpath(), req.Offset, req.Offset+int64(len(req.Data)), len(req.Data))

	fh.dirtyPages.AddPage(req.Offset, data)

	resp.Size = len(data)

	if req.Offset == 0 {
		// detect mime type
		fh.contentType = http.DetectContentType(data)
		fh.f.dirtyMetadata = true
	}

	fh.f.dirtyMetadata = true

	return nil
}

func (fh *FileHandle) Release(ctx context.Context, req *fuse.ReleaseRequest) error {

	glog.V(4).Infof("Release %v fh %d open=%d", fh.f.fullpath(), fh.handle, fh.f.isOpen)

	fh.f.wfs.handlesLock.Lock()
	fh.f.isOpen--
	fh.f.wfs.handlesLock.Unlock()

	if fh.f.isOpen <= 0 {
		fh.f.entry = nil
		fh.entryViewCache = nil
		fh.reader = nil

		fh.f.wfs.ReleaseHandle(fh.f.fullpath(), fuse.HandleID(fh.handle))
		fh.dirtyPages.Destroy()
	}

	if fh.f.isOpen < 0 {
		glog.V(0).Infof("Release reset %s open count %d => %d", fh.f.Name, fh.f.isOpen, 0)
		fh.f.isOpen = 0
		return nil
	}

	return nil
}

func (fh *FileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error {

	glog.V(4).Infof("Flush %v fh %d", fh.f.fullpath(), fh.handle)

	if fh.isDeleted {
		glog.V(4).Infof("Flush %v fh %d skip deleted", fh.f.fullpath(), fh.handle)
		return nil
	}

	fh.Lock()
	defer fh.Unlock()

	if err := fh.doFlush(ctx, req.Header); err != nil {
		glog.Errorf("Flush doFlush %s: %v", fh.f.Name, err)
		return err
	}

	return nil
}

func (fh *FileHandle) doFlush(ctx context.Context, header fuse.Header) error {
	// flush works at fh level
	// send the data to the OS
	glog.V(4).Infof("doFlush %s fh %d", fh.f.fullpath(), fh.handle)

	if err := fh.dirtyPages.FlushData(); err != nil {
		glog.Errorf("%v doFlush: %v", fh.f.fullpath(), err)
		return fuse.EIO
	}

	if !fh.f.dirtyMetadata {
		return nil
	}

	err := fh.f.wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {

		entry := fh.f.getEntry()
		if entry == nil {
			return nil
		}

		if entry.Attributes != nil {
			entry.Attributes.Mime = fh.contentType
			if entry.Attributes.Uid == 0 {
				entry.Attributes.Uid = header.Uid
			}
			if entry.Attributes.Gid == 0 {
				entry.Attributes.Gid = header.Gid
			}
			if entry.Attributes.Crtime == 0 {
				entry.Attributes.Crtime = time.Now().Unix()
			}
			entry.Attributes.Mtime = time.Now().Unix()
			entry.Attributes.FileMode = uint32(os.FileMode(entry.Attributes.FileMode) &^ fh.f.wfs.option.Umask)
			entry.Attributes.Collection, entry.Attributes.Replication = fh.dirtyPages.GetStorageOptions()
		}

		request := &filer_pb.CreateEntryRequest{
			Directory:  fh.f.dir.FullPath(),
			Entry:      entry,
			Signatures: []int32{fh.f.wfs.signature},
		}

		glog.V(4).Infof("%s set chunks: %v", fh.f.fullpath(), len(entry.Chunks))
		for i, chunk := range entry.Chunks {
			glog.V(4).Infof("%s chunks %d: %v [%d,%d)", fh.f.fullpath(), i, chunk.GetFileIdString(), chunk.Offset, chunk.Offset+int64(chunk.Size))
		}

		manifestChunks, nonManifestChunks := filer.SeparateManifestChunks(entry.Chunks)

		chunks, _ := filer.CompactFileChunks(fh.f.wfs.LookupFn(), nonManifestChunks)
		chunks, manifestErr := filer.MaybeManifestize(fh.f.wfs.saveDataAsChunk(fh.f.fullpath()), chunks)
		if manifestErr != nil {
			// not good, but should be ok
			glog.V(0).Infof("MaybeManifestize: %v", manifestErr)
		}
		entry.Chunks = append(chunks, manifestChunks...)

		fh.f.wfs.mapPbIdFromLocalToFiler(request.Entry)
		defer fh.f.wfs.mapPbIdFromFilerToLocal(request.Entry)

		if err := filer_pb.CreateEntry(client, request); err != nil {
			glog.Errorf("fh flush create %s: %v", fh.f.fullpath(), err)
			return fmt.Errorf("fh flush create %s: %v", fh.f.fullpath(), err)
		}

		fh.f.wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, request.Entry))

		return nil
	})

	if err == nil {
		fh.f.dirtyMetadata = false
	}

	if err != nil {
		glog.Errorf("%v fh %d flush: %v", fh.f.fullpath(), fh.handle, err)
		return fuse.EIO
	}

	return nil
}
<|endoftext|>