<commit_before>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package ssh_test

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/juju/testing"
	gc "launchpad.net/gocheck"

	"launchpad.net/juju-core/cmd"
	"launchpad.net/juju-core/utils/ssh"
)

type SSHCommandSuite struct {
	testing.IsolationSuite
	originalPath string
	testbin      string
	fakessh      string
	fakescp      string
	echoCommand  string
	echoScript   string
	client       ssh.Client
}

var _ = gc.Suite(&SSHCommandSuite{})

func (s *SSHCommandSuite) SetUpSuite(c *gc.C) {
	s.IsolationSuite.SetUpSuite(c)
	s.echoCommand = "/bin/echo"
	s.echoScript = fmt.Sprintf("#!/bin/sh\n%s $0 \"$@\" | /usr/bin/tee $0.args", s.echoCommand)
}

func (s *SSHCommandSuite) SetUpTest(c *gc.C) {
	s.IsolationSuite.SetUpTest(c)
	s.testbin = c.MkDir()
	s.fakessh = filepath.Join(s.testbin, "ssh")
	s.fakescp = filepath.Join(s.testbin, "scp")
	err := ioutil.WriteFile(s.fakessh, []byte(s.echoScript), 0755)
	c.Assert(err, gc.IsNil)
	err = ioutil.WriteFile(s.fakescp, []byte(s.echoScript), 0755)
	c.Assert(err, gc.IsNil)
	s.PatchEnvPathPrepend(s.testbin)
	s.client, err = ssh.NewOpenSSHClient()
	c.Assert(err, gc.IsNil)
	s.PatchValue(ssh.DefaultIdentities, nil)
}

func (s *SSHCommandSuite) command(args ...string) *ssh.Cmd {
	return s.commandOptions(args, nil)
}

func (s *SSHCommandSuite) commandOptions(args []string, opts *ssh.Options) *ssh.Cmd {
	return s.client.Command("localhost", args, opts)
}

func (s *SSHCommandSuite) assertCommandArgs(c *gc.C, cmd *ssh.Cmd, expected string) {
	out, err := cmd.Output()
	c.Assert(err, gc.IsNil)
	c.Assert(strings.TrimSpace(string(out)), gc.Equals, expected)
}

func (s *SSHCommandSuite) TestDefaultClient(c *gc.C) {
	ssh.InitDefaultClient()
	c.Assert(ssh.DefaultClient, gc.FitsTypeOf, &ssh.OpenSSHClient{})
	s.PatchEnvironment("PATH", "")
	ssh.InitDefaultClient()
	c.Assert(ssh.DefaultClient, gc.FitsTypeOf, &ssh.GoCryptoClient{})
}

func (s *SSHCommandSuite) TestCommandSSHPass(c *gc.C) {
	// First create a fake sshpass, but don't set $SSHPASS
	fakesshpass := filepath.Join(s.testbin, "sshpass")
	err := ioutil.WriteFile(fakesshpass, []byte(s.echoScript), 0755)
	s.assertCommandArgs(c, s.command(s.echoCommand, "123"),
		fmt.Sprintf("%s -o StrictHostKeyChecking no -o PasswordAuthentication no localhost %s 123",
			s.fakessh, s.echoCommand),
	)
	// Now set $SSHPASS.
	s.PatchEnvironment("SSHPASS", "anyoldthing")
	s.assertCommandArgs(c, s.command(s.echoCommand, "123"),
		fmt.Sprintf("%s -e ssh -o StrictHostKeyChecking no -o PasswordAuthentication no localhost %s 123",
			fakesshpass, s.echoCommand),
	)
	// Finally, remove sshpass from $PATH.
	err = os.Remove(fakesshpass)
	c.Assert(err, gc.IsNil)
	s.assertCommandArgs(c, s.command(s.echoCommand, "123"),
		fmt.Sprintf("%s -o StrictHostKeyChecking no -o PasswordAuthentication no localhost %s 123",
			s.fakessh, s.echoCommand),
	)
}

func (s *SSHCommandSuite) TestCommand(c *gc.C) {
	s.assertCommandArgs(c, s.command(s.echoCommand, "123"),
		fmt.Sprintf("%s -o StrictHostKeyChecking no -o PasswordAuthentication no localhost %s 123",
			s.fakessh, s.echoCommand),
	)
}

func (s *SSHCommandSuite) TestCommandEnablePTY(c *gc.C) {
	var opts ssh.Options
	opts.EnablePTY()
	s.assertCommandArgs(c, s.commandOptions([]string{s.echoCommand, "123"}, &opts),
		fmt.Sprintf("%s -o StrictHostKeyChecking no -o PasswordAuthentication no -t -t localhost %s 123",
			s.fakessh, s.echoCommand),
	)
}

func (s *SSHCommandSuite) TestCommandAllowPasswordAuthentication(c *gc.C) {
	var opts ssh.Options
	opts.AllowPasswordAuthentication()
	s.assertCommandArgs(c, s.commandOptions([]string{s.echoCommand, "123"}, &opts),
		fmt.Sprintf("%s -o StrictHostKeyChecking no localhost %s 123",
			s.fakessh, s.echoCommand),
	)
}

func (s *SSHCommandSuite) TestCommandIdentities(c *gc.C) {
	var opts ssh.Options
	opts.SetIdentities("x", "y")
	s.assertCommandArgs(c, s.commandOptions([]string{s.echoCommand, "123"}, &opts),
		fmt.Sprintf("%s -o StrictHostKeyChecking no -o PasswordAuthentication no -i x -i y localhost %s 123",
			s.fakessh, s.echoCommand),
	)
}

func (s *SSHCommandSuite) TestCommandPort(c *gc.C) {
	var opts ssh.Options
	opts.SetPort(2022)
	s.assertCommandArgs(c, s.commandOptions([]string{s.echoCommand, "123"}, &opts),
		fmt.Sprintf("%s -o StrictHostKeyChecking no -o PasswordAuthentication no -p 2022 localhost %s 123",
			s.fakessh, s.echoCommand),
	)
}

func (s *SSHCommandSuite) TestCopy(c *gc.C) {
	var opts ssh.Options
	opts.EnablePTY()
	opts.AllowPasswordAuthentication()
	opts.SetIdentities("x", "y")
	opts.SetPort(2022)
	err := s.client.Copy([]string{"/tmp/blah", "foo@bar.com:baz"}, &opts)
	c.Assert(err, gc.IsNil)
	out, err := ioutil.ReadFile(s.fakescp + ".args")
	c.Assert(err, gc.IsNil)
	// EnablePTY has no effect for Copy
	c.Assert(string(out), gc.Equals, s.fakescp+" -o StrictHostKeyChecking no -i x -i y -P 2022 /tmp/blah foo@bar.com:baz\n")

	// Try passing extra args
	err = s.client.Copy([]string{"/tmp/blah", "foo@bar.com:baz", "-r", "-v"}, &opts)
	c.Assert(err, gc.IsNil)
	out, err = ioutil.ReadFile(s.fakescp + ".args")
	c.Assert(err, gc.IsNil)
	c.Assert(string(out), gc.Equals, s.fakescp+" -o StrictHostKeyChecking no -i x -i y -P 2022 /tmp/blah foo@bar.com:baz -r -v\n")

	// Try interspersing extra args
	err = s.client.Copy([]string{"-r", "/tmp/blah", "-v", "foo@bar.com:baz"}, &opts)
	c.Assert(err, gc.IsNil)
	out, err = ioutil.ReadFile(s.fakescp + ".args")
	c.Assert(err, gc.IsNil)
	c.Assert(string(out), gc.Equals, s.fakescp+" -o StrictHostKeyChecking no -i x -i y -P 2022 -r /tmp/blah -v foo@bar.com:baz\n")
}

func (s *SSHCommandSuite) TestCommandClientKeys(c *gc.C) {
	defer overrideGenerateKey(c).Restore()
	clientKeysDir := c.MkDir()
	defer ssh.ClearClientKeys()
	err := ssh.LoadClientKeys(clientKeysDir)
	c.Assert(err, gc.IsNil)
	ck := filepath.Join(clientKeysDir, "juju_id_rsa")
	var opts ssh.Options
	opts.SetIdentities("x", "y")
	s.assertCommandArgs(c, s.commandOptions([]string{s.echoCommand, "123"}, &opts),
		fmt.Sprintf("%s -o StrictHostKeyChecking no -o PasswordAuthentication no -i x -i y -i %s localhost %s 123",
			s.fakessh, ck, s.echoCommand),
	)
}

func (s *SSHCommandSuite) TestCommandError(c *gc.C) {
	var opts ssh.Options
	err := ioutil.WriteFile(s.fakessh, []byte("#!/bin/sh\nexit 42"), 0755)
	c.Assert(err, gc.IsNil)
	command := s.client.Command("ignored", []string{s.echoCommand, "foo"}, &opts)
	err = command.Run()
	c.Assert(cmd.IsRcPassthroughError(err), gc.Equals, true)
}

func (s *SSHCommandSuite) TestCommandDefaultIdentities(c *gc.C) {
	var opts ssh.Options
	tempdir := c.MkDir()
	def1 := filepath.Join(tempdir, "def1")
	def2 := filepath.Join(tempdir, "def2")
	s.PatchValue(ssh.DefaultIdentities, []string{def1, def2})
	// If no identities are specified, then the defaults aren't added.
	s.assertCommandArgs(c, s.commandOptions([]string{s.echoCommand, "123"}, &opts),
		fmt.Sprintf("%s -o StrictHostKeyChecking no -o PasswordAuthentication no localhost %s 123",
			s.fakessh, s.echoCommand),
	)
	// If identities are specified, then the defaults are must added.
	// Only the defaults that exist on disk will be added.
	err := ioutil.WriteFile(def2, nil, 0644)
	c.Assert(err, gc.IsNil)
	opts.SetIdentities("x", "y")
	s.assertCommandArgs(c, s.commandOptions([]string{s.echoCommand, "123"}, &opts),
		fmt.Sprintf("%s -o StrictHostKeyChecking no -o PasswordAuthentication no -i x -i y -i %s localhost %s 123",
			s.fakessh, def2, s.echoCommand),
	)
}
<commit_msg>Constant echo commands.<commit_after>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package ssh_test

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/juju/testing"
	gc "launchpad.net/gocheck"

	"launchpad.net/juju-core/cmd"
	"launchpad.net/juju-core/utils/ssh"
)

const (
	echoCommand = "/bin/echo"
	echoScript  = "#!/bin/sh\n" + echoCommand + " $0 \"$@\" | /usr/bin/tee $0.args"
)

type SSHCommandSuite struct {
	testing.IsolationSuite
	originalPath string
	testbin      string
	fakessh      string
	fakescp      string
	client       ssh.Client
}

var _ = gc.Suite(&SSHCommandSuite{})

func (s *SSHCommandSuite) SetUpTest(c *gc.C) {
	s.IsolationSuite.SetUpTest(c)
	s.testbin = c.MkDir()
	s.fakessh = filepath.Join(s.testbin, "ssh")
	s.fakescp = filepath.Join(s.testbin, "scp")
	err := ioutil.WriteFile(s.fakessh, []byte(echoScript), 0755)
	c.Assert(err, gc.IsNil)
	err = ioutil.WriteFile(s.fakescp, []byte(echoScript), 0755)
	c.Assert(err, gc.IsNil)
	s.PatchEnvPathPrepend(s.testbin)
	s.client, err = ssh.NewOpenSSHClient()
	c.Assert(err, gc.IsNil)
	s.PatchValue(ssh.DefaultIdentities, nil)
}

func (s *SSHCommandSuite) command(args ...string) *ssh.Cmd {
	return s.commandOptions(args, nil)
}

func (s *SSHCommandSuite) commandOptions(args []string, opts *ssh.Options) *ssh.Cmd {
	return s.client.Command("localhost", args, opts)
}

func (s *SSHCommandSuite) assertCommandArgs(c *gc.C, cmd *ssh.Cmd, expected string) {
	out, err := cmd.Output()
	c.Assert(err, gc.IsNil)
	c.Assert(strings.TrimSpace(string(out)), gc.Equals, expected)
}

func (s *SSHCommandSuite) TestDefaultClient(c *gc.C) {
	ssh.InitDefaultClient()
	c.Assert(ssh.DefaultClient, gc.FitsTypeOf, &ssh.OpenSSHClient{})
	s.PatchEnvironment("PATH", "")
	ssh.InitDefaultClient()
	c.Assert(ssh.DefaultClient, gc.FitsTypeOf, &ssh.GoCryptoClient{})
}

func (s *SSHCommandSuite) TestCommandSSHPass(c *gc.C) {
	// First create a fake sshpass, but don't set $SSHPASS
	fakesshpass := filepath.Join(s.testbin, "sshpass")
	err := ioutil.WriteFile(fakesshpass, []byte(echoScript), 0755)
	s.assertCommandArgs(c, s.command(echoCommand, "123"),
		fmt.Sprintf("%s -o StrictHostKeyChecking no -o PasswordAuthentication no localhost %s 123",
			s.fakessh, echoCommand),
	)
	// Now set $SSHPASS.
	s.PatchEnvironment("SSHPASS", "anyoldthing")
	s.assertCommandArgs(c, s.command(echoCommand, "123"),
		fmt.Sprintf("%s -e ssh -o StrictHostKeyChecking no -o PasswordAuthentication no localhost %s 123",
			fakesshpass, echoCommand),
	)
	// Finally, remove sshpass from $PATH.
	err = os.Remove(fakesshpass)
	c.Assert(err, gc.IsNil)
	s.assertCommandArgs(c, s.command(echoCommand, "123"),
		fmt.Sprintf("%s -o StrictHostKeyChecking no -o PasswordAuthentication no localhost %s 123",
			s.fakessh, echoCommand),
	)
}

func (s *SSHCommandSuite) TestCommand(c *gc.C) {
	s.assertCommandArgs(c, s.command(echoCommand, "123"),
		fmt.Sprintf("%s -o StrictHostKeyChecking no -o PasswordAuthentication no localhost %s 123",
			s.fakessh, echoCommand),
	)
}

func (s *SSHCommandSuite) TestCommandEnablePTY(c *gc.C) {
	var opts ssh.Options
	opts.EnablePTY()
	s.assertCommandArgs(c, s.commandOptions([]string{echoCommand, "123"}, &opts),
		fmt.Sprintf("%s -o StrictHostKeyChecking no -o PasswordAuthentication no -t -t localhost %s 123",
			s.fakessh, echoCommand),
	)
}

func (s *SSHCommandSuite) TestCommandAllowPasswordAuthentication(c *gc.C) {
	var opts ssh.Options
	opts.AllowPasswordAuthentication()
	s.assertCommandArgs(c, s.commandOptions([]string{echoCommand, "123"}, &opts),
		fmt.Sprintf("%s -o StrictHostKeyChecking no localhost %s 123",
			s.fakessh, echoCommand),
	)
}

func (s *SSHCommandSuite) TestCommandIdentities(c *gc.C) {
	var opts ssh.Options
	opts.SetIdentities("x", "y")
	s.assertCommandArgs(c, s.commandOptions([]string{echoCommand, "123"}, &opts),
		fmt.Sprintf("%s -o StrictHostKeyChecking no -o PasswordAuthentication no -i x -i y localhost %s 123",
			s.fakessh, echoCommand),
	)
}

func (s *SSHCommandSuite) TestCommandPort(c *gc.C) {
	var opts ssh.Options
	opts.SetPort(2022)
	s.assertCommandArgs(c, s.commandOptions([]string{echoCommand, "123"}, &opts),
		fmt.Sprintf("%s -o StrictHostKeyChecking no -o PasswordAuthentication no -p 2022 localhost %s 123",
			s.fakessh, echoCommand),
	)
}

func (s *SSHCommandSuite) TestCopy(c *gc.C) {
	var opts ssh.Options
	opts.EnablePTY()
	opts.AllowPasswordAuthentication()
	opts.SetIdentities("x", "y")
	opts.SetPort(2022)
	err := s.client.Copy([]string{"/tmp/blah", "foo@bar.com:baz"}, &opts)
	c.Assert(err, gc.IsNil)
	out, err := ioutil.ReadFile(s.fakescp + ".args")
	c.Assert(err, gc.IsNil)
	// EnablePTY has no effect for Copy
	c.Assert(string(out), gc.Equals, s.fakescp+" -o StrictHostKeyChecking no -i x -i y -P 2022 /tmp/blah foo@bar.com:baz\n")

	// Try passing extra args
	err = s.client.Copy([]string{"/tmp/blah", "foo@bar.com:baz", "-r", "-v"}, &opts)
	c.Assert(err, gc.IsNil)
	out, err = ioutil.ReadFile(s.fakescp + ".args")
	c.Assert(err, gc.IsNil)
	c.Assert(string(out), gc.Equals, s.fakescp+" -o StrictHostKeyChecking no -i x -i y -P 2022 /tmp/blah foo@bar.com:baz -r -v\n")

	// Try interspersing extra args
	err = s.client.Copy([]string{"-r", "/tmp/blah", "-v", "foo@bar.com:baz"}, &opts)
	c.Assert(err, gc.IsNil)
	out, err = ioutil.ReadFile(s.fakescp + ".args")
	c.Assert(err, gc.IsNil)
	c.Assert(string(out), gc.Equals, s.fakescp+" -o StrictHostKeyChecking no -i x -i y -P 2022 -r /tmp/blah -v foo@bar.com:baz\n")
}

func (s *SSHCommandSuite) TestCommandClientKeys(c *gc.C) {
	defer overrideGenerateKey(c).Restore()
	clientKeysDir := c.MkDir()
	defer ssh.ClearClientKeys()
	err := ssh.LoadClientKeys(clientKeysDir)
	c.Assert(err, gc.IsNil)
	ck := filepath.Join(clientKeysDir, "juju_id_rsa")
	var opts ssh.Options
	opts.SetIdentities("x", "y")
	s.assertCommandArgs(c, s.commandOptions([]string{echoCommand, "123"}, &opts),
		fmt.Sprintf("%s -o StrictHostKeyChecking no -o PasswordAuthentication no -i x -i y -i %s localhost %s 123",
			s.fakessh, ck, echoCommand),
	)
}

func (s *SSHCommandSuite) TestCommandError(c *gc.C) {
	var opts ssh.Options
	err := ioutil.WriteFile(s.fakessh, []byte("#!/bin/sh\nexit 42"), 0755)
	c.Assert(err, gc.IsNil)
	command := s.client.Command("ignored", []string{echoCommand, "foo"}, &opts)
	err = command.Run()
	c.Assert(cmd.IsRcPassthroughError(err), gc.Equals, true)
}

func (s *SSHCommandSuite) TestCommandDefaultIdentities(c *gc.C) {
	var opts ssh.Options
	tempdir := c.MkDir()
	def1 := filepath.Join(tempdir, "def1")
	def2 := filepath.Join(tempdir, "def2")
	s.PatchValue(ssh.DefaultIdentities, []string{def1, def2})
	// If no identities are specified, then the defaults aren't added.
	s.assertCommandArgs(c, s.commandOptions([]string{echoCommand, "123"}, &opts),
		fmt.Sprintf("%s -o StrictHostKeyChecking no -o PasswordAuthentication no localhost %s 123",
			s.fakessh, echoCommand),
	)
	// If identities are specified, then the defaults are must added.
	// Only the defaults that exist on disk will be added.
	err := ioutil.WriteFile(def2, nil, 0644)
	c.Assert(err, gc.IsNil)
	opts.SetIdentities("x", "y")
	s.assertCommandArgs(c, s.commandOptions([]string{echoCommand, "123"}, &opts),
		fmt.Sprintf("%s -o StrictHostKeyChecking no -o PasswordAuthentication no -i x -i y -i %s localhost %s 123",
			s.fakessh, def2, echoCommand),
	)
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)

package fs

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"

	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/syncutil"
	"golang.org/x/net/context"

	"bazil.org/fuse"
	fusefs "bazil.org/fuse/fs"
)

// A remote object's name and metadata, along with a local temporary file that
// contains its contents (when initialized).
//
// TODO(jacobsa): After becoming comfortable with the representation of dir and
// its concurrency protection, audit this file and make sure it is up to par.
type file struct {
	/////////////////////////
	// Dependencies
	/////////////////////////

	logger *log.Logger
	bucket gcs.Bucket

	/////////////////////////
	// Constant data
	/////////////////////////

	objectName string

	/////////////////////////
	// Mutable state
	/////////////////////////

	mu syncutil.InvariantMutex

	// A local temporary file containing the current contents of the logical
	// file. Lazily created. When non-ni, this is authoritative.
	tempFile *os.File // GUARDED_BY(mu)

	// Set to true when we need to flush tempFile to GCS before allowing the user
	// to successfully close the file. false implies that the GCS object is up to
	// date (or has been modified only by a foreign machine).
	//
	// INVARIANT: If true, then tempFile != nil
	tempFileDirty bool // GUARDED_BY(mu)

	// When tempFile == nil, the current size of the object named objectName on
	// GCS, as far as we are aware.
	//
	// INVARIANT: If tempFile != nil, then remoteSize == 0
	remoteSize uint64 // GUARDED_BY(mu)
}

// Make sure file implements the interfaces we think it does.
var (
	_ fusefs.Node = &file{}

	_ fusefs.Handle         = &file{}
	_ fusefs.HandleFlusher  = &file{}
	_ fusefs.HandleReader   = &file{}
	_ fusefs.HandleReleaser = &file{}
	_ fusefs.HandleWriter   = &file{}
)

func newFile(
	logger *log.Logger,
	bucket gcs.Bucket,
	objectName string,
	remoteSize uint64) *file {
	f := &file{
		logger:     logger,
		bucket:     bucket,
		objectName: objectName,
		remoteSize: remoteSize,
	}

	f.mu = syncutil.NewInvariantMutex(func() { f.checkInvariants() })

	return f
}

func (f *file) checkInvariants() {
	if f.tempFileDirty && f.tempFile == nil {
		panic("Expected !tempFileDirty when tempFile == nil.")
	}

	if f.tempFile != nil && f.remoteSize != 0 {
		panic("Expected remoteSize == 0 when tempFile != nil.")
	}
}

func (f *file) Attr() fuse.Attr {
	return fuse.Attr{
		// TODO(jacobsa): Expose ACLs from GCS?
		Mode: 0400,
		// TODO(jacobsa): Catch the bug here (that this may be wrong when
		// f.tempFile != nil) with a test, then fix it.
		Size: f.remoteSize,
	}
}

// If the file contents have not yet been fetched to a temporary file, fetch
// them.
//
// EXCLUSIVE_LOCKS_REQUIRED(f.mu)
func (f *file) ensureTempFile(ctx context.Context) error {
	// Do we already have a file?
	if f.tempFile != nil {
		return nil
	}

	// Create a temporary file.
	tempFile, err := ioutil.TempFile("", "gcsfuse")
	if err != nil {
		return fmt.Errorf("ioutil.TempFile: %v", err)
	}

	// Create a reader for the object.
	readCloser, err := f.bucket.NewReader(ctx, f.objectName)
	if err != nil {
		return fmt.Errorf("bucket.NewReader: %v", err)
	}

	defer readCloser.Close()

	// Copy the object contents into the file.
	if _, err := io.Copy(tempFile, readCloser); err != nil {
		return fmt.Errorf("io.Copy: %v", err)
	}

	// Save the file for later.
	f.tempFile = tempFile

	// remoteSize is no longer authoritative.
	f.remoteSize = 0

	return nil
}

// Throw away the local temporary file, if any.
//
// LOCKS_EXCLUDED(f.mu)
func (f *file) Release(ctx context.Context, req *fuse.ReleaseRequest) error {
	f.mu.Lock()
	defer f.mu.Unlock()

	// Is there a file to close?
	if f.tempFile == nil {
		return nil
	}

	// Close it, after grabbing its path.
	path := f.tempFile.Name()
	if err := f.tempFile.Close(); err != nil {
		f.logger.Println("Error closing temp file:", err)
	}

	// Attempt to delete it.
	if err := os.Remove(path); err != nil {
		f.logger.Println("Error deleting temp file:", err)
	}

	f.tempFile = nil
	return nil
}

// Ensure that the local temporary file is initialized, then read from it.
//
// LOCKS_EXCLUDED(f.mu)
func (f *file) Read(
	ctx context.Context,
	req *fuse.ReadRequest,
	resp *fuse.ReadResponse) error {
	f.mu.Lock()
	defer f.mu.Unlock()

	// Ensure the temp file is present.
	if err := f.ensureTempFile(ctx); err != nil {
		return err
	}

	// Allocate a response buffer.
	resp.Data = make([]byte, req.Size)

	// Read the data.
	n, err := f.tempFile.ReadAt(resp.Data, req.Offset)
	resp.Data = resp.Data[:n]

	// Special case: read(2) doesn't return EOF errors.
	if err == io.EOF {
		err = nil
	}

	return err
}

// Ensure that the local temporary file is initialized, then write to it.
//
// LOCKS_EXCLUDED(f.mu)
func (f *file) Write(
	ctx context.Context,
	req *fuse.WriteRequest,
	resp *fuse.WriteResponse) (err error) {
	f.mu.Lock()
	defer f.mu.Unlock()

	// Ensure the temp file is present. If it's not, grab the current contents
	// from GCS.
	if err = f.ensureTempFile(ctx); err != nil {
		err = fmt.Errorf("ensureTempFile: %v", err)
		return
	}

	// Write to the temp file.
	resp.Size, err = f.tempFile.WriteAt(req.Data, req.Offset)
	return
}

// Put the temporary file back in the bucket if it's dirty.
//
// LOCKS_EXCLUDED(f.mu)
func (f *file) Flush(
	ctx context.Context,
	req *fuse.FlushRequest) (err error) {
	// Is there anything interesting for us to do?
	if !f.tempFileDirty {
		return
	}

	err = errors.New("TODO(jacobsa): file.Flush.")
	return
}
<commit_msg>Fixed handling of tempFileDirty.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)

package fs

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"

	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/syncutil"
	"golang.org/x/net/context"

	"bazil.org/fuse"
	fusefs "bazil.org/fuse/fs"
)

// A remote object's name and metadata, along with a local temporary file that
// contains its contents (when initialized).
//
// TODO(jacobsa): After becoming comfortable with the representation of dir and
// its concurrency protection, audit this file and make sure it is up to par.
type file struct {
	/////////////////////////
	// Dependencies
	/////////////////////////

	logger *log.Logger
	bucket gcs.Bucket

	/////////////////////////
	// Constant data
	/////////////////////////

	objectName string

	/////////////////////////
	// Mutable state
	/////////////////////////

	mu syncutil.InvariantMutex

	// A local temporary file containing the current contents of the logical
	// file. Lazily created. When non-ni, this is authoritative.
	tempFile *os.File // GUARDED_BY(mu)

	// Set to true when we need to flush tempFile to GCS before allowing the user
	// to successfully close the file. false implies that the GCS object is up to
	// date (or has been modified only by a foreign machine).
	//
	// INVARIANT: If true, then tempFile != nil
	tempFileDirty bool // GUARDED_BY(mu)

	// When tempFile == nil, the current size of the object named objectName on
	// GCS, as far as we are aware.
	//
	// INVARIANT: If tempFile != nil, then remoteSize == 0
	remoteSize uint64 // GUARDED_BY(mu)
}

// Make sure file implements the interfaces we think it does.
var (
	_ fusefs.Node = &file{}

	_ fusefs.Handle         = &file{}
	_ fusefs.HandleFlusher  = &file{}
	_ fusefs.HandleReader   = &file{}
	_ fusefs.HandleReleaser = &file{}
	_ fusefs.HandleWriter   = &file{}
)

func newFile(
	logger *log.Logger,
	bucket gcs.Bucket,
	objectName string,
	remoteSize uint64) *file {
	f := &file{
		logger:     logger,
		bucket:     bucket,
		objectName: objectName,
		remoteSize: remoteSize,
	}

	f.mu = syncutil.NewInvariantMutex(func() { f.checkInvariants() })

	return f
}

func (f *file) checkInvariants() {
	if f.tempFileDirty && f.tempFile == nil {
		panic("Expected !tempFileDirty when tempFile == nil.")
	}

	if f.tempFile != nil && f.remoteSize != 0 {
		panic("Expected remoteSize == 0 when tempFile != nil.")
	}
}

func (f *file) Attr() fuse.Attr {
	return fuse.Attr{
		// TODO(jacobsa): Expose ACLs from GCS?
		Mode: 0400,
		// TODO(jacobsa): Catch the bug here (that this may be wrong when
		// f.tempFile != nil) with a test, then fix it.
		Size: f.remoteSize,
	}
}

// If the file contents have not yet been fetched to a temporary file, fetch
// them.
//
// EXCLUSIVE_LOCKS_REQUIRED(f.mu)
func (f *file) ensureTempFile(ctx context.Context) error {
	// Do we already have a file?
	if f.tempFile != nil {
		return nil
	}

	// Create a temporary file.
	tempFile, err := ioutil.TempFile("", "gcsfuse")
	if err != nil {
		return fmt.Errorf("ioutil.TempFile: %v", err)
	}

	// Create a reader for the object.
	readCloser, err := f.bucket.NewReader(ctx, f.objectName)
	if err != nil {
		return fmt.Errorf("bucket.NewReader: %v", err)
	}

	defer readCloser.Close()

	// Copy the object contents into the file.
	if _, err := io.Copy(tempFile, readCloser); err != nil {
		return fmt.Errorf("io.Copy: %v", err)
	}

	// Save the file for later.
	f.tempFile = tempFile

	// remoteSize is no longer authoritative.
	f.remoteSize = 0

	return nil
}

// Throw away the local temporary file, if any.
//
// LOCKS_EXCLUDED(f.mu)
func (f *file) Release(ctx context.Context, req *fuse.ReleaseRequest) error {
	f.mu.Lock()
	defer f.mu.Unlock()

	// Is there a file to close?
	if f.tempFile == nil {
		return nil
	}

	// Close it, after grabbing its path.
	path := f.tempFile.Name()
	if err := f.tempFile.Close(); err != nil {
		f.logger.Println("Error closing temp file:", err)
	}

	// Attempt to delete it.
	if err := os.Remove(path); err != nil {
		f.logger.Println("Error deleting temp file:", err)
	}

	f.tempFile = nil
	f.tempFileDirty = false

	return nil
}

// Ensure that the local temporary file is initialized, then read from it.
//
// LOCKS_EXCLUDED(f.mu)
func (f *file) Read(
	ctx context.Context,
	req *fuse.ReadRequest,
	resp *fuse.ReadResponse) error {
	f.mu.Lock()
	defer f.mu.Unlock()

	// Ensure the temp file is present.
	if err := f.ensureTempFile(ctx); err != nil {
		return err
	}

	// Allocate a response buffer.
	resp.Data = make([]byte, req.Size)

	// Read the data.
	n, err := f.tempFile.ReadAt(resp.Data, req.Offset)
	resp.Data = resp.Data[:n]

	// Special case: read(2) doesn't return EOF errors.
	if err == io.EOF {
		err = nil
	}

	return err
}

// Ensure that the local temporary file is initialized, then write to it.
//
// LOCKS_EXCLUDED(f.mu)
func (f *file) Write(
	ctx context.Context,
	req *fuse.WriteRequest,
	resp *fuse.WriteResponse) (err error) {
	f.mu.Lock()
	defer f.mu.Unlock()

	// Ensure the temp file is present. If it's not, grab the current contents
	// from GCS.
	if err = f.ensureTempFile(ctx); err != nil {
		err = fmt.Errorf("ensureTempFile: %v", err)
		return
	}

	// Mark us dirty.
	f.tempFileDirty = true

	// Write to the temp file.
	resp.Size, err = f.tempFile.WriteAt(req.Data, req.Offset)

	return
}

// Put the temporary file back in the bucket if it's dirty.
//
// LOCKS_EXCLUDED(f.mu)
func (f *file) Flush(
	ctx context.Context,
	req *fuse.FlushRequest) (err error) {
	// Is there anything interesting for us to do?
	if !f.tempFileDirty {
		return
	}

	err = errors.New("TODO(jacobsa): file.Flush.")
	return
}
<|endoftext|>


<commit_before>// Copyright 2014 tsuru-client authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"archive/tar"
	"bytes"
	"compress/gzip"
	"io"
	"io/ioutil"
	"net/http"

	"github.com/tsuru/tsuru/cmd"
	"github.com/tsuru/tsuru/cmd/testing"
	"github.com/tsuru/tsuru/cmd/tsuru-base"
	"launchpad.net/gocheck"
)

func (s *S) TestDeployInfo(c *gocheck.C) {
	desc := `Deploys set of files and/or directories to tsuru server. Some examples of calls are:

tsuru deploy .
tsuru deploy myfile.jar Procfile
`
	expected := &cmd.Info{
		Name:    "deploy",
		Usage:   "deploy [-a/--app <appname>] <file-or-dir-1> [file-or-dir-2] ... [file-or-dir-n]",
		Desc:    desc,
		MinArgs: 1,
	}
	cmd := deploy{}
	c.Assert(cmd.Info(), gocheck.DeepEquals, expected)
}

func (s *S) TestDeployRun(c *gocheck.C) {
	var called bool
	var buf bytes.Buffer
	err := targz(nil, &buf, "testdata")
	c.Assert(err, gocheck.IsNil)
	trans := testing.ConditionalTransport{
		Transport: testing.Transport{Message: "deploy worked\nOK\n", Status: http.StatusOK},
		CondFunc: func(req *http.Request) bool {
			defer req.Body.Close()
			called = true
			file, _, err := req.FormFile("file")
			c.Assert(err, gocheck.IsNil)
			content, err := ioutil.ReadAll(file)
			c.Assert(err, gocheck.IsNil)
			c.Assert(content, gocheck.DeepEquals, buf.Bytes())
			return req.Method == "POST" && req.URL.Path == "/apps/secret/deploy"
		},
	}
	client := cmd.NewClient(&http.Client{Transport: &trans}, nil, manager)
	var stdout, stderr bytes.Buffer
	context := cmd.Context{
		Stdout: &stdout,
		Stderr: &stderr,
		Args:   []string{"testdata", ".."},
	}
	fake := FakeGuesser{name: "secret"}
	guessCommand := tsuru.GuessingCommand{G: &fake}
	cmd := deploy{GuessingCommand: guessCommand}
	err = cmd.Run(&context, client)
	c.Assert(err, gocheck.IsNil)
	c.Assert(called, gocheck.Equals, true)
}

func (s *S) TestDeployRunNotOK(c *gocheck.C) {
	trans := testing.Transport{Message: "deploy worked\n", Status: http.StatusOK}
	client := cmd.NewClient(&http.Client{Transport: &trans}, nil, manager)
	var stdout, stderr bytes.Buffer
	context := cmd.Context{
		Stdout: &stdout,
		Stderr: &stderr,
		Args:   []string{"testdata", ".."},
	}
	fake := FakeGuesser{name: "secret"}
	guessCommand := tsuru.GuessingCommand{G: &fake}
	command := deploy{GuessingCommand: guessCommand}
	err := command.Run(&context, client)
	c.Assert(err, gocheck.Equals, cmd.ErrAbortCommand)
}

func (s *S) TestDeployRunFileNotFound(c *gocheck.C) {
	var stdout, stderr bytes.Buffer
	context := cmd.Context{
		Stdout: &stdout,
		Stderr: &stderr,
		Args:   []string{"/tmp/something/that/doesnt/really/exist/im/sure"},
	}
	fake := FakeGuesser{name: "secret"}
	guessCommand := tsuru.GuessingCommand{G: &fake}
	command := deploy{GuessingCommand: guessCommand}
	err := command.Run(&context, nil)
	c.Assert(err, gocheck.NotNil)
}

func (s *S) TestDeployRunRequestFailure(c *gocheck.C) {
	trans := testing.Transport{Message: "app not found\n", Status: http.StatusNotFound}
	client := cmd.NewClient(&http.Client{Transport: &trans}, nil, manager)
	var stdout, stderr bytes.Buffer
	context := cmd.Context{
		Stdout: &stdout,
		Stderr: &stderr,
		Args:   []string{"testdata", ".."},
	}
	fake := FakeGuesser{name: "secret"}
	guessCommand := tsuru.GuessingCommand{G: &fake}
	command := deploy{GuessingCommand: guessCommand}
	err := command.Run(&context, client)
	c.Assert(err, gocheck.NotNil)
	c.Assert(err.Error(), gocheck.Equals, "app not found\n")
}

func (s *S) TestTargz(c *gocheck.C) {
	var buf bytes.Buffer
	ctx := cmd.Context{Stderr: &buf}
	var gzipBuf, tarBuf bytes.Buffer
	err := targz(&ctx, &gzipBuf, "testdata", "..")
	c.Assert(err, gocheck.IsNil)
	gzipReader, err := gzip.NewReader(&gzipBuf)
	c.Assert(err, gocheck.IsNil)
	_, err = io.Copy(&tarBuf, gzipReader)
	c.Assert(err, gocheck.IsNil)
	tarReader := tar.NewReader(&tarBuf)
	var headers []string
	var contents []string
	for header, err := tarReader.Next(); err == nil; header, err = tarReader.Next() {
		headers = append(headers, header.Name)
		if !header.FileInfo().IsDir() {
			content, err := ioutil.ReadAll(tarReader)
			c.Assert(err, gocheck.IsNil)
			contents = append(contents, string(content))
		}
	}
	expected := []string{
		"testdata", "testdata/directory", "testdata/directory/file.txt",
		"testdata/file1.txt", "testdata/file2.txt",
	}
	c.Assert(headers, gocheck.DeepEquals, expected)
	expectedContents := []string{"wat\n", "something happened\n", "twice\n"}
	c.Assert(contents, gocheck.DeepEquals, expectedContents)
	c.Assert(buf.String(), gocheck.Equals, `Warning: skipping ".."`)
}

func (s *S) TestTargzFailure(c *gocheck.C) {
	var stderr bytes.Buffer
	ctx := cmd.Context{Stderr: &stderr}
	var buf bytes.Buffer
	err := targz(&ctx, &buf, "/tmp/something/that/definitely/doesnt/exist/right", "testdata")
	c.Assert(err, gocheck.NotNil)
	c.Assert(err.Error(), gocheck.Equals, "stat /tmp/something/that/definitely/doesnt/exist/right: no such file or directory")
}
<commit_msg>tsuru/deploy_test: sort strings before comparing<commit_after>// Copyright 2014 tsuru-client authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"archive/tar"
	"bytes"
	"compress/gzip"
	"io"
	"io/ioutil"
	"net/http"
	"sort"

	"github.com/tsuru/tsuru/cmd"
	"github.com/tsuru/tsuru/cmd/testing"
	"github.com/tsuru/tsuru/cmd/tsuru-base"
	"launchpad.net/gocheck"
)

func (s *S) TestDeployInfo(c *gocheck.C) {
	desc := `Deploys set of files and/or directories to tsuru server. Some examples of calls are:

tsuru deploy .
tsuru deploy myfile.jar Procfile
`
	expected := &cmd.Info{
		Name:    "deploy",
		Usage:   "deploy [-a/--app <appname>] <file-or-dir-1> [file-or-dir-2] ... [file-or-dir-n]",
		Desc:    desc,
		MinArgs: 1,
	}
	cmd := deploy{}
	c.Assert(cmd.Info(), gocheck.DeepEquals, expected)
}

func (s *S) TestDeployRun(c *gocheck.C) {
	var called bool
	var buf bytes.Buffer
	err := targz(nil, &buf, "testdata")
	c.Assert(err, gocheck.IsNil)
	trans := testing.ConditionalTransport{
		Transport: testing.Transport{Message: "deploy worked\nOK\n", Status: http.StatusOK},
		CondFunc: func(req *http.Request) bool {
			defer req.Body.Close()
			called = true
			file, _, err := req.FormFile("file")
			c.Assert(err, gocheck.IsNil)
			content, err := ioutil.ReadAll(file)
			c.Assert(err, gocheck.IsNil)
			c.Assert(content, gocheck.DeepEquals, buf.Bytes())
			return req.Method == "POST" && req.URL.Path == "/apps/secret/deploy"
		},
	}
	client := cmd.NewClient(&http.Client{Transport: &trans}, nil, manager)
	var stdout, stderr bytes.Buffer
	context := cmd.Context{
		Stdout: &stdout,
		Stderr: &stderr,
		Args:   []string{"testdata", ".."},
	}
	fake := FakeGuesser{name: "secret"}
	guessCommand := tsuru.GuessingCommand{G: &fake}
	cmd := deploy{GuessingCommand: guessCommand}
	err = cmd.Run(&context, client)
	c.Assert(err, gocheck.IsNil)
	c.Assert(called, gocheck.Equals, true)
}

func (s *S) TestDeployRunNotOK(c *gocheck.C) {
	trans := testing.Transport{Message: "deploy worked\n", Status: http.StatusOK}
	client := cmd.NewClient(&http.Client{Transport: &trans}, nil, manager)
	var stdout, stderr bytes.Buffer
	context := cmd.Context{
		Stdout: &stdout,
		Stderr: &stderr,
		Args:   []string{"testdata", ".."},
	}
	fake := FakeGuesser{name: "secret"}
	guessCommand := tsuru.GuessingCommand{G: &fake}
	command := deploy{GuessingCommand: guessCommand}
	err := command.Run(&context, client)
	c.Assert(err, gocheck.Equals, cmd.ErrAbortCommand)
}

func (s *S) TestDeployRunFileNotFound(c *gocheck.C) {
	var stdout, stderr bytes.Buffer
	context := cmd.Context{
		Stdout: &stdout,
		Stderr: &stderr,
		Args:   []string{"/tmp/something/that/doesnt/really/exist/im/sure"},
	}
	fake := FakeGuesser{name: "secret"}
	guessCommand := tsuru.GuessingCommand{G: &fake}
	command := deploy{GuessingCommand: guessCommand}
	err := command.Run(&context, nil)
	c.Assert(err, gocheck.NotNil)
}

func (s *S) TestDeployRunRequestFailure(c *gocheck.C) {
	trans := testing.Transport{Message: "app not found\n", Status: http.StatusNotFound}
	client := cmd.NewClient(&http.Client{Transport: &trans}, nil, manager)
	var stdout, stderr bytes.Buffer
	context := cmd.Context{
		Stdout: &stdout,
		Stderr: &stderr,
		Args:   []string{"testdata", ".."},
	}
	fake := FakeGuesser{name: "secret"}
	guessCommand := tsuru.GuessingCommand{G: &fake}
	command := deploy{GuessingCommand: guessCommand}
	err := command.Run(&context, client)
	c.Assert(err, gocheck.NotNil)
	c.Assert(err.Error(), gocheck.Equals, "app not found\n")
}

func (s *S) TestTargz(c *gocheck.C) {
	var buf bytes.Buffer
	ctx := cmd.Context{Stderr: &buf}
	var gzipBuf, tarBuf bytes.Buffer
	err := targz(&ctx, &gzipBuf, "testdata", "..")
	c.Assert(err, gocheck.IsNil)
	gzipReader, err := gzip.NewReader(&gzipBuf)
	c.Assert(err, gocheck.IsNil)
	_, err = io.Copy(&tarBuf, gzipReader)
	c.Assert(err, gocheck.IsNil)
	tarReader := tar.NewReader(&tarBuf)
	var headers []string
	var contents []string
	for header, err := tarReader.Next(); err == nil; header, err = tarReader.Next() {
		headers = append(headers, header.Name)
		if !header.FileInfo().IsDir() {
			content, err := ioutil.ReadAll(tarReader)
			c.Assert(err, gocheck.IsNil)
			contents = append(contents, string(content))
		}
	}
	expected := []string{
		"testdata", "testdata/directory", "testdata/directory/file.txt",
		"testdata/file1.txt", "testdata/file2.txt",
	}
	sort.Strings(expected)
	sort.Strings(headers)
	c.Assert(headers, gocheck.DeepEquals, expected)
	expectedContents := []string{"wat\n", "something happened\n", "twice\n"}
	sort.Strings(expectedContents)
	sort.Strings(contents)
	c.Assert(contents, gocheck.DeepEquals, expectedContents)
	c.Assert(buf.String(), gocheck.Equals, `Warning: skipping ".."`)
}

func (s *S) TestTargzFailure(c *gocheck.C) {
	var stderr bytes.Buffer
	ctx := cmd.Context{Stderr: &stderr}
	var buf bytes.Buffer
	err := targz(&ctx, &buf, "/tmp/something/that/definitely/doesnt/exist/right", "testdata")
	c.Assert(err, gocheck.NotNil)
	c.Assert(err.Error(), gocheck.Equals, "stat /tmp/something/that/definitely/doesnt/exist/right: no such file or directory")
}
<|endoftext|>


<commit_before>package govuk_crawler_worker_test

import (
	. "github.com/alphagov/govuk_crawler_worker"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"github.com/fzzy/radix/redis"
)

var _ = Describe("TTLHashSet", func() {
	prefix := "govuk_mirror_crawler_test"

	It("returns an error when asking for a TTLHashSet object that can't connect to redis", func() {
		ttlHashSet, err := NewTTLHashSet(prefix, "127.0.0.1:20000")

		Expect(err).ToNot(BeNil())
		Expect(ttlHashSet).To(BeNil())
	})

	Describe("Working with a redis service", func() {
		var (
			ttlHashSet    *TTLHashSet
			ttlHashSetErr error
		)

		BeforeEach(func() {
			ttlHashSet, ttlHashSetErr = NewTTLHashSet(prefix, "127.0.0.1:6379")
		})

		AfterEach(func() {
			Expect(ttlHashSet.Close()).To(BeNil())
			Expect(purgeAllKeys(prefix, "127.0.0.1:6379"))
		})

		It("should connect successfully with no errors", func() {
			Expect(ttlHashSetErr).To(BeNil())
			Expect(ttlHashSet).NotTo(BeNil())
		})

		It("should return false when a key doesn't exist", func() {
			exists, err := ttlHashSet.Exists("foobar")

			Expect(err).To(BeNil())
			Expect(exists).To(Equal(false))
		})

		It("exposes a way of adding a key to redis", func() {
			key := "foo.bar.baz"
			added, addedErr := ttlHashSet.Add(key)

			Expect(addedErr).To(BeNil())
			Expect(added).To(Equal(true))

			exists, existsErr := ttlHashSet.Exists(key)

			Expect(existsErr).To(BeNil())
			Expect(exists).To(Equal(true))
		})

		Describe("TTL()", func() {
			It("should return a negative TTL on a non-existent key", func() {
				ttl, err := ttlHashSet.TTL("this.key.does.not.exist")

				Expect(err).To(BeNil())
				Expect(ttl).To(Equal(-2))
			})

			It("should expose a positive TTL on key that exists", func() {
				key := "some.ttl.key"
				added, addErr := ttlHashSet.Add(key)

				Expect(addErr).To(BeNil())
				Expect(added).To(Equal(true))

				ttl, err := ttlHashSet.TTL(key)

				Expect(err).To(BeNil())
				Expect(ttl).To(BeNumerically(">", 1000))
			})
		})
	})
})

func purgeAllKeys(prefix string, address string) error {
	client, err := redis.Dial("tcp", address)
	if err != nil {
		return err
	}

	keys, err := client.Cmd("KEYS", prefix + "*").List()
	if err != nil {
		return err
	}

	reply := client.Cmd("DEL", keys)
	if reply.Err != nil {
		return reply.Err
	}

	return nil
}
<commit_msg>Run code through `go fmt`<commit_after>package govuk_crawler_worker_test

import (
	. "github.com/alphagov/govuk_crawler_worker"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"github.com/fzzy/radix/redis"
)

var _ = Describe("TTLHashSet", func() {
	prefix := "govuk_mirror_crawler_test"

	It("returns an error when asking for a TTLHashSet object that can't connect to redis", func() {
		ttlHashSet, err := NewTTLHashSet(prefix, "127.0.0.1:20000")

		Expect(err).ToNot(BeNil())
		Expect(ttlHashSet).To(BeNil())
	})

	Describe("Working with a redis service", func() {
		var (
			ttlHashSet    *TTLHashSet
			ttlHashSetErr error
		)

		BeforeEach(func() {
			ttlHashSet, ttlHashSetErr = NewTTLHashSet(prefix, "127.0.0.1:6379")
		})

		AfterEach(func() {
			Expect(ttlHashSet.Close()).To(BeNil())
			Expect(purgeAllKeys(prefix, "127.0.0.1:6379"))
		})

		It("should connect successfully with no errors", func() {
			Expect(ttlHashSetErr).To(BeNil())
			Expect(ttlHashSet).NotTo(BeNil())
		})

		It("should return false when a key doesn't exist", func() {
			exists, err := ttlHashSet.Exists("foobar")

			Expect(err).To(BeNil())
			Expect(exists).To(Equal(false))
		})

		It("exposes a way of adding a key to redis", func() {
			key := "foo.bar.baz"
			added, addedErr := ttlHashSet.Add(key)

			Expect(addedErr).To(BeNil())
			Expect(added).To(Equal(true))

			exists, existsErr := ttlHashSet.Exists(key)

			Expect(existsErr).To(BeNil())
			Expect(exists).To(Equal(true))
		})

		Describe("TTL()", func() {
			It("should return a negative TTL on a non-existent key", func() {
				ttl, err := ttlHashSet.TTL("this.key.does.not.exist")

				Expect(err).To(BeNil())
				Expect(ttl).To(Equal(-2))
			})

			It("should expose a positive TTL on key that exists", func() {
				key := "some.ttl.key"
				added, addErr := ttlHashSet.Add(key)

				Expect(addErr).To(BeNil())
				Expect(added).To(Equal(true))

				ttl, err := ttlHashSet.TTL(key)

				Expect(err).To(BeNil())
				Expect(ttl).To(BeNumerically(">", 1000))
			})
		})
	})
})

func purgeAllKeys(prefix string, address string) error {
	client, err := redis.Dial("tcp", address)
	if err != nil {
		return err
	}

	keys, err := client.Cmd("KEYS", prefix+"*").List()
	if err != nil {
		return err
	}

	reply := client.Cmd("DEL", keys)
	if reply.Err != nil {
		return reply.Err
	}

	return nil
}
<|endoftext|>


<commit_before>package stats

import (
	"fmt"
	"os"
	"strings"

	"github.com/projecteru2/core/types"

	statsdlib "github.com/CMGS/statsd"
	log "github.com/Sirupsen/logrus"
)

const (
	memStats    = "eru-core.%s.mem"
	deployCount = "eru-core.deploy.count"
)

type statsdClient struct {
	Addr     string
	Hostname string
}

func (s *statsdClient) gauge(keyPattern string, data map[string]float64) error {
	remote, err := statsdlib.New(s.Addr)
	if err != nil {
		log.Errorf("[gauge] Connect statsd failed: %v", err)
		return err
	}
	defer remote.Close()
	defer remote.Flush()
	for k, v := range data {
		key := fmt.Sprintf(keyPattern, k)
		remote.Gauge(key, v)
	}
	return nil
}

func (s *statsdClient) count(key string, n int, rate float32) error {
	remote, err := statsdlib.New(s.Addr)
	if err != nil {
		log.Errorf("[count] Connect statsd failed: %v", err)
		return err
	}
	defer remote.Close()
	defer remote.Flush()
	remote.Count(key, n, rate)
	return nil
}

func (s *statsdClient) isNotSet() bool {
	return s.Addr == ""
}

func (s *statsdClient) SendMemCap(cpumemmap map[string]types.CPUAndMem) {
	if s.isNotSet() {
		return
	}
	data := map[string]float64{}
	for node, cpuandmem := range cpumemmap {
		data[node] = float64(cpuandmem.MemCap)
	}

	keyPattern := fmt.Sprintf(memStats, s.Hostname)
	if err := s.gauge(keyPattern, data); err != nil {
		log.Errorf("[SendMemCap] Error occured while sending data to statsd: %v", err)
	}
}

func (s *statsdClient) SendDeployCount(n int) {
	if s.isNotSet() {
		return
	}
	if err := s.count(deployCount, n, 1.0); err != nil {
		log.Errorf("[SendDeployCount] Error occured while counting: %v", err)
	}
}

//Client ref to statsd client
var Client = statsdClient{}

//NewStatsdClient make a client
func NewStatsdClient(addr string) {
	hostname, _ := os.Hostname()
	cleanHost := strings.Replace(hostname, ".", "-", -1)
	Client = statsdClient{addr, cleanHost}
}
<commit_msg>fix gauge format bug<commit_after>package stats

import (
	"fmt"
	"os"
	"strings"

	"github.com/projecteru2/core/types"

	statsdlib "github.com/CMGS/statsd"
	log "github.com/Sirupsen/logrus"
)

const (
	memStats    = "eru-core.%s.mem"
	deployCount = "eru-core.deploy.count"
)

type statsdClient struct {
	Addr     string
	Hostname string
}

func (s *statsdClient) gauge(keyPattern string, data map[string]float64) error {
	remote, err := statsdlib.New(s.Addr)
	if err != nil {
		log.Errorf("[gauge] Connect statsd failed: %v", err)
		return err
	}
	defer remote.Close()
	defer remote.Flush()
	for k, v := range data {
		key := fmt.Sprintf("%s.%s", keyPattern, k)
		remote.Gauge(key, v)
	}
	return nil
}

func (s *statsdClient) count(key string, n int, rate float32) error {
	remote, err := statsdlib.New(s.Addr)
	if err != nil {
		log.Errorf("[count] Connect statsd failed: %v", err)
		return err
	}
	defer remote.Close()
	defer remote.Flush()
	remote.Count(key, n, rate)
	return nil
}

func (s *statsdClient) isNotSet() bool {
	return s.Addr == ""
}

func (s *statsdClient) SendMemCap(cpumemmap map[string]types.CPUAndMem) {
	if s.isNotSet() {
		return
	}
	data := map[string]float64{}
	for node, cpuandmem := range cpumemmap {
		data[node] = float64(cpuandmem.MemCap)
	}

	keyPattern := fmt.Sprintf(memStats, s.Hostname)
	if err := s.gauge(keyPattern, data); err != nil {
		log.Errorf("[SendMemCap] Error occured while sending data to statsd: %v", err)
	}
}

func (s *statsdClient) SendDeployCount(n int) {
	if s.isNotSet() {
		return
	}
	if err := s.count(deployCount, n, 1.0); err != nil {
		log.Errorf("[SendDeployCount] Error occured while counting: %v", err)
	}
}

//Client ref to statsd client
var Client = statsdClient{}

//NewStatsdClient make a client
func NewStatsdClient(addr string) {
	hostname, _ := os.Hostname()
	cleanHost := strings.Replace(hostname, ".", "-", -1)
	Client = statsdClient{addr, cleanHost}
}
<|endoftext|>


<commit_before>package graval

import (
	"bufio"
	"fmt"
	"log"
	"net"
	"path/filepath"
	"strconv"
	"strings"
)

const (
	welcomeMessage = "Welcome to the Go FTP Server"
)

type FTPConn struct {
	conn          *net.TCPConn
	controlReader *bufio.Reader
	controlWriter *bufio.Writer
	data          *net.TCPConn
	driver        FTPDriver
	namePrefix    string
	reqUser       string
	user          string
	renameFrom    string
}

// NewFTPConn constructs a new object that will handle the FTP protocol over
// an active net.TCPConn. The TCP connection should already be open before
// it is handed to this functions. driver is an instance of FTPDrive that
// will handle all auth and persistence details.
func NewFTPConn(tcpConn *net.TCPConn, driver FTPDriver) *FTPConn {
	c := new(FTPConn)
	c.namePrefix = "/"
	c.conn = tcpConn
	c.controlReader = bufio.NewReader(tcpConn)
	c.controlWriter = bufio.NewWriter(tcpConn)
	c.driver = driver
	return c
}

// Serve starts an endless loop that reads FTP commands from the client and
// responds appropriately. terminated is a channel that will receive a true
// message when the connection closes. This loop will be running inside a
// goroutine, so use this channel to be notified when the connection can be
// cleaned up.
func (ftpConn *FTPConn) Serve() {
	log.Print("Connection Established")
	// send welcome
	ftpConn.writeMessage(220, welcomeMessage)
	// read commands
	for {
		line, err := ftpConn.controlReader.ReadString('\n')
		if err != nil {
			break
		}
		ftpConn.receiveLine(line)
	}
	log.Print("Connection Terminated")
}

// Close will manually close this connection, even if the client isn't ready.
func (ftpConn *FTPConn) Close() {
	ftpConn.conn.Close()
	if ftpConn.data != nil {
		ftpConn.data.Close()
	}
}

// receiveLine accepts a single line FTP command and co-ordinates an
// appropriate response.
func (ftpConn *FTPConn) receiveLine(line string) {
	log.Print(line)
	command, param := ftpConn.parseLine(line)
	switch command {
	case "ALLO":
		ftpConn.cmdAllo()
		break
	case "CDUP", "XCUP":
		ftpConn.cmdCdup()
		break
	case "CWD", "XCWD":
		ftpConn.cmdCwd(param)
		break
	case "DELE":
		ftpConn.cmdDele(param)
		break
	case "MKD":
		ftpConn.cmdMkd(param)
		break
	case "MODE":
		ftpConn.cmdMode(param)
		break
	case "NOOP":
		ftpConn.cmdNoop()
		break
	case "PASS":
		ftpConn.cmdPass(param)
		break
	case "PWD", "XPWD":
		ftpConn.cmdPwd()
		break
	case "QUIT":
		ftpConn.Close()
		break
	case "RMD", "XRMD":
		ftpConn.cmdRmd(param)
		break
	case "RNFR":
		ftpConn.cmdRnfr(param)
		break
	case "RNTO":
		ftpConn.cmdRnto(param)
		break
	case "SIZE":
		ftpConn.cmdSize(param)
		break
	case "STRU":
		ftpConn.cmdStru(param)
		break
	case "SYST":
		ftpConn.cmdSyst()
		break
	case "TYPE":
		ftpConn.cmdType(param)
		break
	case "USER":
		ftpConn.cmdUser(param)
		break
	default:
		ftpConn.writeMessage(500, "Command not found")
	}
}

// cmdNoop responds to the ALLO FTP command.
//
// This is essentially a ping from the client so we just respond with an
// basic OK message.
func (ftpConn *FTPConn) cmdAllo() {
	ftpConn.writeMessage(202, "Obsolete")
}

// cmdCdup responds to the CDUP FTP command.
//
// Allows the client change their current directory to the parent.
func (ftpConn *FTPConn) cmdCdup() {
	ftpConn.cmdCwd("..")
}

// cmdCwd responds to the CWD FTP command. It allows the client to change the
// current working directory.
func (ftpConn *FTPConn) cmdCwd(param string) {
	path := ftpConn.buildPath(param)
	if ftpConn.driver.ChangeDir(path) {
		ftpConn.namePrefix = path
		ftpConn.writeMessage(250, "Directory changed to " + path)
	} else {
		ftpConn.writeMessage(550, "Action not taken")
	}
}

// cmdDele responds to the DELE FTP command. It allows the client to delete
// a file
func (ftpConn *FTPConn) cmdDele(param string) {
	path := ftpConn.buildPath(param)
	if ftpConn.driver.DeleteFile(path) {
		ftpConn.writeMessage(250, "File deleted")
	} else {
		ftpConn.writeMessage(550, "Action not taken")
	}
}

// cmdMkd responds to the MKD FTP command. It allows the client to create
// a new directory
func (ftpConn *FTPConn) cmdMkd(param string) {
	path := ftpConn.buildPath(param)
	if ftpConn.driver.MakeDir(path) {
		ftpConn.writeMessage(257, "Directory created")
	} else {
		ftpConn.writeMessage(550, "Action not taken")
	}
}

// cmdMode responds to the MODE FTP command.
//
// the original FTP spec had various options for hosts to negotiate how data
// would be sent over the data socket, In reality these days (S)tream mode
// is all that is used for the mode - data is just streamed down the data
// socket unchanged.
func (ftpConn *FTPConn) cmdMode(param string) {
	if strings.ToUpper(param) == "S" {
		ftpConn.writeMessage(200, "OK")
	} else {
		ftpConn.writeMessage(504, "MODE is an obsolete command")
	}
}

// cmdNoop responds to the NOOP FTP command.
//
// This is essentially a ping from the client so we just respond with an
// basic 200 message.
func (ftpConn *FTPConn) cmdNoop() {
	ftpConn.writeMessage(200, "OK")
}

// cmdPass respond to the PASS FTP command by asking the driver if the supplied
// username and password are valid
func (ftpConn *FTPConn) cmdPass(param string) {
	if ftpConn.driver.Authenticate(ftpConn.reqUser, param) {
		ftpConn.user = ftpConn.reqUser
		ftpConn.reqUser = ""
		ftpConn.writeMessage(230, "Password ok, continue")
	} else {
		ftpConn.writeMessage(530, "Incorrect password, not logged in")
	}
}

// cmdPwd responds to the PWD FTP command.
//
// Tells the client what the current working directory is.
func (ftpConn *FTPConn) cmdPwd() {
	ftpConn.writeMessage(257, "\"" + ftpConn.namePrefix + "\" is the current directory")
}

// cmdRmd responds to the RMD FTP command. It allows the client to delete a
// directory.
func (ftpConn *FTPConn) cmdRmd(param string) {
	path := ftpConn.buildPath(param)
	if ftpConn.driver.DeleteDir(path) {
		ftpConn.writeMessage(250, "Directory deleted")
	} else {
		ftpConn.writeMessage(550, "Action not taken")
	}
}

// cmdRnfr responds to the RNFR FTP command. It's the first of two commands
// required for a client to rename a file.
func (ftpConn *FTPConn) cmdRnfr(param string) {
	ftpConn.renameFrom = ftpConn.buildPath(param)
	ftpConn.writeMessage(350, "Requested file action pending further information.")
}

// cmdRnto responds to the RNTO FTP command. It's the second of two commands
// required for a client to rename a file.
func (ftpConn *FTPConn) cmdRnto(param string) {
	toPath := ftpConn.buildPath(param)
	if ftpConn.driver.Rename(ftpConn.renameFrom, toPath) {
		ftpConn.writeMessage(250, "File renamed")
	} else {
		ftpConn.writeMessage(550, "Action not taken")
	}
}

// cmdSize responds to the SIZE FTP command. It returns the size of the
// requested path in bytes.
func (ftpConn *FTPConn) cmdSize(param string) {
	path  := ftpConn.buildPath(param)
	bytes := ftpConn.driver.Bytes(path)
	if bytes >= 0 {
		ftpConn.writeMessage(213, strconv.Itoa(bytes))
	} else {
		ftpConn.writeMessage(450, "file not available")
	}
}

// cmdStru responds to the STRU FTP command.
//
// like the MODE and TYPE commands, stru[cture] dates back to a time when the
// FTP protocol was more aware of the content of the files it was transferring,
// and would sometimes be expected to translate things like EOL markers on the
// fly.
//
// These days files are sent unmodified, and F(ile) mode is the only one we
// really need to support.
func (ftpConn *FTPConn) cmdStru(param string) {
	if strings.ToUpper(param) == "F" {
		ftpConn.writeMessage(200, "OK")
	} else {
		ftpConn.writeMessage(504, "STRU is an obsolete command")
	}
}

// cmdSyst responds to the SYST FTP command by providing a canned response.
func (ftpConn *FTPConn) cmdSyst() {
	ftpConn.writeMessage(215, "UNIX Type: L8")
}

// cmdType responds to the TYPE FTP command.
//
//  like the MODE and STRU commands, TYPE dates back to a time when the FTP
//  protocol was more aware of the content of the files it was transferring, and
//  would sometimes be expected to translate things like EOL markers on the fly.
//
//  Valid options were A(SCII), I(mage), E(BCDIC) or LN (for local type). Since
//  we plan to just accept bytes from the client unchanged, I think Image mode is
//  adequate. The RFC requires we accept ASCII mode however, so accept it, but
//  ignore it.
func (ftpConn *FTPConn) cmdType(param string) {
	if strings.ToUpper(param) == "A" {
		ftpConn.writeMessage(200, "Type set to ASCII")
	} else if strings.ToUpper(param) == "I" {
		ftpConn.writeMessage(200, "Type set to binary")
	} else {
		ftpConn.writeMessage(500, "Invalid type")
	}
}

// cmdUser responds to the USER FTP command by asking for the password
func (ftpConn *FTPConn) cmdUser(param string) {
	ftpConn.reqUser = param
	ftpConn.writeMessage(331, "User name ok, password required")
}

func (ftpConn *FTPConn) parseLine(line string) (string, string) {
	params := strings.SplitN(strings.Trim(line, "\r\n"), " ", 2)
	if len(params) == 1 {
		return params[0], ""
	}
	return params[0], params[1]
}

// writeMessage will send a standard FTP response back to the client.
func (ftpConn *FTPConn) writeMessage(code int, message string) (wrote int, err error) {
	line := fmt.Sprintf("%d %s\r\n", code, message)
	log.Print(line)
	wrote, err = ftpConn.controlWriter.WriteString(line)
	ftpConn.controlWriter.Flush()
	return
}

// buildPath takes a client supplied path or filename and generates a safe
// absolute path withing their account sandbox.
//
//    buildpath("/")
//    => "/"
//    buildpath("one.txt")
//    => "/one.txt"
//    buildpath("/files/two.txt")
//    => "/files/two.txt"
//    buildpath("files/two.txt")
//    => "files/two.txt"
//    buildpath("/../../../../etc/passwd")
//    => "/etc/passwd"
//
// The driver implementation is responsible for deciding how to treat this path.
// Obviously they MUST NOT just read the path off disk. The probably want to
// prefix the path with something to scope the users access to a sandbox.
func (ftpConn *FTPConn) buildPath(filename string) (fullPath string){
	if filename[0:1] == "/" {
		fullPath = filepath.Clean(filename)
	} else if filename != "" && filename != "-a" {
		fullPath = filepath.Clean(ftpConn.namePrefix + "/" + filename)
	} else {
		fullPath = filepath.Clean(ftpConn.namePrefix)
	}
	fullPath = strings.Replace(fullPath, "//", "/", -1)
	return
}
<commit_msg>go switch statements don't have fall through<commit_after>package graval

import (
	"bufio"
	"fmt"
	"log"
	"net"
	"path/filepath"
	"strconv"
	"strings"
)

const (
	welcomeMessage = "Welcome to the Go FTP Server"
)

type FTPConn struct {
	conn          *net.TCPConn
	controlReader *bufio.Reader
	controlWriter *bufio.Writer
	data          *net.TCPConn
	driver        FTPDriver
	namePrefix    string
	reqUser       string
	user          string
	renameFrom    string
}

// NewFTPConn constructs a new object that will handle the FTP protocol over
// an active net.TCPConn. The TCP connection should already be open before
// it is handed to this functions. driver is an instance of FTPDrive that
// will handle all auth and persistence details.
func NewFTPConn(tcpConn *net.TCPConn, driver FTPDriver) *FTPConn {
	c := new(FTPConn)
	c.namePrefix = "/"
	c.conn = tcpConn
	c.controlReader = bufio.NewReader(tcpConn)
	c.controlWriter = bufio.NewWriter(tcpConn)
	c.driver = driver
	return c
}

// Serve starts an endless loop that reads FTP commands from the client and
// responds appropriately. terminated is a channel that will receive a true
// message when the connection closes. This loop will be running inside a
// goroutine, so use this channel to be notified when the connection can be
// cleaned up.
func (ftpConn *FTPConn) Serve() {
	log.Print("Connection Established")
	// send welcome
	ftpConn.writeMessage(220, welcomeMessage)
	// read commands
	for {
		line, err := ftpConn.controlReader.ReadString('\n')
		if err != nil {
			break
		}
		ftpConn.receiveLine(line)
	}
	log.Print("Connection Terminated")
}

// Close will manually close this connection, even if the client isn't ready.
func (ftpConn *FTPConn) Close() {
	ftpConn.conn.Close()
	if ftpConn.data != nil {
		ftpConn.data.Close()
	}
}

// receiveLine accepts a single line FTP command and co-ordinates an
// appropriate response.
func (ftpConn *FTPConn) receiveLine(line string) {
	log.Print(line)
	command, param := ftpConn.parseLine(line)
	switch command {
	case "ALLO":
		ftpConn.cmdAllo()
	case "CDUP", "XCUP":
		ftpConn.cmdCdup()
	case "CWD", "XCWD":
		ftpConn.cmdCwd(param)
	case "DELE":
		ftpConn.cmdDele(param)
	case "MKD":
		ftpConn.cmdMkd(param)
	case "MODE":
		ftpConn.cmdMode(param)
	case "NOOP":
		ftpConn.cmdNoop()
	case "PASS":
		ftpConn.cmdPass(param)
	case "PWD", "XPWD":
		ftpConn.cmdPwd()
	case "QUIT":
		ftpConn.Close()
	case "RMD", "XRMD":
		ftpConn.cmdRmd(param)
	case "RNFR":
		ftpConn.cmdRnfr(param)
	case "RNTO":
		ftpConn.cmdRnto(param)
	case "SIZE":
		ftpConn.cmdSize(param)
	case "STRU":
		ftpConn.cmdStru(param)
	case "SYST":
		ftpConn.cmdSyst()
	case "TYPE":
		ftpConn.cmdType(param)
	case "USER":
		ftpConn.cmdUser(param)
	default:
		ftpConn.writeMessage(500, "Command not found")
	}
}

// cmdNoop responds to the ALLO FTP command.
//
// This is essentially a ping from the client so we just respond with an
// basic OK message.
func (ftpConn *FTPConn) cmdAllo() {
	ftpConn.writeMessage(202, "Obsolete")
}

// cmdCdup responds to the CDUP FTP command.
//
// Allows the client change their current directory to the parent.
func (ftpConn *FTPConn) cmdCdup() {
	ftpConn.cmdCwd("..")
}

// cmdCwd responds to the CWD FTP command. It allows the client to change the
// current working directory.
func (ftpConn *FTPConn) cmdCwd(param string) {
	path := ftpConn.buildPath(param)
	if ftpConn.driver.ChangeDir(path) {
		ftpConn.namePrefix = path
		ftpConn.writeMessage(250, "Directory changed to " + path)
	} else {
		ftpConn.writeMessage(550, "Action not taken")
	}
}

// cmdDele responds to the DELE FTP command. It allows the client to delete
// a file
func (ftpConn *FTPConn) cmdDele(param string) {
	path := ftpConn.buildPath(param)
	if ftpConn.driver.DeleteFile(path) {
		ftpConn.writeMessage(250, "File deleted")
	} else {
		ftpConn.writeMessage(550, "Action not taken")
	}
}

// cmdMkd responds to the MKD FTP command. It allows the client to create
// a new directory
func (ftpConn *FTPConn) cmdMkd(param string) {
	path := ftpConn.buildPath(param)
	if ftpConn.driver.MakeDir(path) {
		ftpConn.writeMessage(257, "Directory created")
	} else {
		ftpConn.writeMessage(550, "Action not taken")
	}
}

// cmdMode responds to the MODE FTP command.
//
// the original FTP spec had various options for hosts to negotiate how data
// would be sent over the data socket, In reality these days (S)tream mode
// is all that is used for the mode - data is just streamed down the data
// socket unchanged.
func (ftpConn *FTPConn) cmdMode(param string) {
	if strings.ToUpper(param) == "S" {
		ftpConn.writeMessage(200, "OK")
	} else {
		ftpConn.writeMessage(504, "MODE is an obsolete command")
	}
}

// cmdNoop responds to the NOOP FTP command.
//
// This is essentially a ping from the client so we just respond with an
// basic 200 message.
func (ftpConn *FTPConn) cmdNoop() {
	ftpConn.writeMessage(200, "OK")
}

// cmdPass respond to the PASS FTP command by asking the driver if the supplied
// username and password are valid
func (ftpConn *FTPConn) cmdPass(param string) {
	if ftpConn.driver.Authenticate(ftpConn.reqUser, param) {
		ftpConn.user = ftpConn.reqUser
		ftpConn.reqUser = ""
		ftpConn.writeMessage(230, "Password ok, continue")
	} else {
		ftpConn.writeMessage(530, "Incorrect password, not logged in")
	}
}

// cmdPwd responds to the PWD FTP command.
//
// Tells the client what the current working directory is.
func (ftpConn *FTPConn) cmdPwd() {
	ftpConn.writeMessage(257, "\"" + ftpConn.namePrefix + "\" is the current directory")
}

// cmdRmd responds to the RMD FTP command. It allows the client to delete a
// directory.
func (ftpConn *FTPConn) cmdRmd(param string) {
	path := ftpConn.buildPath(param)
	if ftpConn.driver.DeleteDir(path) {
		ftpConn.writeMessage(250, "Directory deleted")
	} else {
		ftpConn.writeMessage(550, "Action not taken")
	}
}

// cmdRnfr responds to the RNFR FTP command. It's the first of two commands
// required for a client to rename a file.
func (ftpConn *FTPConn) cmdRnfr(param string) {
	ftpConn.renameFrom = ftpConn.buildPath(param)
	ftpConn.writeMessage(350, "Requested file action pending further information.")
}

// cmdRnto responds to the RNTO FTP command. It's the second of two commands
// required for a client to rename a file.
func (ftpConn *FTPConn) cmdRnto(param string) {
	toPath := ftpConn.buildPath(param)
	if ftpConn.driver.Rename(ftpConn.renameFrom, toPath) {
		ftpConn.writeMessage(250, "File renamed")
	} else {
		ftpConn.writeMessage(550, "Action not taken")
	}
}

// cmdSize responds to the SIZE FTP command. It returns the size of the
// requested path in bytes.
func (ftpConn *FTPConn) cmdSize(param string) {
	path  := ftpConn.buildPath(param)
	bytes := ftpConn.driver.Bytes(path)
	if bytes >= 0 {
		ftpConn.writeMessage(213, strconv.Itoa(bytes))
	} else {
		ftpConn.writeMessage(450, "file not available")
	}
}

// cmdStru responds to the STRU FTP command.
//
// like the MODE and TYPE commands, stru[cture] dates back to a time when the
// FTP protocol was more aware of the content of the files it was transferring,
// and would sometimes be expected to translate things like EOL markers on the
// fly.
//
// These days files are sent unmodified, and F(ile) mode is the only one we
// really need to support.
func (ftpConn *FTPConn) cmdStru(param string) {
	if strings.ToUpper(param) == "F" {
		ftpConn.writeMessage(200, "OK")
	} else {
		ftpConn.writeMessage(504, "STRU is an obsolete command")
	}
}

// cmdSyst responds to the SYST FTP command by providing a canned response.
func (ftpConn *FTPConn) cmdSyst() {
	ftpConn.writeMessage(215, "UNIX Type: L8")
}

// cmdType responds to the TYPE FTP command.
//
//  like the MODE and STRU commands, TYPE dates back to a time when the FTP
//  protocol was more aware of the content of the files it was transferring, and
//  would sometimes be expected to translate things like EOL markers on the fly.
//
//  Valid options were A(SCII), I(mage), E(BCDIC) or LN (for local type). Since
//  we plan to just accept bytes from the client unchanged, I think Image mode is
//  adequate. The RFC requires we accept ASCII mode however, so accept it, but
//  ignore it.
func (ftpConn *FTPConn) cmdType(param string) {
	if strings.ToUpper(param) == "A" {
		ftpConn.writeMessage(200, "Type set to ASCII")
	} else if strings.ToUpper(param) == "I" {
		ftpConn.writeMessage(200, "Type set to binary")
	} else {
		ftpConn.writeMessage(500, "Invalid type")
	}
}

// cmdUser responds to the USER FTP command by asking for the password
func (ftpConn *FTPConn) cmdUser(param string) {
	ftpConn.reqUser = param
	ftpConn.writeMessage(331, "User name ok, password required")
}

func (ftpConn *FTPConn) parseLine(line string) (string, string) {
	params := strings.SplitN(strings.Trim(line, "\r\n"), " ", 2)
	if len(params) == 1 {
		return params[0], ""
	}
	return params[0], params[1]
}

// writeMessage will send a standard FTP response back to the client.
func (ftpConn *FTPConn) writeMessage(code int, message string) (wrote int, err error) {
	line := fmt.Sprintf("%d %s\r\n", code, message)
	log.Print(line)
	wrote, err = ftpConn.controlWriter.WriteString(line)
	ftpConn.controlWriter.Flush()
	return
}

// buildPath takes a client supplied path or filename and generates a safe
// absolute path withing their account sandbox.
//
//    buildpath("/")
//    => "/"
//    buildpath("one.txt")
//    => "/one.txt"
//    buildpath("/files/two.txt")
//    => "/files/two.txt"
//    buildpath("files/two.txt")
//    => "files/two.txt"
//    buildpath("/../../../../etc/passwd")
//    => "/etc/passwd"
//
// The driver implementation is responsible for deciding how to treat this path.
// Obviously they MUST NOT just read the path off disk. The probably want to
// prefix the path with something to scope the users access to a sandbox.
func (ftpConn *FTPConn) buildPath(filename string) (fullPath string){
	if filename[0:1] == "/" {
		fullPath = filepath.Clean(filename)
	} else if filename != "" && filename != "-a" {
		fullPath = filepath.Clean(ftpConn.namePrefix + "/" + filename)
	} else {
		fullPath = filepath.Clean(ftpConn.namePrefix)
	}
	fullPath = strings.Replace(fullPath, "//", "/", -1)
	return
}
<|endoftext|>


<commit_before>package image

import (
	"bytes"
	"encoding/base64"
	"errors"
	"fmt"
	simage "image"
	"image/gif"
	"image/jpeg"
	"image/png"
	"io"
	"io/ioutil"

	"github.com/nfnt/resize"
)

// Type for supported profile picture types
type Type string

// Size in pixels
type Size struct {
	Width  int
	Height int
}

// Image content and type
type Image struct {
	Content []byte
	Type    Type
}

const (
	// Invalid is an invalid image type
	Invalid Type = ""
	// GIF image/gif
	GIF Type = "image/gif"
	// JPEG image/jpeg
	JPEG Type = "image/jpeg"
	// PNG image/png
	PNG Type = "image/png"
)

var validImageTypes = [...]Type{
	GIF,
	JPEG,
	PNG,
}

// ErrInvalidData when the image data is invalid
var ErrInvalidData = errors.New("Invalid image data")

// ErrInvalidType when an image type is invalid
var ErrInvalidType = errors.New("Invalid image type")

// ProcessError image processing failed (resizing)
type ProcessError struct {
	err error
}

func (v ProcessError) Error() string {
	return fmt.Sprintf("Failed to process image: %+v", v.err)
}

func (v Type) String() string {
	return string(v)
}

// NewImage creates a new profile picture
func NewImage(r io.Reader, typ Type, maxSize Size) (pic Image, err error) {
	data, err := ioutil.ReadAll(r)
	if err != nil {
		return
	}

	img, err := createProfileImage(typ, data)
	if err != nil {
		return
	}
	// Resize the image
	img = resizeProfileImage(img, maxSize)

	bytes, err := func(img simage.Image, typ Type) (b []byte, err error) {
		var buf bytes.Buffer
		switch typ {
		case GIF:
			err = gif.Encode(&buf, img, nil)
			break
		case PNG:
			err = png.Encode(&buf, img)
			break
		case JPEG:
			err = jpeg.Encode(&buf, img, nil)
			break
		}
		b = buf.Bytes()
		return
	}(img, typ)

	if err != nil {
		err = ProcessError{err}
		return
	}

	pic = Image{bytes, typ}
	return
}

// Base64 returns base64 encoded data
func (v Image) Base64() string {
	return base64.StdEncoding.EncodeToString(v.Content)
}

// Image returns an image from the Content bytes
func (v Image) Image() (img simage.Image, err error) {
	img, err = createProfileImage(v.Type, v.Content)
	return
}

// IsValid checks if the profile pic is valid
func (v Image) IsValid() bool {
	if len(v.Content) < 10 {
		return false
	}

	if v.Type == Invalid {
		return false
	}
	return true
}

func createProfileImage(typ Type, data []byte) (img simage.Image, err error) {
	switch typ {
	case GIF:
		img, err = gif.Decode(bytes.NewReader(data))
		break
	case PNG:
		img, err = png.Decode(bytes.NewReader(data))
		break
	case JPEG:
		img, err = jpeg.Decode(bytes.NewReader(data))
		break
	default:
		err = ErrInvalidType
		break
	}
	if err != nil {
		err = ProcessError{err}
		return
	}
	return
}

func resizeProfileImage(img simage.Image, maxSize Size) simage.Image {
	s := img.Bounds().Size()
	if s.X > maxSize.Width || s.Y > maxSize.Height {
		return resize.Thumbnail(uint(maxSize.Width), uint(maxSize.Height), img, resize.Lanczos3)
	}
	return img
}
<commit_msg>image type IsValid<commit_after>package image

import (
	"bytes"
	"encoding/base64"
	"errors"
	"fmt"
	simage "image"
	"image/gif"
	"image/jpeg"
	"image/png"
	"io"
	"io/ioutil"

	"github.com/nfnt/resize"
)

// Type for supported profile picture types
type Type string

// Size in pixels
type Size struct {
	Width  int
	Height int
}

// Image content and type
type Image struct {
	Content []byte
	Type    Type
}

const (
	// Invalid is an invalid image type
	Invalid Type = ""
	// GIF image/gif
	GIF Type = "image/gif"
	// JPEG image/jpeg
	JPEG Type = "image/jpeg"
	// PNG image/png
	PNG Type = "image/png"
)

var validImageTypes = [...]Type{
	GIF,
	JPEG,
	PNG,
}

// ErrInvalidData when the image data is invalid
var ErrInvalidData = errors.New("Invalid image data")

// ErrInvalidType when an image type is invalid
var ErrInvalidType = errors.New("Invalid image type")

// ProcessError image processing failed (resizing)
type ProcessError struct {
	err error
}

func (v ProcessError) Error() string {
	return fmt.Sprintf("Failed to process image: %+v", v.err)
}

func (v Type) String() string {
	return string(v)
}

// IsValid returns true if the image type is either gif/png/jpeg
func (v Type) IsValid() bool {
	switch v {
	case GIF:
		return true
	case PNG:
		return true
	case JPEG:
		return true
	default:
		return false
	}
}

// NewImage creates a new profile picture
func NewImage(r io.Reader, typ Type, maxSize Size) (pic Image, err error) {
	data, err := ioutil.ReadAll(r)
	if err != nil {
		return
	}

	img, err := createProfileImage(typ, data)
	if err != nil {
		return
	}
	// Resize the image
	img = resizeProfileImage(img, maxSize)

	bytes, err := func(img simage.Image, typ Type) (b []byte, err error) {
		var buf bytes.Buffer
		switch typ {
		case GIF:
			err = gif.Encode(&buf, img, nil)
			break
		case PNG:
			err = png.Encode(&buf, img)
			break
		case JPEG:
			err = jpeg.Encode(&buf, img, nil)
			break
		}
		b = buf.Bytes()
		return
	}(img, typ)

	if err != nil {
		err = ProcessError{err}
		return
	}

	pic = Image{bytes, typ}
	return
}

// Base64 returns base64 encoded data
func (v Image) Base64() string {
	return base64.StdEncoding.EncodeToString(v.Content)
}

// Image returns an image from the Content bytes
func (v Image) Image() (img simage.Image, err error) {
	img, err = createProfileImage(v.Type, v.Content)
	return
}

// IsValid checks if the profile pic is valid
func (v Image) IsValid() bool {
	if len(v.Content) < 10 {
		return false
	}

	if v.Type == Invalid {
		return false
	}
	return true
}

func createProfileImage(typ Type, data []byte) (img simage.Image, err error) {
	switch typ {
	case GIF:
		img, err = gif.Decode(bytes.NewReader(data))
		break
	case PNG:
		img, err = png.Decode(bytes.NewReader(data))
		break
	case JPEG:
		img, err = jpeg.Decode(bytes.NewReader(data))
		break
	default:
		err = ErrInvalidType
		break
	}
	if err != nil {
		err = ProcessError{err}
		return
	}
	return
}

func resizeProfileImage(img simage.Image, maxSize Size) simage.Image {
	s := img.Bounds().Size()
	if s.X > maxSize.Width || s.Y > maxSize.Height {
		return resize.Thumbnail(uint(maxSize.Width), uint(maxSize.Height), img, resize.Lanczos3)
	}
	return img
}
<|endoftext|>


<commit_before>package storage

import (
	"io"
	"os"
	"path/filepath"

	"github.com/anacrolix/missinggo"

	"github.com/anacrolix/torrent/metainfo"
)

// File-based storage for torrents, that isn't yet bound to a particular
// torrent.
type fileClientImpl struct {
	baseDir   string
	pathMaker func(baseDir string, info *metainfo.Info, infoHash metainfo.Hash) string
	pc        PieceCompletion
}

// The Default path maker just returns the current path
func defaultPathMaker(baseDir string, info *metainfo.Info, infoHash metainfo.Hash) string {
	return baseDir
}

func infoHashPathMaker(baseDir string, info *metainfo.Info, infoHash metainfo.Hash) string {
	return filepath.Join(baseDir, infoHash.HexString())
}

// All Torrent data stored in this baseDir
func NewFile(baseDir string) ClientImpl {
	return NewFileWithCompletion(baseDir, pieceCompletionForDir(baseDir))
}

func NewFileWithCompletion(baseDir string, completion PieceCompletion) ClientImpl {
	return newFileWithCustomPathMakerAndCompletion(baseDir, nil, completion)
}

// All Torrent data stored in subdirectorys by infohash
func NewFileByInfoHash(baseDir string) ClientImpl {
	return NewFileWithCustomPathMaker(baseDir, infoHashPathMaker)
}

// Allows passing a function to determine the path for storing torrent data
func NewFileWithCustomPathMaker(baseDir string, pathMaker func(baseDir string, info *metainfo.Info, infoHash metainfo.Hash) string) ClientImpl {
	return newFileWithCustomPathMakerAndCompletion(baseDir, pathMaker, pieceCompletionForDir(baseDir))
}

func newFileWithCustomPathMakerAndCompletion(baseDir string, pathMaker func(baseDir string, info *metainfo.Info, infoHash metainfo.Hash) string, completion PieceCompletion) ClientImpl {
	if pathMaker == nil {
		pathMaker = defaultPathMaker
	}
	return &fileClientImpl{
		baseDir:   baseDir,
		pathMaker: pathMaker,
		pc:        completion,
	}
}

func (me *fileClientImpl) Close() error {
	return me.pc.Close()
}

func (fs *fileClientImpl) OpenTorrent(info *metainfo.Info, infoHash metainfo.Hash) (TorrentImpl, error) {
	dir := fs.pathMaker(fs.baseDir, info, infoHash)
	err := CreateNativeZeroLengthFiles(info, dir)
	if err != nil {
		return nil, err
	}
	return &fileTorrentImpl{
		dir,
		info,
		infoHash,
		fs.pc,
	}, nil
}

// File-based torrent storage, not yet bound to a Torrent.
type fileTorrentImpl struct {
	dir        string
	info       *metainfo.Info
	infoHash   metainfo.Hash
	completion PieceCompletion
}

func (fts *fileTorrentImpl) Piece(p metainfo.Piece) PieceImpl {
	// Create a view onto the file-based torrent storage.
	_io := fileTorrentImplIO{fts}
	// Return the appropriate segments of this.
	return &fileStoragePiece{
		fts,
		p,
		missinggo.NewSectionWriter(_io, p.Offset(), p.Length()),
		io.NewSectionReader(_io, p.Offset(), p.Length()),
	}
}

func (fs *fileTorrentImpl) Close() error {
	return nil
}

// Creates natives files for any zero-length file entries in the info. This is
// a helper for file-based storages, which don't address or write to zero-
// length files because they have no corresponding pieces.
func CreateNativeZeroLengthFiles(info *metainfo.Info, dir string) (err error) {
	for _, fi := range info.UpvertedFiles() {
		if fi.Length != 0 {
			continue
		}
		name := filepath.Join(append([]string{dir, info.Name}, fi.Path...)...)
		os.MkdirAll(filepath.Dir(name), 0750)
		var f io.Closer
		f, err = os.Create(name)
		if err != nil {
			break
		}
		f.Close()
	}
	return
}

// Exposes file-based storage of a torrent, as one big ReadWriterAt.
type fileTorrentImplIO struct {
	fts *fileTorrentImpl
}

// Returns EOF on short or missing file.
func (fst *fileTorrentImplIO) readFileAt(fi metainfo.FileInfo, b []byte, off int64) (n int, err error) {
	f, err := os.Open(fst.fts.fileInfoName(fi))
	if os.IsNotExist(err) {
		// File missing is treated the same as a short file.
		err = io.EOF
		return
	}
	if err != nil {
		return
	}
	defer f.Close()
	// Limit the read to within the expected bounds of this file.
	if int64(len(b)) > fi.Length-off {
		b = b[:fi.Length-off]
	}
	for off < fi.Length && len(b) != 0 {
		n1, err1 := f.ReadAt(b, off)
		b = b[n1:]
		n += n1
		off += int64(n1)
		if n1 == 0 {
			err = err1
			break
		}
	}
	return
}

// Only returns EOF at the end of the torrent. Premature EOF is ErrUnexpectedEOF.
func (fst fileTorrentImplIO) ReadAt(b []byte, off int64) (n int, err error) {
	for _, fi := range fst.fts.info.UpvertedFiles() {
		for off < fi.Length {
			n1, err1 := fst.readFileAt(fi, b, off)
			n += n1
			off += int64(n1)
			b = b[n1:]
			if len(b) == 0 {
				// Got what we need.
				return
			}
			if n1 != 0 {
				// Made progress.
				continue
			}
			err = err1
			if err == io.EOF {
				// Lies.
				err = io.ErrUnexpectedEOF
			}
			return
		}
		off -= fi.Length
	}
	err = io.EOF
	return
}

func (fst fileTorrentImplIO) WriteAt(p []byte, off int64) (n int, err error) {
	for _, fi := range fst.fts.info.UpvertedFiles() {
		if off >= fi.Length {
			off -= fi.Length
			continue
		}
		n1 := len(p)
		if int64(n1) > fi.Length-off {
			n1 = int(fi.Length - off)
		}
		name := fst.fts.fileInfoName(fi)
		os.MkdirAll(filepath.Dir(name), 0770)
		var f *os.File
		f, err = os.OpenFile(name, os.O_WRONLY|os.O_CREATE, 0660)
		if err != nil {
			return
		}
		n1, err = f.WriteAt(p[:n1], off)
		// TODO: On some systems, write errors can be delayed until the Close.
		f.Close()
		if err != nil {
			return
		}
		n += n1
		off = 0
		p = p[n1:]
		if len(p) == 0 {
			break
		}
	}
	return
}

func (fts *fileTorrentImpl) fileInfoName(fi metainfo.FileInfo) string {
	return filepath.Join(append([]string{fts.dir, fts.info.Name}, fi.Path...)...)
}
<commit_msg>storage: Remove incorrect comment<commit_after>package storage

import (
	"io"
	"os"
	"path/filepath"

	"github.com/anacrolix/missinggo"

	"github.com/anacrolix/torrent/metainfo"
)

// File-based storage for torrents, that isn't yet bound to a particular
// torrent.
type fileClientImpl struct {
	baseDir   string
	pathMaker func(baseDir string, info *metainfo.Info, infoHash metainfo.Hash) string
	pc        PieceCompletion
}

// The Default path maker just returns the current path
func defaultPathMaker(baseDir string, info *metainfo.Info, infoHash metainfo.Hash) string {
	return baseDir
}

func infoHashPathMaker(baseDir string, info *metainfo.Info, infoHash metainfo.Hash) string {
	return filepath.Join(baseDir, infoHash.HexString())
}

// All Torrent data stored in this baseDir
func NewFile(baseDir string) ClientImpl {
	return NewFileWithCompletion(baseDir, pieceCompletionForDir(baseDir))
}

func NewFileWithCompletion(baseDir string, completion PieceCompletion) ClientImpl {
	return newFileWithCustomPathMakerAndCompletion(baseDir, nil, completion)
}

// All Torrent data stored in subdirectorys by infohash
func NewFileByInfoHash(baseDir string) ClientImpl {
	return NewFileWithCustomPathMaker(baseDir, infoHashPathMaker)
}

// Allows passing a function to determine the path for storing torrent data
func NewFileWithCustomPathMaker(baseDir string, pathMaker func(baseDir string, info *metainfo.Info, infoHash metainfo.Hash) string) ClientImpl {
	return newFileWithCustomPathMakerAndCompletion(baseDir, pathMaker, pieceCompletionForDir(baseDir))
}

func newFileWithCustomPathMakerAndCompletion(baseDir string, pathMaker func(baseDir string, info *metainfo.Info, infoHash metainfo.Hash) string, completion PieceCompletion) ClientImpl {
	if pathMaker == nil {
		pathMaker = defaultPathMaker
	}
	return &fileClientImpl{
		baseDir:   baseDir,
		pathMaker: pathMaker,
		pc:        completion,
	}
}

func (me *fileClientImpl) Close() error {
	return me.pc.Close()
}

func (fs *fileClientImpl) OpenTorrent(info *metainfo.Info, infoHash metainfo.Hash) (TorrentImpl, error) {
	dir := fs.pathMaker(fs.baseDir, info, infoHash)
	err := CreateNativeZeroLengthFiles(info, dir)
	if err != nil {
		return nil, err
	}
	return &fileTorrentImpl{
		dir,
		info,
		infoHash,
		fs.pc,
	}, nil
}

type fileTorrentImpl struct {
	dir        string
	info       *metainfo.Info
	infoHash   metainfo.Hash
	completion PieceCompletion
}

func (fts *fileTorrentImpl) Piece(p metainfo.Piece) PieceImpl {
	// Create a view onto the file-based torrent storage.
	_io := fileTorrentImplIO{fts}
	// Return the appropriate segments of this.
	return &fileStoragePiece{
		fts,
		p,
		missinggo.NewSectionWriter(_io, p.Offset(), p.Length()),
		io.NewSectionReader(_io, p.Offset(), p.Length()),
	}
}

func (fs *fileTorrentImpl) Close() error {
	return nil
}

// Creates natives files for any zero-length file entries in the info. This is
// a helper for file-based storages, which don't address or write to zero-
// length files because they have no corresponding pieces.
func CreateNativeZeroLengthFiles(info *metainfo.Info, dir string) (err error) {
	for _, fi := range info.UpvertedFiles() {
		if fi.Length != 0 {
			continue
		}
		name := filepath.Join(append([]string{dir, info.Name}, fi.Path...)...)
		os.MkdirAll(filepath.Dir(name), 0750)
		var f io.Closer
		f, err = os.Create(name)
		if err != nil {
			break
		}
		f.Close()
	}
	return
}

// Exposes file-based storage of a torrent, as one big ReadWriterAt.
type fileTorrentImplIO struct {
	fts *fileTorrentImpl
}

// Returns EOF on short or missing file.
func (fst *fileTorrentImplIO) readFileAt(fi metainfo.FileInfo, b []byte, off int64) (n int, err error) {
	f, err := os.Open(fst.fts.fileInfoName(fi))
	if os.IsNotExist(err) {
		// File missing is treated the same as a short file.
		err = io.EOF
		return
	}
	if err != nil {
		return
	}
	defer f.Close()
	// Limit the read to within the expected bounds of this file.
	if int64(len(b)) > fi.Length-off {
		b = b[:fi.Length-off]
	}
	for off < fi.Length && len(b) != 0 {
		n1, err1 := f.ReadAt(b, off)
		b = b[n1:]
		n += n1
		off += int64(n1)
		if n1 == 0 {
			err = err1
			break
		}
	}
	return
}

// Only returns EOF at the end of the torrent. Premature EOF is ErrUnexpectedEOF.
func (fst fileTorrentImplIO) ReadAt(b []byte, off int64) (n int, err error) {
	for _, fi := range fst.fts.info.UpvertedFiles() {
		for off < fi.Length {
			n1, err1 := fst.readFileAt(fi, b, off)
			n += n1
			off += int64(n1)
			b = b[n1:]
			if len(b) == 0 {
				// Got what we need.
				return
			}
			if n1 != 0 {
				// Made progress.
				continue
			}
			err = err1
			if err == io.EOF {
				// Lies.
				err = io.ErrUnexpectedEOF
			}
			return
		}
		off -= fi.Length
	}
	err = io.EOF
	return
}

func (fst fileTorrentImplIO) WriteAt(p []byte, off int64) (n int, err error) {
	for _, fi := range fst.fts.info.UpvertedFiles() {
		if off >= fi.Length {
			off -= fi.Length
			continue
		}
		n1 := len(p)
		if int64(n1) > fi.Length-off {
			n1 = int(fi.Length - off)
		}
		name := fst.fts.fileInfoName(fi)
		os.MkdirAll(filepath.Dir(name), 0770)
		var f *os.File
		f, err = os.OpenFile(name, os.O_WRONLY|os.O_CREATE, 0660)
		if err != nil {
			return
		}
		n1, err = f.WriteAt(p[:n1], off)
		// TODO: On some systems, write errors can be delayed until the Close.
		f.Close()
		if err != nil {
			return
		}
		n += n1
		off = 0
		p = p[n1:]
		if len(p) == 0 {
			break
		}
	}
	return
}

func (fts *fileTorrentImpl) fileInfoName(fi metainfo.FileInfo) string {
	return filepath.Join(append([]string{fts.dir, fts.info.Name}, fi.Path...)...)
}
<|endoftext|>


<commit_before>package storage

import (
	"fmt"
	"io"
	"os"
	"path/filepath"

	"github.com/anacrolix/missinggo"
	"github.com/edsrzf/mmap-go"

	"github.com/anacrolix/torrent/metainfo"
	"github.com/anacrolix/torrent/mmap_span"
)

type mmapStorage struct {
	baseDir string
}

func NewMMap(baseDir string) Client {
	return &mmapStorage{
		baseDir: baseDir,
	}
}

func (s *mmapStorage) OpenTorrent(info *metainfo.InfoEx) (t Torrent, err error) {
	span, err := mMapTorrent(&info.Info, s.baseDir)
	t = &mmapTorrentStorage{
		span: span,
	}
	return
}

type mmapTorrentStorage struct {
	span      mmap_span.MMapSpan
	completed map[metainfo.Hash]bool
}

func (ts *mmapTorrentStorage) Piece(p metainfo.Piece) Piece {
	return mmapStoragePiece{
		storage:  ts,
		p:        p,
		ReaderAt: io.NewSectionReader(ts.span, p.Offset(), p.Length()),
		WriterAt: missinggo.NewSectionWriter(ts.span, p.Offset(), p.Length()),
	}
}

func (ts *mmapTorrentStorage) Close() error {
	ts.span.Close()
	return nil
}

type mmapStoragePiece struct {
	storage *mmapTorrentStorage
	p       metainfo.Piece
	io.ReaderAt
	io.WriterAt
}

func (sp mmapStoragePiece) GetIsComplete() bool {
	return sp.storage.completed[sp.p.Hash()]
}

func (sp mmapStoragePiece) MarkComplete() error {
	if sp.storage.completed == nil {
		sp.storage.completed = make(map[metainfo.Hash]bool)
	}
	sp.storage.completed[sp.p.Hash()] = true
	return nil
}

func mMapTorrent(md *metainfo.Info, location string) (mms mmap_span.MMapSpan, err error) {
	defer func() {
		if err != nil {
			mms.Close()
		}
	}()
	for _, miFile := range md.UpvertedFiles() {
		fileName := filepath.Join(append([]string{location, md.Name}, miFile.Path...)...)
		err = os.MkdirAll(filepath.Dir(fileName), 0777)
		if err != nil {
			err = fmt.Errorf("error creating data directory %q: %s", filepath.Dir(fileName), err)
			return
		}
		var file *os.File
		file, err = os.OpenFile(fileName, os.O_CREATE|os.O_RDWR, 0666)
		if err != nil {
			return
		}
		func() {
			defer file.Close()
			var fi os.FileInfo
			fi, err = file.Stat()
			if err != nil {
				return
			}
			if fi.Size() < miFile.Length {
				err = file.Truncate(miFile.Length)
				if err != nil {
					return
				}
			}
			if miFile.Length == 0 {
				// Can't mmap() regions with length 0.
				return
			}
			var mMap mmap.MMap
			mMap, err = mmap.MapRegion(file,
				int(miFile.Length), // Probably not great on <64 bit systems.
				mmap.RDWR, 0, 0)
			if err != nil {
				err = fmt.Errorf("error mapping file %q, length %d: %s", file.Name(), miFile.Length, err)
				return
			}
			if int64(len(mMap)) != miFile.Length {
				panic("mmap has wrong length")
			}
			mms.Append(mMap)
		}()
		if err != nil {
			return
		}
	}
	return
}
<commit_msg>storage: Also use completion DB in mmap implementation<commit_after>package storage

import (
	"fmt"
	"io"
	"os"
	"path/filepath"

	"github.com/anacrolix/missinggo"
	"github.com/edsrzf/mmap-go"

	"github.com/anacrolix/torrent/metainfo"
	"github.com/anacrolix/torrent/mmap_span"
)

type mmapStorage struct {
	baseDir    string
	completion pieceCompletion
}

func NewMMap(baseDir string) Client {
	return &mmapStorage{
		baseDir:    baseDir,
		completion: pieceCompletionForDir(baseDir),
	}
}

func (s *mmapStorage) OpenTorrent(info *metainfo.InfoEx) (t Torrent, err error) {
	span, err := mMapTorrent(&info.Info, s.baseDir)
	t = &mmapTorrentStorage{
		span: span,
		pc:   s.completion,
	}
	return
}

type mmapTorrentStorage struct {
	span mmap_span.MMapSpan
	pc   pieceCompletion
}

func (ts *mmapTorrentStorage) Piece(p metainfo.Piece) Piece {
	return mmapStoragePiece{
		pc:       ts.pc,
		p:        p,
		ReaderAt: io.NewSectionReader(ts.span, p.Offset(), p.Length()),
		WriterAt: missinggo.NewSectionWriter(ts.span, p.Offset(), p.Length()),
	}
}

func (ts *mmapTorrentStorage) Close() error {
	ts.span.Close()
	return nil
}

type mmapStoragePiece struct {
	pc pieceCompletion
	p  metainfo.Piece
	io.ReaderAt
	io.WriterAt
}

func (sp mmapStoragePiece) GetIsComplete() bool {
	return sp.pc.Get(sp.p)
}

func (sp mmapStoragePiece) MarkComplete() error {
	sp.pc.Set(sp.p, true)
	return nil
}

func mMapTorrent(md *metainfo.Info, location string) (mms mmap_span.MMapSpan, err error) {
	defer func() {
		if err != nil {
			mms.Close()
		}
	}()
	for _, miFile := range md.UpvertedFiles() {
		fileName := filepath.Join(append([]string{location, md.Name}, miFile.Path...)...)
		err = os.MkdirAll(filepath.Dir(fileName), 0777)
		if err != nil {
			err = fmt.Errorf("error creating data directory %q: %s", filepath.Dir(fileName), err)
			return
		}
		var file *os.File
		file, err = os.OpenFile(fileName, os.O_CREATE|os.O_RDWR, 0666)
		if err != nil {
			return
		}
		func() {
			defer file.Close()
			var fi os.FileInfo
			fi, err = file.Stat()
			if err != nil {
				return
			}
			if fi.Size() < miFile.Length {
				err = file.Truncate(miFile.Length)
				if err != nil {
					return
				}
			}
			if miFile.Length == 0 {
				// Can't mmap() regions with length 0.
				return
			}
			var mMap mmap.MMap
			mMap, err = mmap.MapRegion(file,
				int(miFile.Length), // Probably not great on <64 bit systems.
				mmap.RDWR, 0, 0)
			if err != nil {
				err = fmt.Errorf("error mapping file %q, length %d: %s", file.Name(), miFile.Length, err)
				return
			}
			if int64(len(mMap)) != miFile.Length {
				panic("mmap has wrong length")
			}
			mms.Append(mMap)
		}()
		if err != nil {
			return
		}
	}
	return
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"time"

	"encoding/json"

	"github.com/catatsuy/isucon6-final/bench/scenario"
	"github.com/catatsuy/isucon6-final/bench/session"
	"github.com/catatsuy/isucon6-final/bench/sse"
)

type Log struct {
	Time       time.Time `json:"time"`
	RoomID     int       `json:"room_id"`
	StrokeID   int64     `json:"stroke_id"`
	StrokeTime time.Time `json:"stroke_time"`
}

type RoomWatcher struct {
	EndCh  chan struct{}
	Logs   []Log
	Errors []string

	es     *sse.EventSource
	isLeft bool
}

func NewRoomWatcher(target string, roomID int) *RoomWatcher {
	w := &RoomWatcher{
		EndCh:  make(chan struct{}, 1),
		Logs:   make([]Log, 0),
		Errors: make([]string, 0),
		isLeft: false,
	}

	go w.watch(target, roomID)

	return w
}

// 描いたstrokeがこの時間以上経ってから届いたら、ユーザーがストレスに感じてタブを閉じる、という設定にした。
const thresholdResponseTime = 5 * time.Second

func (w *RoomWatcher) watch(target string, roomID int) {

	s := session.New(target)
	s.Client.Timeout = thresholdResponseTime

	path := fmt.Sprintf("/rooms/%d", roomID)
	token, err := scenario.GetCSRFToken(s, target+path)
	if err != nil {
		w.addError(fmt.Sprintf("GET %s リクエストに失敗しました", path))
		fmt.Println(err)
		w.EndCh <- struct{}{}
		return
	}

	startTime := time.Now()
	path = "/api/strokes" + path

	if w.isLeft {
		w.EndCh <- struct{}{}
		return
	}
	w.es = sse.NewEventSource(s.Client, target+path+"?csrf_token="+token)

	w.es.On("stroke", func(data string) {
		var stroke scenario.Stroke
		err := json.Unmarshal([]byte(data), &stroke)
		if err != nil {
			w.Errors = append(w.Errors, err.Error())
			fmt.Println(err)
			w.es.Close()
		}
		now := time.Now()
		// strokes APIには最初はLast-Event-IDをつけずに送るので、これまでに描かれたstrokeが全部降ってくるが、それは無視する。
		if stroke.CreatedAt.After(startTime) && now.Sub(stroke.CreatedAt) > thresholdResponseTime {
			fmt.Println("response too late")
			w.es.Close()
		}
		w.Logs = append(w.Logs, Log{
			Time:       now,
			RoomID:     roomID,
			StrokeID:   stroke.ID,
			StrokeTime: stroke.CreatedAt,
		})
	})
	w.es.On("bad_request", func(data string) {
		w.addError(path + " bad_request: " + data)
		w.es.Close()
	})
	//w.es.On("watcher_count", func(data string) {
	//	fmt.Println("watcher_count")
	//	fmt.Println(data)
	//})
	w.es.OnError(func(err error) {
		if e, ok := err.(*sse.BadContentType); ok {
			w.addError(path + " Content-Typeが正しくありません: " + e.ContentType)
			return
		}
		if e, ok := err.(*sse.BadStatusCode); ok {
			w.addError(fmt.Sprintf("%s ステータスコードが正しくありません: %d\n", path, e.StatusCode))
			w.es.Close()
			return
		}
		fmt.Println(err)
		w.addError(path + " 予期せぬエラー")
	})
	w.es.OnEnd(func() {
		w.EndCh <- struct{}{}
	})

	w.es.Start()
}

func (w *RoomWatcher) addError(msg string) {
	w.Errors = append(w.Errors, fmt.Sprintf("%s", msg))
}

func (w *RoomWatcher) Leave() {
	w.isLeft = true
	if w.es != nil {
		w.es.Close()
	}
}
<commit_msg>Add comment<commit_after>package main

import (
	"fmt"
	"time"

	"encoding/json"

	"github.com/catatsuy/isucon6-final/bench/scenario"
	"github.com/catatsuy/isucon6-final/bench/session"
	"github.com/catatsuy/isucon6-final/bench/sse"
)

type Log struct {
	Time       time.Time `json:"time"`
	RoomID     int       `json:"room_id"`
	StrokeID   int64     `json:"stroke_id"`
	StrokeTime time.Time `json:"stroke_time"`
}

type RoomWatcher struct {
	EndCh  chan struct{}
	Logs   []Log
	Errors []string

	es     *sse.EventSource
	isLeft bool
}

func NewRoomWatcher(target string, roomID int) *RoomWatcher {
	w := &RoomWatcher{
		EndCh:  make(chan struct{}, 1),
		Logs:   make([]Log, 0),
		Errors: make([]string, 0),
		isLeft: false,
	}

	go w.watch(target, roomID)

	return w
}

// 描いたstrokeがこの時間以上経ってから届いたら、ユーザーがストレスに感じてタブを閉じる、という設定にした。
const thresholdResponseTime = 5 * time.Second

func (w *RoomWatcher) watch(target string, roomID int) {

	// TODO:用途がだいぶ特殊なので普通のベンチマークと同じsessionを使うべきか悩ましい
	s := session.New(target)
	s.Client.Timeout = thresholdResponseTime

	path := fmt.Sprintf("/rooms/%d", roomID)
	token, err := scenario.GetCSRFToken(s, target+path)
	if err != nil {
		w.addError(fmt.Sprintf("GET %s リクエストに失敗しました", path))
		fmt.Println(err)
		w.EndCh <- struct{}{}
		return
	}

	startTime := time.Now()
	path = "/api/strokes" + path

	if w.isLeft {
		w.EndCh <- struct{}{}
		return
	}
	w.es = sse.NewEventSource(s.Client, target+path+"?csrf_token="+token)

	w.es.On("stroke", func(data string) {
		var stroke scenario.Stroke
		err := json.Unmarshal([]byte(data), &stroke)
		if err != nil {
			w.Errors = append(w.Errors, err.Error())
			fmt.Println(err)
			w.es.Close()
		}
		now := time.Now()
		// strokes APIには最初はLast-Event-IDをつけずに送るので、これまでに描かれたstrokeが全部降ってくるが、それは無視する。
		if stroke.CreatedAt.After(startTime) && now.Sub(stroke.CreatedAt) > thresholdResponseTime {
			fmt.Println("response too late")
			w.es.Close()
		}
		w.Logs = append(w.Logs, Log{
			Time:       now,
			RoomID:     roomID,
			StrokeID:   stroke.ID,
			StrokeTime: stroke.CreatedAt,
		})
	})
	w.es.On("bad_request", func(data string) {
		w.addError(path + " bad_request: " + data)
		w.es.Close()
	})
	//w.es.On("watcher_count", func(data string) {
	//	fmt.Println("watcher_count")
	//	fmt.Println(data)
	//})
	w.es.OnError(func(err error) {
		if e, ok := err.(*sse.BadContentType); ok {
			w.addError(path + " Content-Typeが正しくありません: " + e.ContentType)
			return
		}
		if e, ok := err.(*sse.BadStatusCode); ok {
			w.addError(fmt.Sprintf("%s ステータスコードが正しくありません: %d\n", path, e.StatusCode))
			w.es.Close()
			return
		}
		fmt.Println(err)
		w.addError(path + " 予期せぬエラー")
	})
	w.es.OnEnd(func() {
		w.EndCh <- struct{}{}
	})

	w.es.Start()
}

func (w *RoomWatcher) addError(msg string) {
	w.Errors = append(w.Errors, fmt.Sprintf("%s", msg))
}

func (w *RoomWatcher) Leave() {
	w.isLeft = true
	if w.es != nil {
		w.es.Close()
	}
}
<|endoftext|>