<commit_before>package sanitize_test

import (
	"testing"

	"github.com/jackc/pgx/internal/sanitize"
)

func TestNewQuery(t *testing.T) {
	successTests := []struct {
		sql      string
		expected sanitize.Query
	}{
		{
			sql:      "select 42",
			expected: sanitize.Query{Parts: []sanitize.Part{"select 42"}},
		},
		{
			sql:      "select $1",
			expected: sanitize.Query{Parts: []sanitize.Part{"select ", 1}},
		},
		{
			sql:      "select 'quoted $42', $1",
			expected: sanitize.Query{Parts: []sanitize.Part{"select 'quoted $42', ", 1}},
		},
		{
			sql:      `select "doubled quoted $42", $1`,
			expected: sanitize.Query{Parts: []sanitize.Part{`select "doubled quoted $42", `, 1}},
		},
		{
			sql:      "select 'foo''bar', $1",
			expected: sanitize.Query{Parts: []sanitize.Part{"select 'foo''bar', ", 1}},
		},
		{
			sql:      `select "foo""bar", $1`,
			expected: sanitize.Query{Parts: []sanitize.Part{`select "foo""bar", `, 1}},
		},
		{
			sql:      "select '''', $1",
			expected: sanitize.Query{Parts: []sanitize.Part{"select '''', ", 1}},
		},
		{
			sql:      `select """", $1`,
			expected: sanitize.Query{Parts: []sanitize.Part{`select """", `, 1}},
		},
		{
			sql:      "select $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11",
			expected: sanitize.Query{Parts: []sanitize.Part{"select ", 1, ", ", 2, ", ", 3, ", ", 4, ", ", 5, ", ", 6, ", ", 7, ", ", 8, ", ", 9, ", ", 10, ", ", 11}},
		},
		{
			sql:      `select "adsf""$1""adsf", $1, 'foo''$$12bar', $2, '$3'`,
			expected: sanitize.Query{Parts: []sanitize.Part{`select "adsf""$1""adsf", `, 1, `, 'foo''$$12bar', `, 2, `, '$3'`}},
		},
		{
			sql:      `select E'escape string\' $42', $1`,
			expected: sanitize.Query{Parts: []sanitize.Part{`select E'escape string\' $42', `, 1}},
		},
		{
			sql:      `select e'escape string\' $42', $1`,
			expected: sanitize.Query{Parts: []sanitize.Part{`select e'escape string\' $42', `, 1}},
		},
	}

	for i, tt := range successTests {
		query, err := sanitize.NewQuery(tt.sql)
		if err != nil {
			t.Errorf("%d. %v", i, err)
		}

		if len(query.Parts) == len(tt.expected.Parts) {
			for j := range query.Parts {
				if query.Parts[j] != tt.expected.Parts[j] {
					t.Errorf("%d. expected part %d to be %v but it was %v", i, j, tt.expected.Parts[j], query.Parts[j])
				}
			}
		} else {
			t.Errorf("%d. expected query parts to be %v but it was %v", i, tt.expected.Parts, query.Parts)
		}
	}
}

func TestQuerySanitize(t *testing.T) {
	successfulTests := []struct {
		query    sanitize.Query
		args     []interface{}
		expected string
	}{
		{
			query:    sanitize.Query{Parts: []sanitize.Part{"select 42"}},
			args:     []interface{}{},
			expected: `select 42`,
		},
		{
			query:    sanitize.Query{Parts: []sanitize.Part{"select ", 1}},
			args:     []interface{}{int64(42)},
			expected: `select 42`,
		},
		{
			query:    sanitize.Query{Parts: []sanitize.Part{"select ", 1}},
			args:     []interface{}{float64(1.23)},
			expected: `select 1.23`,
		},
		{
			query:    sanitize.Query{Parts: []sanitize.Part{"select ", 1}},
			args:     []interface{}{true},
			expected: `select true`,
		},
		{
			query:    sanitize.Query{Parts: []sanitize.Part{"select ", 1}},
			args:     []interface{}{[]byte{0, 1, 2, 3, 255}},
			expected: `select '\x00010203ff'`,
		},
		{
			query:    sanitize.Query{Parts: []sanitize.Part{"select ", 1}},
			args:     []interface{}{nil},
			expected: `select null`,
		},
		{
			query:    sanitize.Query{Parts: []sanitize.Part{"select ", 1}},
			args:     []interface{}{"foobar"},
			expected: `select 'foobar'`,
		},
		{
			query:    sanitize.Query{Parts: []sanitize.Part{"select ", 1}},
			args:     []interface{}{"foo'bar"},
			expected: `select 'foo''bar'`,
		},
		{
			query:    sanitize.Query{Parts: []sanitize.Part{"select ", 1}},
			args:     []interface{}{`foo\'bar`},
			expected: `select 'foo\''bar'`,
		},
	}

	for i, tt := range successfulTests {
		actual, err := tt.query.Sanitize(tt.args...)
		if err != nil {
			t.Errorf("%d. %v", i, err)
			continue
		}

		if tt.expected != actual {
			t.Errorf("%d. expected %s, but got %s", i, tt.expected, actual)
		}
	}

	errorTests := []struct {
		query    sanitize.Query
		args     []interface{}
		expected string
	}{
		{
			query:    sanitize.Query{Parts: []sanitize.Part{"select ", 1, ", ", 2}},
			args:     []interface{}{int64(42)},
			expected: `insufficient arguments`,
		},
		{
			query:    sanitize.Query{Parts: []sanitize.Part{"select 'foo'"}},
			args:     []interface{}{int64(42)},
			expected: `unused argument: 0`,
		},
		{
			query:    sanitize.Query{Parts: []sanitize.Part{"select ", 1}},
			args:     []interface{}{42},
			expected: `invalid arg type: int`,
		},
	}

	for i, tt := range errorTests {
		_, err := tt.query.Sanitize(tt.args...)
		if err == nil || err.Error() != tt.expected {
			t.Errorf("%d. expected error %v, got %v", i, tt.expected, err)
		}
	}
}
<commit_msg>adjust the test for the patch<commit_after>package sanitize_test

import (
	"testing"

	"github.com/jackc/pgx/internal/sanitize"
)

func TestNewQuery(t *testing.T) {
	successTests := []struct {
		sql      string
		expected sanitize.Query
	}{
		{
			sql:      "select 42",
			expected: sanitize.Query{Parts: []sanitize.Part{"select 42"}},
		},
		{
			sql:      "select $1",
			expected: sanitize.Query{Parts: []sanitize.Part{"select ", 1}},
		},
		{
			sql:      "select 'quoted $42', $1",
			expected: sanitize.Query{Parts: []sanitize.Part{"select 'quoted $42', ", 1}},
		},
		{
			sql:      `select "doubled quoted $42", $1`,
			expected: sanitize.Query{Parts: []sanitize.Part{`select "doubled quoted $42", `, 1}},
		},
		{
			sql:      "select 'foo''bar', $1",
			expected: sanitize.Query{Parts: []sanitize.Part{"select 'foo''bar', ", 1}},
		},
		{
			sql:      `select "foo""bar", $1`,
			expected: sanitize.Query{Parts: []sanitize.Part{`select "foo""bar", `, 1}},
		},
		{
			sql:      "select '''', $1",
			expected: sanitize.Query{Parts: []sanitize.Part{"select '''', ", 1}},
		},
		{
			sql:      `select """", $1`,
			expected: sanitize.Query{Parts: []sanitize.Part{`select """", `, 1}},
		},
		{
			sql:      "select $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11",
			expected: sanitize.Query{Parts: []sanitize.Part{"select ", 1, ", ", 2, ", ", 3, ", ", 4, ", ", 5, ", ", 6, ", ", 7, ", ", 8, ", ", 9, ", ", 10, ", ", 11}},
		},
		{
			sql:      `select "adsf""$1""adsf", $1, 'foo''$$12bar', $2, '$3'`,
			expected: sanitize.Query{Parts: []sanitize.Part{`select "adsf""$1""adsf", `, 1, `, 'foo''$$12bar', `, 2, `, '$3'`}},
		},
		{
			sql:      `select E'escape string\' $42', $1`,
			expected: sanitize.Query{Parts: []sanitize.Part{`select E'escape string\' $42', `, 1}},
		},
		{
			sql:      `select e'escape string\' $42', $1`,
			expected: sanitize.Query{Parts: []sanitize.Part{`select e'escape string\' $42', `, 1}},
		},
	}

	for i, tt := range successTests {
		query, err := sanitize.NewQuery(tt.sql)
		if err != nil {
			t.Errorf("%d. %v", i, err)
		}

		if len(query.Parts) == len(tt.expected.Parts) {
			for j := range query.Parts {
				if query.Parts[j] != tt.expected.Parts[j] {
					t.Errorf("%d. expected part %d to be %v but it was %v", i, j, tt.expected.Parts[j], query.Parts[j])
				}
			}
		} else {
			t.Errorf("%d. expected query parts to be %v but it was %v", i, tt.expected.Parts, query.Parts)
		}
	}
}

func TestQuerySanitize(t *testing.T) {
	successfulTests := []struct {
		query    sanitize.Query
		args     []interface{}
		expected string
	}{
		{
			query:    sanitize.Query{Parts: []sanitize.Part{"select 42"}},
			args:     []interface{}{},
			expected: `select 42`,
		},
		{
			query:    sanitize.Query{Parts: []sanitize.Part{"select ", 1}},
			args:     []interface{}{int64(42)},
			expected: `select 42`,
		},
		{
			query:    sanitize.Query{Parts: []sanitize.Part{"select ", 1}},
			args:     []interface{}{float64(1.23)},
			expected: `select 1.23`,
		},
		{
			query:    sanitize.Query{Parts: []sanitize.Part{"select ", 1}},
			args:     []interface{}{true},
			expected: `select true`,
		},
		{
			query:    sanitize.Query{Parts: []sanitize.Part{"select ", 1}},
			args:     []interface{}{[]byte{0, 1, 2, 3, 255}},
			expected: `select '\x00010203ff'::bytea`,
		},
		{
			query:    sanitize.Query{Parts: []sanitize.Part{"select ", 1}},
			args:     []interface{}{nil},
			expected: `select null`,
		},
		{
			query:    sanitize.Query{Parts: []sanitize.Part{"select ", 1}},
			args:     []interface{}{"foobar"},
			expected: `select 'foobar'`,
		},
		{
			query:    sanitize.Query{Parts: []sanitize.Part{"select ", 1}},
			args:     []interface{}{"foo'bar"},
			expected: `select 'foo''bar'`,
		},
		{
			query:    sanitize.Query{Parts: []sanitize.Part{"select ", 1}},
			args:     []interface{}{`foo\'bar`},
			expected: `select 'foo\''bar'`,
		},
	}

	for i, tt := range successfulTests {
		actual, err := tt.query.Sanitize(tt.args...)
		if err != nil {
			t.Errorf("%d. %v", i, err)
			continue
		}

		if tt.expected != actual {
			t.Errorf("%d. expected %s, but got %s", i, tt.expected, actual)
		}
	}

	errorTests := []struct {
		query    sanitize.Query
		args     []interface{}
		expected string
	}{
		{
			query:    sanitize.Query{Parts: []sanitize.Part{"select ", 1, ", ", 2}},
			args:     []interface{}{int64(42)},
			expected: `insufficient arguments`,
		},
		{
			query:    sanitize.Query{Parts: []sanitize.Part{"select 'foo'"}},
			args:     []interface{}{int64(42)},
			expected: `unused argument: 0`,
		},
		{
			query:    sanitize.Query{Parts: []sanitize.Part{"select ", 1}},
			args:     []interface{}{42},
			expected: `invalid arg type: int`,
		},
	}

	for i, tt := range errorTests {
		_, err := tt.query.Sanitize(tt.args...)
		if err == nil || err.Error() != tt.expected {
			t.Errorf("%d. expected error %v, got %v", i, tt.expected, err)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)

package memfs_test

import (
	"io/ioutil"
	"log"
	"os"
	"os/user"
	"path"
	"strconv"
	"strings"
	"syscall"
	"testing"
	"time"

	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/samples/memfs"
	"github.com/jacobsa/gcsfuse/timeutil"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
	"golang.org/x/net/context"
)

func TestMemFS(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func currentUid() uint32 {
	user, err := user.Current()
	if err != nil {
		panic(err)
	}

	uid, err := strconv.ParseUint(user.Uid, 10, 32)
	if err != nil {
		panic(err)
	}

	return uint32(uid)
}

func currentGid() uint32 {
	user, err := user.Current()
	if err != nil {
		panic(err)
	}

	gid, err := strconv.ParseUint(user.Gid, 10, 32)
	if err != nil {
		panic(err)
	}

	return uint32(gid)
}

func timespecToTime(ts syscall.Timespec) time.Time {
	return time.Unix(ts.Sec, ts.Nsec)
}

////////////////////////////////////////////////////////////////////////
// Boilerplate
////////////////////////////////////////////////////////////////////////

type MemFSTest struct {
	clock timeutil.SimulatedClock
	mfs   *fuse.MountedFileSystem
}

var _ SetUpInterface = &MemFSTest{}
var _ TearDownInterface = &MemFSTest{}

func init() { RegisterTestSuite(&MemFSTest{}) }

func (t *MemFSTest) SetUp(ti *TestInfo) {
	var err error

	// Set up a fixed, non-zero time.
	t.clock.SetTime(time.Now())

	// Set up a temporary directory for mounting.
	mountPoint, err := ioutil.TempDir("", "memfs_test")
	if err != nil {
		panic("ioutil.TempDir: " + err.Error())
	}

	// Mount a file system.
	fs := memfs.NewMemFS(&t.clock)
	if t.mfs, err = fuse.Mount(mountPoint, fs); err != nil {
		panic("Mount: " + err.Error())
	}

	if err = t.mfs.WaitForReady(context.Background()); err != nil {
		panic("MountedFileSystem.WaitForReady: " + err.Error())
	}
}

func (t *MemFSTest) TearDown() {
	// Unmount the file system. Try again on "resource busy" errors.
	delay := 10 * time.Millisecond
	for {
		err := t.mfs.Unmount()
		if err == nil {
			break
		}

		if strings.Contains(err.Error(), "resource busy") {
			log.Println("Resource busy error while unmounting; trying again")
			time.Sleep(delay)
			delay = time.Duration(1.3 * float64(delay))
			continue
		}

		panic("MountedFileSystem.Unmount: " + err.Error())
	}

	if err := t.mfs.Join(context.Background()); err != nil {
		panic("MountedFileSystem.Join: " + err.Error())
	}
}

////////////////////////////////////////////////////////////////////////
// Test functions
////////////////////////////////////////////////////////////////////////

func (t *MemFSTest) ContentsOfEmptyFileSystem() {
	entries, err := ioutil.ReadDir(t.mfs.Dir())

	AssertEq(nil, err)
	ExpectThat(entries, ElementsAre())
}

func (t *MemFSTest) Mkdir_OneLevel() {
	var err error
	var fi os.FileInfo
	var stat *syscall.Stat_t
	var entries []os.FileInfo

	dirName := path.Join(t.mfs.Dir(), "dir")

	// Create a directory within the root.
	createTime := t.clock.Now()
	err = os.Mkdir(dirName, 0754)
	AssertEq(nil, err)

	// Simulate time advancing.
	t.clock.AdvanceTime(time.Second)

	// Stat the directory.
	fi, err = os.Stat(dirName)
	stat = fi.Sys().(*syscall.Stat_t)

	AssertEq(nil, err)
	ExpectEq("dir", fi.Name())
	ExpectEq(0, fi.Size())
	ExpectEq(os.ModeDir|0754, fi.Mode())
	ExpectEq(0, fi.ModTime().Sub(createTime))
	ExpectTrue(fi.IsDir())

	ExpectNe(0, stat.Ino)
	ExpectEq(1, stat.Nlink)
	ExpectEq(currentUid(), stat.Uid)
	ExpectEq(currentGid(), stat.Gid)
	ExpectEq(0, stat.Size)
	ExpectEq(0, timespecToTime(stat.Atimespec).Sub(createTime))
	ExpectEq(0, timespecToTime(stat.Mtimespec).Sub(createTime))
	ExpectEq(0, timespecToTime(stat.Ctimespec).Sub(createTime))

	// Read the directory.
	entries, err = ioutil.ReadDir(dirName)

	AssertEq(nil, err)
	ExpectThat(entries, ElementsAre())

	// Read the root.
	entries, err = ioutil.ReadDir(t.mfs.Dir())

	AssertEq(nil, err)
	AssertEq(1, len(entries))

	fi = entries[0]
	ExpectEq("dir", fi.Name())
	ExpectEq(os.ModeDir|0754, fi.Mode())
}

func (t *MemFSTest) Mkdir_TwoLevels() {
	var err error
	var fi os.FileInfo
	var stat *syscall.Stat_t
	var entries []os.FileInfo

	// Create a directory within the root.
	err = os.Mkdir(path.Join(t.mfs.Dir(), "parent"), 0700)
	AssertEq(nil, err)

	// Create a child of that directory.
	createTime := t.clock.Now()
	err = os.Mkdir(path.Join(t.mfs.Dir(), "parent/dir"), 0754)
	AssertEq(nil, err)

	// Simulate time advancing.
	t.clock.AdvanceTime(time.Second)

	// Stat the directory.
	fi, err = os.Stat(path.Join(t.mfs.Dir(), "parent/dir"))
	stat = fi.Sys().(*syscall.Stat_t)

	AssertEq(nil, err)
	ExpectEq("dir", fi.Name())
	ExpectEq(0, fi.Size())
	ExpectEq(os.ModeDir|0754, fi.Mode())
	ExpectEq(0, fi.ModTime().Sub(createTime))
	ExpectTrue(fi.IsDir())

	ExpectNe(0, stat.Ino)
	ExpectEq(1, stat.Nlink)
	ExpectEq(currentUid(), stat.Uid)
	ExpectEq(currentGid(), stat.Gid)
	ExpectEq(0, stat.Size)
	ExpectEq(0, timespecToTime(stat.Atimespec).Sub(createTime))
	ExpectEq(0, timespecToTime(stat.Mtimespec).Sub(createTime))
	ExpectEq(0, timespecToTime(stat.Ctimespec).Sub(createTime))

	// Read the directory.
	entries, err = ioutil.ReadDir(path.Join(t.mfs.Dir(), "parent/dir"))

	AssertEq(nil, err)
	ExpectThat(entries, ElementsAre())

	// Read the parent.
	entries, err = ioutil.ReadDir(path.Join(t.mfs.Dir(), "parent"))

	AssertEq(nil, err)
	AssertEq(1, len(entries))

	fi = entries[0]
	ExpectEq("dir", fi.Name())
	ExpectEq(os.ModeDir|0754, fi.Mode())
}

func (t *MemFSTest) Mkdir_AlreadyExists() {
	var err error
	dirName := path.Join(t.mfs.Dir(), "dir")

	// Create the directory once.
	err = os.Mkdir(dirName, 0754)
	AssertEq(nil, err)

	// Attempt to create it again.
	err = os.Mkdir(dirName, 0754)

	AssertNe(nil, err)
	ExpectThat(err, Error(HasSubstr("exists")))
}

func (t *MemFSTest) Mkdir_IntermediateIsFile() {
	var err error

	// Create a file.
	fileName := path.Join(t.mfs.Dir(), "foo")
	err = ioutil.WriteFile(fileName, []byte{}, 0700)
	AssertEq(nil, err)

	// Attempt to create a directory within the file.
	dirName := path.Join(fileName, "dir")
	err = os.Mkdir(dirName, 0754)

	AssertNe(nil, err)
	ExpectThat(err, Error(HasSubstr("TODO")))
}

func (t *MemFSTest) Mkdir_IntermediateIsNonExistent() {
	var err error

	// Attempt to create a sub-directory of a non-existent sub-directory.
	dirName := path.Join(t.mfs.Dir(), "foo/dir")
	err = os.Mkdir(dirName, 0754)

	AssertNe(nil, err)
	ExpectThat(err, Error(HasSubstr("no such file or directory")))
}

func (t *MemFSTest) Mkdir_PermissionDenied() {
	var err error

	// Create a directory within the root without write permissions.
	err = os.Mkdir(path.Join(t.mfs.Dir(), "parent"), 0500)
	AssertEq(nil, err)

	// Attempt to create a child of that directory.
	err = os.Mkdir(path.Join(t.mfs.Dir(), "parent/dir"), 0754)

	AssertNe(nil, err)
	ExpectThat(err, Error(HasSubstr("permission denied")))
}

func (t *MemFSTest) CreateNewFile_InRoot() {
	AssertTrue(false, "TODO")
}

func (t *MemFSTest) CreateNewFile_InSubDir() {
	AssertTrue(false, "TODO")
}

func (t *MemFSTest) ModifyExistingFile_InRoot() {
	AssertTrue(false, "TODO")
}

func (t *MemFSTest) ModifyExistingFile_InSubDir() {
	AssertTrue(false, "TODO")
}

func (t *MemFSTest) UnlinkFile_Exists() {
	AssertTrue(false, "TODO")
}

func (t *MemFSTest) UnlinkFile_NotAFile() {
	AssertTrue(false, "TODO")
}

func (t *MemFSTest) UnlinkFile_NonExistent() {
	AssertTrue(false, "TODO")
}

func (t *MemFSTest) Rmdir_NonEmpty() {
	var err error

	// Create two levels of directories.
	err = os.MkdirAll(path.Join(t.mfs.Dir(), "foo/bar"), 0754)
	AssertEq(nil, err)

	// Attempt to remove the parent.
	err = os.Remove(path.Join(t.mfs.Dir(), "foo"))

	AssertNe(nil, err)
	ExpectThat(err, Error(HasSubstr("not empty")))
}

func (t *MemFSTest) Rmdir_Empty() {
	var err error
	var entries []os.FileInfo

	// Create two levels of directories.
	err = os.MkdirAll(path.Join(t.mfs.Dir(), "foo/bar"), 0754)
	AssertEq(nil, err)

	// Remove the leaf.
	err = os.Remove(path.Join(t.mfs.Dir(), "foo/bar"))
	AssertEq(nil, err)

	// There should be nothing left in the parent.
	entries, err = ioutil.ReadDir(path.Join(t.mfs.Dir(), "foo"))

	AssertEq(nil, err)
	ExpectThat(entries, ElementsAre())

	// Remove the parent.
	err = os.Remove(path.Join(t.mfs.Dir(), "foo"))
	AssertEq(nil, err)

	// Now the root directory should be empty, too.
	entries, err = ioutil.ReadDir(t.mfs.Dir())

	AssertEq(nil, err)
	ExpectThat(entries, ElementsAre())
}

func (t *MemFSTest) Rmdir_NonExistent() {
	err := os.Remove(path.Join(t.mfs.Dir(), "blah"))

	AssertNe(nil, err)
	ExpectThat(err, Error(HasSubstr("no such file or directory")))
}

func (t *MemFSTest) Rmdir_ReusesInodeID() {
	var err error
	var fi os.FileInfo

	// Create a directory and record its inode ID.
	err = os.Mkdir(path.Join(t.mfs.Dir(), "dir"), 0700)
	AssertEq(nil, err)

	fi, err = os.Stat(path.Join(t.mfs.Dir(), "dir"))
	AssertEq(nil, err)
	inodeID := fi.Sys().(*syscall.Stat_t).Ino

	// Remove the directory.
	err = os.Remove(path.Join(t.mfs.Dir(), "dir"))
	AssertEq(nil, err)

	// Create a new directory. It should receive the most recently de-allocated
	// inode ID.
	err = os.Mkdir(path.Join(t.mfs.Dir(), "blah"), 0700)
	AssertEq(nil, err)

	fi, err = os.Stat(path.Join(t.mfs.Dir(), "blah"))
	AssertEq(nil, err)
	ExpectEq(inodeID, fi.Sys().(*syscall.Stat_t).Ino)
}

func (t *MemFSTest) Rmdir_OpenedForReading() {
	var err error

	// Create a directory.
	createTime := t.clock.Now()
	err = os.Mkdir(path.Join(t.mfs.Dir(), "dir"), 0700)
	AssertEq(nil, err)

	// Simulate time advancing.
	t.clock.AdvanceTime(time.Second)

	// Open the directory for reading.
	f, err := os.Open(path.Join(t.mfs.Dir(), "dir"))
	defer func() {
		if f != nil {
			ExpectEq(nil, f.Close())
		}
	}()

	AssertEq(nil, err)

	// Remove the directory.
	err = os.Remove(path.Join(t.mfs.Dir(), "dir"))
	AssertEq(nil, err)

	// Create a new directory, with the same name even, and add some contents
	// within it.
	err = os.MkdirAll(path.Join(t.mfs.Dir(), "dir/foo"), 0700)
	AssertEq(nil, err)

	err = os.MkdirAll(path.Join(t.mfs.Dir(), "dir/bar"), 0700)
	AssertEq(nil, err)

	err = os.MkdirAll(path.Join(t.mfs.Dir(), "dir/baz"), 0700)
	AssertEq(nil, err)

	// We should still be able to stat the open file handle. It should show up as
	// unlinked.
	fi, err := f.Stat()

	ExpectEq("dir", fi.Name())
	ExpectEq(0, fi.ModTime().Sub(createTime))

	// TODO(jacobsa): Re-enable this assertion if the following issue is fixed:
	//     https://github.com/bazillion/fuse/issues/66
	// ExpectEq(0, fi.Sys().(*syscall.Stat_t).Nlink)

	// Attempt to read from the directory. This should succeed even though it has
	// been unlinked, and we shouldn't see any junk from the new directory.
	entries, err := f.Readdir(0)

	AssertEq(nil, err)
	ExpectThat(entries, ElementsAre())
}
<commit_msg>Deleted MemFSTest.Rmdir_ReusesInodeID.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)

package memfs_test

import (
	"io/ioutil"
	"log"
	"os"
	"os/user"
	"path"
	"strconv"
	"strings"
	"syscall"
	"testing"
	"time"

	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/samples/memfs"
	"github.com/jacobsa/gcsfuse/timeutil"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
	"golang.org/x/net/context"
)

func TestMemFS(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func currentUid() uint32 {
	user, err := user.Current()
	if err != nil {
		panic(err)
	}

	uid, err := strconv.ParseUint(user.Uid, 10, 32)
	if err != nil {
		panic(err)
	}

	return uint32(uid)
}

func currentGid() uint32 {
	user, err := user.Current()
	if err != nil {
		panic(err)
	}

	gid, err := strconv.ParseUint(user.Gid, 10, 32)
	if err != nil {
		panic(err)
	}

	return uint32(gid)
}

func timespecToTime(ts syscall.Timespec) time.Time {
	return time.Unix(ts.Sec, ts.Nsec)
}

////////////////////////////////////////////////////////////////////////
// Boilerplate
////////////////////////////////////////////////////////////////////////

type MemFSTest struct {
	clock timeutil.SimulatedClock
	mfs   *fuse.MountedFileSystem
}

var _ SetUpInterface = &MemFSTest{}
var _ TearDownInterface = &MemFSTest{}

func init() { RegisterTestSuite(&MemFSTest{}) }

func (t *MemFSTest) SetUp(ti *TestInfo) {
	var err error

	// Set up a fixed, non-zero time.
	t.clock.SetTime(time.Now())

	// Set up a temporary directory for mounting.
	mountPoint, err := ioutil.TempDir("", "memfs_test")
	if err != nil {
		panic("ioutil.TempDir: " + err.Error())
	}

	// Mount a file system.
	fs := memfs.NewMemFS(&t.clock)
	if t.mfs, err = fuse.Mount(mountPoint, fs); err != nil {
		panic("Mount: " + err.Error())
	}

	if err = t.mfs.WaitForReady(context.Background()); err != nil {
		panic("MountedFileSystem.WaitForReady: " + err.Error())
	}
}

func (t *MemFSTest) TearDown() {
	// Unmount the file system. Try again on "resource busy" errors.
	delay := 10 * time.Millisecond
	for {
		err := t.mfs.Unmount()
		if err == nil {
			break
		}

		if strings.Contains(err.Error(), "resource busy") {
			log.Println("Resource busy error while unmounting; trying again")
			time.Sleep(delay)
			delay = time.Duration(1.3 * float64(delay))
			continue
		}

		panic("MountedFileSystem.Unmount: " + err.Error())
	}

	if err := t.mfs.Join(context.Background()); err != nil {
		panic("MountedFileSystem.Join: " + err.Error())
	}
}

////////////////////////////////////////////////////////////////////////
// Test functions
////////////////////////////////////////////////////////////////////////

func (t *MemFSTest) ContentsOfEmptyFileSystem() {
	entries, err := ioutil.ReadDir(t.mfs.Dir())

	AssertEq(nil, err)
	ExpectThat(entries, ElementsAre())
}

func (t *MemFSTest) Mkdir_OneLevel() {
	var err error
	var fi os.FileInfo
	var stat *syscall.Stat_t
	var entries []os.FileInfo

	dirName := path.Join(t.mfs.Dir(), "dir")

	// Create a directory within the root.
	createTime := t.clock.Now()
	err = os.Mkdir(dirName, 0754)
	AssertEq(nil, err)

	// Simulate time advancing.
	t.clock.AdvanceTime(time.Second)

	// Stat the directory.
	fi, err = os.Stat(dirName)
	stat = fi.Sys().(*syscall.Stat_t)

	AssertEq(nil, err)
	ExpectEq("dir", fi.Name())
	ExpectEq(0, fi.Size())
	ExpectEq(os.ModeDir|0754, fi.Mode())
	ExpectEq(0, fi.ModTime().Sub(createTime))
	ExpectTrue(fi.IsDir())

	ExpectNe(0, stat.Ino)
	ExpectEq(1, stat.Nlink)
	ExpectEq(currentUid(), stat.Uid)
	ExpectEq(currentGid(), stat.Gid)
	ExpectEq(0, stat.Size)
	ExpectEq(0, timespecToTime(stat.Atimespec).Sub(createTime))
	ExpectEq(0, timespecToTime(stat.Mtimespec).Sub(createTime))
	ExpectEq(0, timespecToTime(stat.Ctimespec).Sub(createTime))

	// Read the directory.
	entries, err = ioutil.ReadDir(dirName)

	AssertEq(nil, err)
	ExpectThat(entries, ElementsAre())

	// Read the root.
	entries, err = ioutil.ReadDir(t.mfs.Dir())

	AssertEq(nil, err)
	AssertEq(1, len(entries))

	fi = entries[0]
	ExpectEq("dir", fi.Name())
	ExpectEq(os.ModeDir|0754, fi.Mode())
}

func (t *MemFSTest) Mkdir_TwoLevels() {
	var err error
	var fi os.FileInfo
	var stat *syscall.Stat_t
	var entries []os.FileInfo

	// Create a directory within the root.
	err = os.Mkdir(path.Join(t.mfs.Dir(), "parent"), 0700)
	AssertEq(nil, err)

	// Create a child of that directory.
	createTime := t.clock.Now()
	err = os.Mkdir(path.Join(t.mfs.Dir(), "parent/dir"), 0754)
	AssertEq(nil, err)

	// Simulate time advancing.
	t.clock.AdvanceTime(time.Second)

	// Stat the directory.
	fi, err = os.Stat(path.Join(t.mfs.Dir(), "parent/dir"))
	stat = fi.Sys().(*syscall.Stat_t)

	AssertEq(nil, err)
	ExpectEq("dir", fi.Name())
	ExpectEq(0, fi.Size())
	ExpectEq(os.ModeDir|0754, fi.Mode())
	ExpectEq(0, fi.ModTime().Sub(createTime))
	ExpectTrue(fi.IsDir())

	ExpectNe(0, stat.Ino)
	ExpectEq(1, stat.Nlink)
	ExpectEq(currentUid(), stat.Uid)
	ExpectEq(currentGid(), stat.Gid)
	ExpectEq(0, stat.Size)
	ExpectEq(0, timespecToTime(stat.Atimespec).Sub(createTime))
	ExpectEq(0, timespecToTime(stat.Mtimespec).Sub(createTime))
	ExpectEq(0, timespecToTime(stat.Ctimespec).Sub(createTime))

	// Read the directory.
	entries, err = ioutil.ReadDir(path.Join(t.mfs.Dir(), "parent/dir"))

	AssertEq(nil, err)
	ExpectThat(entries, ElementsAre())

	// Read the parent.
	entries, err = ioutil.ReadDir(path.Join(t.mfs.Dir(), "parent"))

	AssertEq(nil, err)
	AssertEq(1, len(entries))

	fi = entries[0]
	ExpectEq("dir", fi.Name())
	ExpectEq(os.ModeDir|0754, fi.Mode())
}

func (t *MemFSTest) Mkdir_AlreadyExists() {
	var err error
	dirName := path.Join(t.mfs.Dir(), "dir")

	// Create the directory once.
	err = os.Mkdir(dirName, 0754)
	AssertEq(nil, err)

	// Attempt to create it again.
	err = os.Mkdir(dirName, 0754)

	AssertNe(nil, err)
	ExpectThat(err, Error(HasSubstr("exists")))
}

func (t *MemFSTest) Mkdir_IntermediateIsFile() {
	var err error

	// Create a file.
	fileName := path.Join(t.mfs.Dir(), "foo")
	err = ioutil.WriteFile(fileName, []byte{}, 0700)
	AssertEq(nil, err)

	// Attempt to create a directory within the file.
	dirName := path.Join(fileName, "dir")
	err = os.Mkdir(dirName, 0754)

	AssertNe(nil, err)
	ExpectThat(err, Error(HasSubstr("TODO")))
}

func (t *MemFSTest) Mkdir_IntermediateIsNonExistent() {
	var err error

	// Attempt to create a sub-directory of a non-existent sub-directory.
	dirName := path.Join(t.mfs.Dir(), "foo/dir")
	err = os.Mkdir(dirName, 0754)

	AssertNe(nil, err)
	ExpectThat(err, Error(HasSubstr("no such file or directory")))
}

func (t *MemFSTest) Mkdir_PermissionDenied() {
	var err error

	// Create a directory within the root without write permissions.
	err = os.Mkdir(path.Join(t.mfs.Dir(), "parent"), 0500)
	AssertEq(nil, err)

	// Attempt to create a child of that directory.
	err = os.Mkdir(path.Join(t.mfs.Dir(), "parent/dir"), 0754)

	AssertNe(nil, err)
	ExpectThat(err, Error(HasSubstr("permission denied")))
}

func (t *MemFSTest) CreateNewFile_InRoot() {
	AssertTrue(false, "TODO")
}

func (t *MemFSTest) CreateNewFile_InSubDir() {
	AssertTrue(false, "TODO")
}

func (t *MemFSTest) ModifyExistingFile_InRoot() {
	AssertTrue(false, "TODO")
}

func (t *MemFSTest) ModifyExistingFile_InSubDir() {
	AssertTrue(false, "TODO")
}

func (t *MemFSTest) UnlinkFile_Exists() {
	AssertTrue(false, "TODO")
}

func (t *MemFSTest) UnlinkFile_NotAFile() {
	AssertTrue(false, "TODO")
}

func (t *MemFSTest) UnlinkFile_NonExistent() {
	AssertTrue(false, "TODO")
}

func (t *MemFSTest) Rmdir_NonEmpty() {
	var err error

	// Create two levels of directories.
	err = os.MkdirAll(path.Join(t.mfs.Dir(), "foo/bar"), 0754)
	AssertEq(nil, err)

	// Attempt to remove the parent.
	err = os.Remove(path.Join(t.mfs.Dir(), "foo"))

	AssertNe(nil, err)
	ExpectThat(err, Error(HasSubstr("not empty")))
}

func (t *MemFSTest) Rmdir_Empty() {
	var err error
	var entries []os.FileInfo

	// Create two levels of directories.
	err = os.MkdirAll(path.Join(t.mfs.Dir(), "foo/bar"), 0754)
	AssertEq(nil, err)

	// Remove the leaf.
	err = os.Remove(path.Join(t.mfs.Dir(), "foo/bar"))
	AssertEq(nil, err)

	// There should be nothing left in the parent.
	entries, err = ioutil.ReadDir(path.Join(t.mfs.Dir(), "foo"))

	AssertEq(nil, err)
	ExpectThat(entries, ElementsAre())

	// Remove the parent.
	err = os.Remove(path.Join(t.mfs.Dir(), "foo"))
	AssertEq(nil, err)

	// Now the root directory should be empty, too.
	entries, err = ioutil.ReadDir(t.mfs.Dir())

	AssertEq(nil, err)
	ExpectThat(entries, ElementsAre())
}

func (t *MemFSTest) Rmdir_NonExistent() {
	err := os.Remove(path.Join(t.mfs.Dir(), "blah"))

	AssertNe(nil, err)
	ExpectThat(err, Error(HasSubstr("no such file or directory")))
}

func (t *MemFSTest) Rmdir_OpenedForReading() {
	var err error

	// Create a directory.
	createTime := t.clock.Now()
	err = os.Mkdir(path.Join(t.mfs.Dir(), "dir"), 0700)
	AssertEq(nil, err)

	// Simulate time advancing.
	t.clock.AdvanceTime(time.Second)

	// Open the directory for reading.
	f, err := os.Open(path.Join(t.mfs.Dir(), "dir"))
	defer func() {
		if f != nil {
			ExpectEq(nil, f.Close())
		}
	}()

	AssertEq(nil, err)

	// Remove the directory.
	err = os.Remove(path.Join(t.mfs.Dir(), "dir"))
	AssertEq(nil, err)

	// Create a new directory, with the same name even, and add some contents
	// within it.
	err = os.MkdirAll(path.Join(t.mfs.Dir(), "dir/foo"), 0700)
	AssertEq(nil, err)

	err = os.MkdirAll(path.Join(t.mfs.Dir(), "dir/bar"), 0700)
	AssertEq(nil, err)

	err = os.MkdirAll(path.Join(t.mfs.Dir(), "dir/baz"), 0700)
	AssertEq(nil, err)

	// We should still be able to stat the open file handle. It should show up as
	// unlinked.
	fi, err := f.Stat()

	ExpectEq("dir", fi.Name())
	ExpectEq(0, fi.ModTime().Sub(createTime))

	// TODO(jacobsa): Re-enable this assertion if the following issue is fixed:
	//     https://github.com/bazillion/fuse/issues/66
	// ExpectEq(0, fi.Sys().(*syscall.Stat_t).Nlink)

	// Attempt to read from the directory. This should succeed even though it has
	// been unlinked, and we shouldn't see any junk from the new directory.
	entries, err := f.Readdir(0)

	AssertEq(nil, err)
	ExpectThat(entries, ElementsAre())
}
<|endoftext|>


<commit_before>package service

import (
	"context"
	"fmt"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/micro/go-micro/v2/auth"
	pb "github.com/micro/go-micro/v2/auth/service/proto"
	"github.com/micro/go-micro/v2/auth/token"
	"github.com/micro/go-micro/v2/auth/token/jwt"
	"github.com/micro/go-micro/v2/client"
	log "github.com/micro/go-micro/v2/logger"
	"github.com/micro/go-micro/v2/util/jitter"
)

// NewAuth returns a new instance of the Auth service
func NewAuth(opts ...auth.Option) auth.Auth {
	return &svc{options: auth.NewOptions(opts...)}
}

// svc is the service implementation of the Auth interface
type svc struct {
	options auth.Options
	auth    pb.AuthService
	rule    pb.RulesService
	jwt     token.Provider

	rules []*pb.Rule
	sync.Mutex
}

func (s *svc) String() string {
	return "service"
}

func (s *svc) Init(opts ...auth.Option) {
	for _, o := range opts {
		o(&s.options)
	}

	if s.options.Client == nil {
		s.options.Client = client.DefaultClient
	}
	s.auth = pb.NewAuthService("go.micro.auth", s.options.Client)
	s.rule = pb.NewRulesService("go.micro.auth", s.options.Client)

	// if we have a JWT public key passed as an option,
	// we can decode tokens with the type "JWT" locally
	// and not have to make an RPC call
	if key := s.options.PublicKey; len(key) > 0 {
		s.jwt = jwt.NewTokenProvider(token.WithPublicKey(key))
	}

	// load rules periodically from the auth service
	go func() {
		ruleTimer := time.NewTicker(time.Second * 30)

		// load rules immediately on startup
		s.loadRules()

		for {
			<-ruleTimer.C

			// jitter for up to 5 seconds, this stops
			// all the services calling the auth service
			// at the exact same time
			time.Sleep(jitter.Do(time.Second * 5))
			s.loadRules()
		}
	}()
}

func (s *svc) Options() auth.Options {
	s.Lock()
	defer s.Unlock()
	return s.options
}

// Generate a new account
func (s *svc) Generate(id string, opts ...auth.GenerateOption) (*auth.Account, error) {
	options := auth.NewGenerateOptions(opts...)

	rsp, err := s.auth.Generate(context.TODO(), &pb.GenerateRequest{
		Id:        id,
		Type:      options.Type,
		Secret:    options.Secret,
		Roles:     options.Roles,
		Metadata:  options.Metadata,
		Provider:  options.Provider,
		Namespace: options.Namespace,
	})
	if err != nil {
		return nil, err
	}

	return serializeAccount(rsp.Account), nil
}

// Grant access to a resource
func (s *svc) Grant(role string, res *auth.Resource) error {
	_, err := s.rule.Create(context.TODO(), &pb.CreateRequest{
		Role:   role,
		Access: pb.Access_GRANTED,
		Resource: &pb.Resource{
			Namespace: res.Namespace,
			Type:      res.Type,
			Name:      res.Name,
			Endpoint:  res.Endpoint,
		},
	})
	return err
}

// Revoke access to a resource
func (s *svc) Revoke(role string, res *auth.Resource) error {
	_, err := s.rule.Delete(context.TODO(), &pb.DeleteRequest{
		Role:   role,
		Access: pb.Access_GRANTED,
		Resource: &pb.Resource{
			Namespace: res.Namespace,
			Type:      res.Type,
			Name:      res.Name,
			Endpoint:  res.Endpoint,
		},
	})
	return err
}

// Verify an account has access to a resource
func (s *svc) Verify(acc *auth.Account, res *auth.Resource) error {
	// set the namespace on the resource
	if len(res.Namespace) == 0 {
		res.Namespace = s.Options().Namespace
	}

	queries := [][]string{
		{res.Namespace, res.Type, res.Name, res.Endpoint}, // check for specific role, e.g. service.foo.ListFoo:admin (role is checked in accessForRule)
		{res.Namespace, res.Type, res.Name, "*"},          // check for wildcard endpoint, e.g. service.foo*
		{res.Namespace, res.Type, "*"},                    // check for wildcard name, e.g. service.*
		{res.Namespace, "*"},                              // check for wildcard type, e.g. *
		{"*"},                                             // check for wildcard namespace
	}

	// endpoint is a url which can have wildcard excludes, e.g.
	// "/foo/*" will allow "/foo/bar"
	if comps := strings.Split(res.Endpoint, "/"); len(comps) > 1 {
		for i := 1; i < len(comps); i++ {
			wildcard := fmt.Sprintf("%v/*", strings.Join(comps[0:i], "/"))
			queries = append(queries, []string{res.Type, res.Name, wildcard})
		}
	}

	// set a default account id / namespace to log
	logID := acc.ID
	if len(logID) == 0 {
		logID = "[no account]"
	}
	logNamespace := acc.Namespace
	if len(logNamespace) == 0 {
		logNamespace = "[no namespace]"
	}

	for _, q := range queries {
		for _, rule := range s.listRules(q...) {
			switch accessForRule(rule, acc, res) {
			case pb.Access_UNKNOWN:
				continue // rule did not specify access, check the next rule
			case pb.Access_GRANTED:
				log.Tracef("%v:%v granted access to %v:%v:%v:%v by rule %v", logNamespace, logID, res.Namespace, res.Type, res.Name, res.Endpoint, rule.Id)
				return nil // rule grants the account access to the resource
			case pb.Access_DENIED:
				log.Tracef("%v:%v denied access to %v:%v:%v:%v by rule %v", logNamespace, logID, res.Namespace, res.Type, res.Name, res.Endpoint, rule.Id)
				return auth.ErrForbidden // rule denies access to the resource
			}
		}
	}

	// no rules were found for the resource, default to denying access
	log.Tracef("%v:%v denied access to %v:%v:%v:%v by lack of rule (%v rules found for namespace)", logNamespace, logID, res.Namespace, res.Type, res.Name, res.Endpoint, len(s.listRules(res.Namespace)))
	return auth.ErrForbidden
}

// Inspect a token
func (s *svc) Inspect(token string) (*auth.Account, error) {
	// try to decode JWT locally and fall back to srv if an error occurs
	if len(strings.Split(token, ".")) == 3 && s.jwt != nil {
		return s.jwt.Inspect(token)
	}

	// the token is not a JWT or we do not have the keys to decode it,
	// fall back to the auth service
	rsp, err := s.auth.Inspect(context.TODO(), &pb.InspectRequest{Token: token})
	if err != nil {
		return nil, err
	}
	return serializeAccount(rsp.Account), nil
}

// Token generation using an account ID and secret
func (s *svc) Token(opts ...auth.TokenOption) (*auth.Token, error) {
	options := auth.NewTokenOptions(opts...)

	rsp, err := s.auth.Token(context.Background(), &pb.TokenRequest{
		Id:           options.ID,
		Secret:       options.Secret,
		RefreshToken: options.RefreshToken,
		TokenExpiry:  int64(options.Expiry.Seconds()),
	})
	if err != nil {
		return nil, err
	}

	return serializeToken(rsp.Token), nil
}

var ruleJoinKey = ":"

// accessForRule returns a rule status, indicating if a rule permits access to a
// resource for a given account
func accessForRule(rule *pb.Rule, acc *auth.Account, res *auth.Resource) pb.Access {
	if rule.Role == "*" {
		return rule.Access
	}

	for _, role := range acc.Roles {
		if rule.Role == role {
			return rule.Access
		}

		// allow user.anything if role is user.*
		if strings.HasSuffix(rule.Role, ".*") && strings.HasPrefix(rule.Role, role+".") {
			return rule.Access
		}
	}

	return pb.Access_UNKNOWN
}

// listRules gets all the rules from the store which match the filters.
// filters are namespace, type, name and then endpoint.
func (s *svc) listRules(filters ...string) []*pb.Rule {
	s.Lock()
	defer s.Unlock()

	var rules []*pb.Rule
	for _, r := range s.rules {
		if len(filters) > 0 && r.Resource.Namespace != filters[0] {
			continue
		}
		if len(filters) > 1 && r.Resource.Type != filters[1] {
			continue
		}
		if len(filters) > 2 && r.Resource.Name != filters[2] {
			continue
		}
		if len(filters) > 3 && r.Resource.Endpoint != filters[3] {
			continue
		}

		rules = append(rules, r)
	}

	// sort rules by priority
	sort.Slice(rules, func(i, j int) bool {
		return rules[i].Priority < rules[j].Priority
	})

	return rules
}

// loadRules retrieves the rules from the auth service
func (s *svc) loadRules() {
	rsp, err := s.rule.List(context.TODO(), &pb.ListRequest{})
	s.Lock()
	defer s.Unlock()

	if err != nil {
		log.Errorf("Error listing rules: %v", err)
		return
	}

	s.rules = rsp.Rules
}

func serializeToken(t *pb.Token) *auth.Token {
	return &auth.Token{
		AccessToken:  t.AccessToken,
		RefreshToken: t.RefreshToken,
		Created:      time.Unix(t.Created, 0),
		Expiry:       time.Unix(t.Expiry, 0),
	}
}

func serializeAccount(a *pb.Account) *auth.Account {
	return &auth.Account{
		ID:        a.Id,
		Roles:     a.Roles,
		Secret:    a.Secret,
		Metadata:  a.Metadata,
		Provider:  a.Provider,
		Namespace: a.Namespace,
	}
}
<commit_msg>Auth: Load rules if not present<commit_after>package service

import (
	"context"
	"fmt"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/micro/go-micro/v2/auth"
	pb "github.com/micro/go-micro/v2/auth/service/proto"
	"github.com/micro/go-micro/v2/auth/token"
	"github.com/micro/go-micro/v2/auth/token/jwt"
	"github.com/micro/go-micro/v2/client"
	log "github.com/micro/go-micro/v2/logger"
	"github.com/micro/go-micro/v2/util/jitter"
)

// NewAuth returns a new instance of the Auth service
func NewAuth(opts ...auth.Option) auth.Auth {
	return &svc{options: auth.NewOptions(opts...)}
}

// svc is the service implementation of the Auth interface
type svc struct {
	options auth.Options
	auth    pb.AuthService
	rule    pb.RulesService
	jwt     token.Provider

	rules []*pb.Rule
	sync.Mutex
}

func (s *svc) String() string {
	return "service"
}

func (s *svc) Init(opts ...auth.Option) {
	for _, o := range opts {
		o(&s.options)
	}

	if s.options.Client == nil {
		s.options.Client = client.DefaultClient
	}
	s.auth = pb.NewAuthService("go.micro.auth", s.options.Client)
	s.rule = pb.NewRulesService("go.micro.auth", s.options.Client)

	// if we have a JWT public key passed as an option,
	// we can decode tokens with the type "JWT" locally
	// and not have to make an RPC call
	if key := s.options.PublicKey; len(key) > 0 {
		s.jwt = jwt.NewTokenProvider(token.WithPublicKey(key))
	}

	// load rules periodically from the auth service
	go func() {
		ruleTimer := time.NewTicker(time.Second * 30)

		for {
			// jitter for up to 5 seconds, this stops
			// all the services calling the auth service
			// at the exact same time
			time.Sleep(jitter.Do(time.Second * 5))
			s.loadRules()

			<-ruleTimer.C
		}
	}()
}

func (s *svc) Options() auth.Options {
	s.Lock()
	defer s.Unlock()
	return s.options
}

// Generate a new account
func (s *svc) Generate(id string, opts ...auth.GenerateOption) (*auth.Account, error) {
	options := auth.NewGenerateOptions(opts...)

	rsp, err := s.auth.Generate(context.TODO(), &pb.GenerateRequest{
		Id:        id,
		Type:      options.Type,
		Secret:    options.Secret,
		Roles:     options.Roles,
		Metadata:  options.Metadata,
		Provider:  options.Provider,
		Namespace: options.Namespace,
	})
	if err != nil {
		return nil, err
	}

	return serializeAccount(rsp.Account), nil
}

// Grant access to a resource
func (s *svc) Grant(role string, res *auth.Resource) error {
	_, err := s.rule.Create(context.TODO(), &pb.CreateRequest{
		Role:   role,
		Access: pb.Access_GRANTED,
		Resource: &pb.Resource{
			Namespace: res.Namespace,
			Type:      res.Type,
			Name:      res.Name,
			Endpoint:  res.Endpoint,
		},
	})
	return err
}

// Revoke access to a resource
func (s *svc) Revoke(role string, res *auth.Resource) error {
	_, err := s.rule.Delete(context.TODO(), &pb.DeleteRequest{
		Role:   role,
		Access: pb.Access_GRANTED,
		Resource: &pb.Resource{
			Namespace: res.Namespace,
			Type:      res.Type,
			Name:      res.Name,
			Endpoint:  res.Endpoint,
		},
	})
	return err
}

// Verify an account has access to a resource
func (s *svc) Verify(acc *auth.Account, res *auth.Resource) error {
	// load the rules if none are loaded
	s.loadRulesIfEmpty()

	// set the namespace on the resource
	if len(res.Namespace) == 0 {
		res.Namespace = s.Options().Namespace
	}

	queries := [][]string{
		{res.Namespace, res.Type, res.Name, res.Endpoint}, // check for specific role, e.g. service.foo.ListFoo:admin (role is checked in accessForRule)
		{res.Namespace, res.Type, res.Name, "*"},          // check for wildcard endpoint, e.g. service.foo*
		{res.Namespace, res.Type, "*"},                    // check for wildcard name, e.g. service.*
		{res.Namespace, "*"},                              // check for wildcard type, e.g. *
		{"*"},                                             // check for wildcard namespace
	}

	// endpoint is a url which can have wildcard excludes, e.g.
	// "/foo/*" will allow "/foo/bar"
	if comps := strings.Split(res.Endpoint, "/"); len(comps) > 1 {
		for i := 1; i < len(comps); i++ {
			wildcard := fmt.Sprintf("%v/*", strings.Join(comps[0:i], "/"))
			queries = append(queries, []string{res.Type, res.Name, wildcard})
		}
	}

	// set a default account id / namespace to log
	logID := acc.ID
	if len(logID) == 0 {
		logID = "[no account]"
	}
	logNamespace := acc.Namespace
	if len(logNamespace) == 0 {
		logNamespace = "[no namespace]"
	}

	for _, q := range queries {
		for _, rule := range s.listRules(q...) {
			switch accessForRule(rule, acc, res) {
			case pb.Access_UNKNOWN:
				continue // rule did not specify access, check the next rule
			case pb.Access_GRANTED:
				log.Tracef("%v:%v granted access to %v:%v:%v:%v by rule %v", logNamespace, logID, res.Namespace, res.Type, res.Name, res.Endpoint, rule.Id)
				return nil // rule grants the account access to the resource
			case pb.Access_DENIED:
				log.Tracef("%v:%v denied access to %v:%v:%v:%v by rule %v", logNamespace, logID, res.Namespace, res.Type, res.Name, res.Endpoint, rule.Id)
				return auth.ErrForbidden // rule denies access to the resource
			}
		}
	}

	// no rules were found for the resource, default to denying access
	log.Tracef("%v:%v denied access to %v:%v:%v:%v by lack of rule (%v rules found for namespace)", logNamespace, logID, res.Namespace, res.Type, res.Name, res.Endpoint, len(s.listRules(res.Namespace)))
	return auth.ErrForbidden
}

// Inspect a token
func (s *svc) Inspect(token string) (*auth.Account, error) {
	// try to decode JWT locally and fall back to srv if an error occurs
	if len(strings.Split(token, ".")) == 3 && s.jwt != nil {
		return s.jwt.Inspect(token)
	}

	// the token is not a JWT or we do not have the keys to decode it,
	// fall back to the auth service
	rsp, err := s.auth.Inspect(context.TODO(), &pb.InspectRequest{Token: token})
	if err != nil {
		return nil, err
	}
	return serializeAccount(rsp.Account), nil
}

// Token generation using an account ID and secret
func (s *svc) Token(opts ...auth.TokenOption) (*auth.Token, error) {
	options := auth.NewTokenOptions(opts...)

	rsp, err := s.auth.Token(context.Background(), &pb.TokenRequest{
		Id:           options.ID,
		Secret:       options.Secret,
		RefreshToken: options.RefreshToken,
		TokenExpiry:  int64(options.Expiry.Seconds()),
	})
	if err != nil {
		return nil, err
	}

	return serializeToken(rsp.Token), nil
}

var ruleJoinKey = ":"

// accessForRule returns a rule status, indicating if a rule permits access to a
// resource for a given account
func accessForRule(rule *pb.Rule, acc *auth.Account, res *auth.Resource) pb.Access {
	if rule.Role == "*" {
		return rule.Access
	}

	for _, role := range acc.Roles {
		if rule.Role == role {
			return rule.Access
		}

		// allow user.anything if role is user.*
		if strings.HasSuffix(rule.Role, ".*") && strings.HasPrefix(rule.Role, role+".") {
			return rule.Access
		}
	}

	return pb.Access_UNKNOWN
}

// listRules gets all the rules from the store which match the filters.
// filters are namespace, type, name and then endpoint.
func (s *svc) listRules(filters ...string) []*pb.Rule {
	s.Lock()
	defer s.Unlock()

	var rules []*pb.Rule
	for _, r := range s.rules {
		if len(filters) > 0 && r.Resource.Namespace != filters[0] {
			continue
		}
		if len(filters) > 1 && r.Resource.Type != filters[1] {
			continue
		}
		if len(filters) > 2 && r.Resource.Name != filters[2] {
			continue
		}
		if len(filters) > 3 && r.Resource.Endpoint != filters[3] {
			continue
		}

		rules = append(rules, r)
	}

	// sort rules by priority
	sort.Slice(rules, func(i, j int) bool {
		return rules[i].Priority < rules[j].Priority
	})

	return rules
}

// loadRules retrieves the rules from the auth service
func (s *svc) loadRules() {
	rsp, err := s.rule.List(context.TODO(), &pb.ListRequest{})
	s.Lock()
	defer s.Unlock()

	if err != nil {
		log.Errorf("Error listing rules: %v", err)
		return
	}

	s.rules = rsp.Rules
}

func (s *svc) loadRulesIfEmpty() {
	s.Lock()
	rules := s.rules
	s.Unlock()

	if len(rules) == 0 {
		s.loadRules()
	}
}

func serializeToken(t *pb.Token) *auth.Token {
	return &auth.Token{
		AccessToken:  t.AccessToken,
		RefreshToken: t.RefreshToken,
		Created:      time.Unix(t.Created, 0),
		Expiry:       time.Unix(t.Expiry, 0),
	}
}

func serializeAccount(a *pb.Account) *auth.Account {
	return &auth.Account{
		ID:        a.Id,
		Roles:     a.Roles,
		Secret:    a.Secret,
		Metadata:  a.Metadata,
		Provider:  a.Provider,
		Namespace: a.Namespace,
	}
}
<|endoftext|>


<commit_before>// Package azure provides Azure-specific implementations used with AutoRest.
// See the included examples for more detail.
package azure

// Copyright 2017 Microsoft Corporation
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"regexp"
	"strconv"
	"strings"

	"github.com/Azure/go-autorest/autorest"
)

const (
	// HeaderClientID is the Azure extension header to set a user-specified request ID.
	HeaderClientID = "x-ms-client-request-id"

	// HeaderReturnClientID is the Azure extension header to set if the user-specified request ID
	// should be included in the response.
	HeaderReturnClientID = "x-ms-return-client-request-id"

	// HeaderContentType is the type of the content in the HTTP response.
	HeaderContentType = "Content-Type"

	// HeaderRequestID is the Azure extension header of the service generated request ID returned
	// in the response.
	HeaderRequestID = "x-ms-request-id"
)

// ServiceError encapsulates the error response from an Azure service.
// It adhears to the OData v4 specification for error responses.
type ServiceError struct {
	Code           string                   `json:"code"`
	Message        string                   `json:"message"`
	Target         *string                  `json:"target"`
	Details        []map[string]interface{} `json:"details"`
	InnerError     map[string]interface{}   `json:"innererror"`
	AdditionalInfo []map[string]interface{} `json:"additionalInfo"`
}

func (se ServiceError) Error() string {
	result := fmt.Sprintf("Code=%q Message=%q", se.Code, se.Message)

	if se.Target != nil {
		result += fmt.Sprintf(" Target=%q", *se.Target)
	}

	if se.Details != nil {
		d, err := json.Marshal(se.Details)
		if err != nil {
			result += fmt.Sprintf(" Details=%v", se.Details)
		}
		result += fmt.Sprintf(" Details=%v", string(d))
	}

	if se.InnerError != nil {
		d, err := json.Marshal(se.InnerError)
		if err != nil {
			result += fmt.Sprintf(" InnerError=%v", se.InnerError)
		}
		result += fmt.Sprintf(" InnerError=%v", string(d))
	}

	if se.AdditionalInfo != nil {
		d, err := json.Marshal(se.AdditionalInfo)
		if err != nil {
			result += fmt.Sprintf(" AdditionalInfo=%v", se.AdditionalInfo)
		}
		result += fmt.Sprintf(" AdditionalInfo=%v", string(d))
	}

	return result
}

// UnmarshalJSON implements the json.Unmarshaler interface for the ServiceError type.
func (se *ServiceError) UnmarshalJSON(b []byte) error {
	// http://docs.oasis-open.org/odata/odata-json-format/v4.0/os/odata-json-format-v4.0-os.html#_Toc372793091

	type serviceErrorInternal struct {
		Code           string                   `json:"code"`
		Message        string                   `json:"message"`
		Target         *string                  `json:"target,omitempty"`
		AdditionalInfo []map[string]interface{} `json:"additionalInfo,omitempty"`
		// not all services conform to the OData v4 spec.
		// the following fields are where we've seen discrepancies

		// spec calls for []map[string]interface{} but have seen map[string]interface{}
		Details interface{} `json:"details,omitempty"`

		// spec calls for map[string]interface{} but have seen []map[string]interface{} and string
		InnerError interface{} `json:"innererror,omitempty"`
	}

	sei := serviceErrorInternal{}
	if err := json.Unmarshal(b, &sei); err != nil {
		return err
	}

	// copy the fields we know to be correct
	se.AdditionalInfo = sei.AdditionalInfo
	se.Code = sei.Code
	se.Message = sei.Message
	se.Target = sei.Target

	// converts an []interface{} to []map[string]interface{}
	arrayOfObjs := func(v interface{}) ([]map[string]interface{}, bool) {
		arrayOf, ok := v.([]interface{})
		if !ok {
			return nil, false
		}
		final := []map[string]interface{}{}
		for _, item := range arrayOf {
			as, ok := item.(map[string]interface{})
			if !ok {
				return nil, false
			}
			final = append(final, as)
		}
		return final, true
	}

	// convert the remaining fields, falling back to raw JSON if necessary

	if c, ok := arrayOfObjs(sei.Details); ok {
		se.Details = c
	} else if c, ok := sei.Details.(map[string]interface{}); ok {
		se.Details = []map[string]interface{}{c}
	} else if sei.Details != nil {
		// stuff into Details
		se.Details = []map[string]interface{}{
			{"raw": sei.Details},
		}
	}

	if c, ok := sei.InnerError.(map[string]interface{}); ok {
		se.InnerError = c
	} else if c, ok := arrayOfObjs(sei.InnerError); ok {
		// if there's only one error extract it
		if len(c) == 1 {
			se.InnerError = c[0]
		} else {
			// multiple errors, stuff them into the value
			se.InnerError = map[string]interface{}{
				"multi": c,
			}
		}
	} else if c, ok := sei.InnerError.(string); ok {
		se.InnerError = map[string]interface{}{"error": c}
	} else if sei.InnerError != nil {
		// stuff into InnerError
		se.InnerError = map[string]interface{}{
			"raw": sei.InnerError,
		}
	}
	return nil
}

// RequestError describes an error response returned by Azure service.
type RequestError struct {
	autorest.DetailedError

	// The error returned by the Azure service.
	ServiceError *ServiceError `json:"error" xml:"Error"`

	// The request id (from the x-ms-request-id-header) of the request.
	RequestID string
}

// Error returns a human-friendly error message from service error.
func (e RequestError) Error() string {
	return fmt.Sprintf("autorest/azure: Service returned an error. Status=%v %v",
		e.StatusCode, e.ServiceError)
}

// IsAzureError returns true if the passed error is an Azure Service error; false otherwise.
func IsAzureError(e error) bool {
	_, ok := e.(*RequestError)
	return ok
}

// Resource contains details about an Azure resource.
type Resource struct {
	SubscriptionID string
	ResourceGroup  string
	Provider       string
	ResourceType   string
	ResourceName   string
}

// String function returns a string in form of azureResourceID
func (r Resource) String() string {
	return fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/%s/%s/%s", r.SubscriptionID, r.ResourceGroup, r.Provider, r.ResourceType, r.ResourceName)
}

// ParseResourceID parses a resource ID into a ResourceDetails struct.
// See https://docs.microsoft.com/en-us/azure/azure-resource-manager/templates/template-functions-resource?tabs=json#resourceid.
func ParseResourceID(resourceID string) (Resource, error) {

	const resourceIDPatternText = `(?i)subscriptions/(.+)/resourceGroups/(.+)/providers/(.+?)/(.+?)/(.+)`
	resourceIDPattern := regexp.MustCompile(resourceIDPatternText)
	match := resourceIDPattern.FindStringSubmatch(resourceID)

	if len(match) == 0 {
		return Resource{}, fmt.Errorf("parsing failed for %s. Invalid resource Id format", resourceID)
	}

	v := strings.Split(match[5], "/")
	resourceName := v[len(v)-1]

	result := Resource{
		SubscriptionID: match[1],
		ResourceGroup:  match[2],
		Provider:       match[3],
		ResourceType:   match[4],
		ResourceName:   resourceName,
	}

	return result, nil
}

// NewErrorWithError creates a new Error conforming object from the
// passed packageType, method, statusCode of the given resp (UndefinedStatusCode
// if resp is nil), message, and original error. message is treated as a format
// string to which the optional args apply.
func NewErrorWithError(original error, packageType string, method string, resp *http.Response, message string, args ...interface{}) RequestError {
	if v, ok := original.(*RequestError); ok {
		return *v
	}

	statusCode := autorest.UndefinedStatusCode
	if resp != nil {
		statusCode = resp.StatusCode
	}
	return RequestError{
		DetailedError: autorest.DetailedError{
			Original:    original,
			PackageType: packageType,
			Method:      method,
			StatusCode:  statusCode,
			Message:     fmt.Sprintf(message, args...),
		},
	}
}

// WithReturningClientID returns a PrepareDecorator that adds an HTTP extension header of
// x-ms-client-request-id whose value is the passed, undecorated UUID (e.g.,
// "0F39878C-5F76-4DB8-A25D-61D2C193C3CA"). It also sets the x-ms-return-client-request-id
// header to true such that UUID accompanies the http.Response.
func WithReturningClientID(uuid string) autorest.PrepareDecorator {
	preparer := autorest.CreatePreparer(
		WithClientID(uuid),
		WithReturnClientID(true))

	return func(p autorest.Preparer) autorest.Preparer {
		return autorest.PreparerFunc(func(r *http.Request) (*http.Request, error) {
			r, err := p.Prepare(r)
			if err != nil {
				return r, err
			}
			return preparer.Prepare(r)
		})
	}
}

// WithClientID returns a PrepareDecorator that adds an HTTP extension header of
// x-ms-client-request-id whose value is passed, undecorated UUID (e.g.,
// "0F39878C-5F76-4DB8-A25D-61D2C193C3CA").
func WithClientID(uuid string) autorest.PrepareDecorator {
	return autorest.WithHeader(HeaderClientID, uuid)
}

// WithReturnClientID returns a PrepareDecorator that adds an HTTP extension header of
// x-ms-return-client-request-id whose boolean value indicates if the value of the
// x-ms-client-request-id header should be included in the http.Response.
func WithReturnClientID(b bool) autorest.PrepareDecorator {
	return autorest.WithHeader(HeaderReturnClientID, strconv.FormatBool(b))
}

// ExtractClientID extracts the client identifier from the x-ms-client-request-id header set on the
// http.Request sent to the service (and returned in the http.Response)
func ExtractClientID(resp *http.Response) string {
	return autorest.ExtractHeaderValue(HeaderClientID, resp)
}

// ExtractRequestID extracts the Azure server generated request identifier from the
// x-ms-request-id header.
func ExtractRequestID(resp *http.Response) string {
	return autorest.ExtractHeaderValue(HeaderRequestID, resp)
}

// WithErrorUnlessStatusCode returns a RespondDecorator that emits an
// azure.RequestError by reading the response body unless the response HTTP status code
// is among the set passed.
//
// If there is a chance service may return responses other than the Azure error
// format and the response cannot be parsed into an error, a decoding error will
// be returned containing the response body. In any case, the Responder will
// return an error if the status code is not satisfied.
//
// If this Responder returns an error, the response body will be replaced with
// an in-memory reader, which needs no further closing.
func WithErrorUnlessStatusCode(codes ...int) autorest.RespondDecorator {
	return func(r autorest.Responder) autorest.Responder {
		return autorest.ResponderFunc(func(resp *http.Response) error {
			err := r.Respond(resp)
			if err == nil && !autorest.ResponseHasStatusCode(resp, codes...) {
				var e RequestError
				defer resp.Body.Close()

				encodedAs := autorest.EncodedAsJSON
				if strings.Contains(resp.Header.Get("Content-Type"), "xml") {
					encodedAs = autorest.EncodedAsXML
				}

				// Copy and replace the Body in case it does not contain an error object.
				// This will leave the Body available to the caller.
				b, decodeErr := autorest.CopyAndDecode(encodedAs, resp.Body, &e)
				resp.Body = ioutil.NopCloser(&b)
				if decodeErr != nil {
					return fmt.Errorf("autorest/azure: error response cannot be parsed: %q error: %v", b.String(), decodeErr)
				}
				if e.ServiceError == nil {
					// Check if error is unwrapped ServiceError
					decoder := autorest.NewDecoder(encodedAs, bytes.NewReader(b.Bytes()))
					if err := decoder.Decode(&e.ServiceError); err != nil {
						return fmt.Errorf("autorest/azure: error response cannot be parsed: %q error: %v", b.String(), err)
					}

					// for example, should the API return the literal value `null` as the response
					if e.ServiceError == nil {
						e.ServiceError = &ServiceError{
							Code:    "Unknown",
							Message: "Unknown service error",
							Details: []map[string]interface{}{
								{
									"HttpResponse.Body": b.String(),
								},
							},
						}
					}
				}

				if e.ServiceError != nil && e.ServiceError.Message == "" {
					// if we're here it means the returned error wasn't OData v4 compliant.
					// try to unmarshal the body in hopes of getting something.
					rawBody := map[string]interface{}{}
					decoder := autorest.NewDecoder(encodedAs, bytes.NewReader(b.Bytes()))
					if err := decoder.Decode(&rawBody); err != nil {
						return fmt.Errorf("autorest/azure: error response cannot be parsed: %q error: %v", b.String(), err)
					}

					e.ServiceError = &ServiceError{
						Code:    "Unknown",
						Message: "Unknown service error",
					}
					if len(rawBody) > 0 {
						e.ServiceError.Details = []map[string]interface{}{rawBody}
					}
				}
				e.Response = resp
				e.RequestID = ExtractRequestID(resp)
				if e.StatusCode == nil {
					e.StatusCode = resp.StatusCode
				}
				err = &e
			}
			return err
		})
	}
}
<commit_msg>all: remove unnecessary string([]byte) conversion in fmt.Sprintf yet %s (#663)<commit_after>// Package azure provides Azure-specific implementations used with AutoRest.
// See the included examples for more detail.
package azure

// Copyright 2017 Microsoft Corporation
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"regexp"
	"strconv"
	"strings"

	"github.com/Azure/go-autorest/autorest"
)

const (
	// HeaderClientID is the Azure extension header to set a user-specified request ID.
	HeaderClientID = "x-ms-client-request-id"

	// HeaderReturnClientID is the Azure extension header to set if the user-specified request ID
	// should be included in the response.
	HeaderReturnClientID = "x-ms-return-client-request-id"

	// HeaderContentType is the type of the content in the HTTP response.
	HeaderContentType = "Content-Type"

	// HeaderRequestID is the Azure extension header of the service generated request ID returned
	// in the response.
	HeaderRequestID = "x-ms-request-id"
)

// ServiceError encapsulates the error response from an Azure service.
// It adhears to the OData v4 specification for error responses.
type ServiceError struct {
	Code           string                   `json:"code"`
	Message        string                   `json:"message"`
	Target         *string                  `json:"target"`
	Details        []map[string]interface{} `json:"details"`
	InnerError     map[string]interface{}   `json:"innererror"`
	AdditionalInfo []map[string]interface{} `json:"additionalInfo"`
}

func (se ServiceError) Error() string {
	result := fmt.Sprintf("Code=%q Message=%q", se.Code, se.Message)

	if se.Target != nil {
		result += fmt.Sprintf(" Target=%q", *se.Target)
	}

	if se.Details != nil {
		d, err := json.Marshal(se.Details)
		if err != nil {
			result += fmt.Sprintf(" Details=%v", se.Details)
		}
		result += fmt.Sprintf(" Details=%s", d)
	}

	if se.InnerError != nil {
		d, err := json.Marshal(se.InnerError)
		if err != nil {
			result += fmt.Sprintf(" InnerError=%v", se.InnerError)
		}
		result += fmt.Sprintf(" InnerError=%s", d)
	}

	if se.AdditionalInfo != nil {
		d, err := json.Marshal(se.AdditionalInfo)
		if err != nil {
			result += fmt.Sprintf(" AdditionalInfo=%v", se.AdditionalInfo)
		}
		result += fmt.Sprintf(" AdditionalInfo=%s", d)
	}

	return result
}

// UnmarshalJSON implements the json.Unmarshaler interface for the ServiceError type.
func (se *ServiceError) UnmarshalJSON(b []byte) error {
	// http://docs.oasis-open.org/odata/odata-json-format/v4.0/os/odata-json-format-v4.0-os.html#_Toc372793091

	type serviceErrorInternal struct {
		Code           string                   `json:"code"`
		Message        string                   `json:"message"`
		Target         *string                  `json:"target,omitempty"`
		AdditionalInfo []map[string]interface{} `json:"additionalInfo,omitempty"`
		// not all services conform to the OData v4 spec.
		// the following fields are where we've seen discrepancies

		// spec calls for []map[string]interface{} but have seen map[string]interface{}
		Details interface{} `json:"details,omitempty"`

		// spec calls for map[string]interface{} but have seen []map[string]interface{} and string
		InnerError interface{} `json:"innererror,omitempty"`
	}

	sei := serviceErrorInternal{}
	if err := json.Unmarshal(b, &sei); err != nil {
		return err
	}

	// copy the fields we know to be correct
	se.AdditionalInfo = sei.AdditionalInfo
	se.Code = sei.Code
	se.Message = sei.Message
	se.Target = sei.Target

	// converts an []interface{} to []map[string]interface{}
	arrayOfObjs := func(v interface{}) ([]map[string]interface{}, bool) {
		arrayOf, ok := v.([]interface{})
		if !ok {
			return nil, false
		}
		final := []map[string]interface{}{}
		for _, item := range arrayOf {
			as, ok := item.(map[string]interface{})
			if !ok {
				return nil, false
			}
			final = append(final, as)
		}
		return final, true
	}

	// convert the remaining fields, falling back to raw JSON if necessary

	if c, ok := arrayOfObjs(sei.Details); ok {
		se.Details = c
	} else if c, ok := sei.Details.(map[string]interface{}); ok {
		se.Details = []map[string]interface{}{c}
	} else if sei.Details != nil {
		// stuff into Details
		se.Details = []map[string]interface{}{
			{"raw": sei.Details},
		}
	}

	if c, ok := sei.InnerError.(map[string]interface{}); ok {
		se.InnerError = c
	} else if c, ok := arrayOfObjs(sei.InnerError); ok {
		// if there's only one error extract it
		if len(c) == 1 {
			se.InnerError = c[0]
		} else {
			// multiple errors, stuff them into the value
			se.InnerError = map[string]interface{}{
				"multi": c,
			}
		}
	} else if c, ok := sei.InnerError.(string); ok {
		se.InnerError = map[string]interface{}{"error": c}
	} else if sei.InnerError != nil {
		// stuff into InnerError
		se.InnerError = map[string]interface{}{
			"raw": sei.InnerError,
		}
	}
	return nil
}

// RequestError describes an error response returned by Azure service.
type RequestError struct {
	autorest.DetailedError

	// The error returned by the Azure service.
	ServiceError *ServiceError `json:"error" xml:"Error"`

	// The request id (from the x-ms-request-id-header) of the request.
	RequestID string
}

// Error returns a human-friendly error message from service error.
func (e RequestError) Error() string {
	return fmt.Sprintf("autorest/azure: Service returned an error. Status=%v %v",
		e.StatusCode, e.ServiceError)
}

// IsAzureError returns true if the passed error is an Azure Service error; false otherwise.
func IsAzureError(e error) bool {
	_, ok := e.(*RequestError)
	return ok
}

// Resource contains details about an Azure resource.
type Resource struct {
	SubscriptionID string
	ResourceGroup  string
	Provider       string
	ResourceType   string
	ResourceName   string
}

// String function returns a string in form of azureResourceID
func (r Resource) String() string {
	return fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/%s/%s/%s", r.SubscriptionID, r.ResourceGroup, r.Provider, r.ResourceType, r.ResourceName)
}

// ParseResourceID parses a resource ID into a ResourceDetails struct.
// See https://docs.microsoft.com/en-us/azure/azure-resource-manager/templates/template-functions-resource?tabs=json#resourceid.
func ParseResourceID(resourceID string) (Resource, error) {

	const resourceIDPatternText = `(?i)subscriptions/(.+)/resourceGroups/(.+)/providers/(.+?)/(.+?)/(.+)`
	resourceIDPattern := regexp.MustCompile(resourceIDPatternText)
	match := resourceIDPattern.FindStringSubmatch(resourceID)

	if len(match) == 0 {
		return Resource{}, fmt.Errorf("parsing failed for %s. Invalid resource Id format", resourceID)
	}

	v := strings.Split(match[5], "/")
	resourceName := v[len(v)-1]

	result := Resource{
		SubscriptionID: match[1],
		ResourceGroup:  match[2],
		Provider:       match[3],
		ResourceType:   match[4],
		ResourceName:   resourceName,
	}

	return result, nil
}

// NewErrorWithError creates a new Error conforming object from the
// passed packageType, method, statusCode of the given resp (UndefinedStatusCode
// if resp is nil), message, and original error. message is treated as a format
// string to which the optional args apply.
func NewErrorWithError(original error, packageType string, method string, resp *http.Response, message string, args ...interface{}) RequestError {
	if v, ok := original.(*RequestError); ok {
		return *v
	}

	statusCode := autorest.UndefinedStatusCode
	if resp != nil {
		statusCode = resp.StatusCode
	}
	return RequestError{
		DetailedError: autorest.DetailedError{
			Original:    original,
			PackageType: packageType,
			Method:      method,
			StatusCode:  statusCode,
			Message:     fmt.Sprintf(message, args...),
		},
	}
}

// WithReturningClientID returns a PrepareDecorator that adds an HTTP extension header of
// x-ms-client-request-id whose value is the passed, undecorated UUID (e.g.,
// "0F39878C-5F76-4DB8-A25D-61D2C193C3CA"). It also sets the x-ms-return-client-request-id
// header to true such that UUID accompanies the http.Response.
func WithReturningClientID(uuid string) autorest.PrepareDecorator {
	preparer := autorest.CreatePreparer(
		WithClientID(uuid),
		WithReturnClientID(true))

	return func(p autorest.Preparer) autorest.Preparer {
		return autorest.PreparerFunc(func(r *http.Request) (*http.Request, error) {
			r, err := p.Prepare(r)
			if err != nil {
				return r, err
			}
			return preparer.Prepare(r)
		})
	}
}

// WithClientID returns a PrepareDecorator that adds an HTTP extension header of
// x-ms-client-request-id whose value is passed, undecorated UUID (e.g.,
// "0F39878C-5F76-4DB8-A25D-61D2C193C3CA").
func WithClientID(uuid string) autorest.PrepareDecorator {
	return autorest.WithHeader(HeaderClientID, uuid)
}

// WithReturnClientID returns a PrepareDecorator that adds an HTTP extension header of
// x-ms-return-client-request-id whose boolean value indicates if the value of the
// x-ms-client-request-id header should be included in the http.Response.
func WithReturnClientID(b bool) autorest.PrepareDecorator {
	return autorest.WithHeader(HeaderReturnClientID, strconv.FormatBool(b))
}

// ExtractClientID extracts the client identifier from the x-ms-client-request-id header set on the
// http.Request sent to the service (and returned in the http.Response)
func ExtractClientID(resp *http.Response) string {
	return autorest.ExtractHeaderValue(HeaderClientID, resp)
}

// ExtractRequestID extracts the Azure server generated request identifier from the
// x-ms-request-id header.
func ExtractRequestID(resp *http.Response) string {
	return autorest.ExtractHeaderValue(HeaderRequestID, resp)
}

// WithErrorUnlessStatusCode returns a RespondDecorator that emits an
// azure.RequestError by reading the response body unless the response HTTP status code
// is among the set passed.
//
// If there is a chance service may return responses other than the Azure error
// format and the response cannot be parsed into an error, a decoding error will
// be returned containing the response body. In any case, the Responder will
// return an error if the status code is not satisfied.
//
// If this Responder returns an error, the response body will be replaced with
// an in-memory reader, which needs no further closing.
func WithErrorUnlessStatusCode(codes ...int) autorest.RespondDecorator {
	return func(r autorest.Responder) autorest.Responder {
		return autorest.ResponderFunc(func(resp *http.Response) error {
			err := r.Respond(resp)
			if err == nil && !autorest.ResponseHasStatusCode(resp, codes...) {
				var e RequestError
				defer resp.Body.Close()

				encodedAs := autorest.EncodedAsJSON
				if strings.Contains(resp.Header.Get("Content-Type"), "xml") {
					encodedAs = autorest.EncodedAsXML
				}

				// Copy and replace the Body in case it does not contain an error object.
				// This will leave the Body available to the caller.
				b, decodeErr := autorest.CopyAndDecode(encodedAs, resp.Body, &e)
				resp.Body = ioutil.NopCloser(&b)
				if decodeErr != nil {
					return fmt.Errorf("autorest/azure: error response cannot be parsed: %q error: %v", b, decodeErr)
				}
				if e.ServiceError == nil {
					// Check if error is unwrapped ServiceError
					decoder := autorest.NewDecoder(encodedAs, bytes.NewReader(b.Bytes()))
					if err := decoder.Decode(&e.ServiceError); err != nil {
						return fmt.Errorf("autorest/azure: error response cannot be parsed: %q error: %v", b, err)
					}

					// for example, should the API return the literal value `null` as the response
					if e.ServiceError == nil {
						e.ServiceError = &ServiceError{
							Code:    "Unknown",
							Message: "Unknown service error",
							Details: []map[string]interface{}{
								{
									"HttpResponse.Body": b.String(),
								},
							},
						}
					}
				}

				if e.ServiceError != nil && e.ServiceError.Message == "" {
					// if we're here it means the returned error wasn't OData v4 compliant.
					// try to unmarshal the body in hopes of getting something.
					rawBody := map[string]interface{}{}
					decoder := autorest.NewDecoder(encodedAs, bytes.NewReader(b.Bytes()))
					if err := decoder.Decode(&rawBody); err != nil {
						return fmt.Errorf("autorest/azure: error response cannot be parsed: %q error: %v", b, err)
					}

					e.ServiceError = &ServiceError{
						Code:    "Unknown",
						Message: "Unknown service error",
					}
					if len(rawBody) > 0 {
						e.ServiceError.Details = []map[string]interface{}{rawBody}
					}
				}
				e.Response = resp
				e.RequestID = ExtractRequestID(resp)
				if e.StatusCode == nil {
					e.StatusCode = resp.StatusCode
				}
				err = &e
			}
			return err
		})
	}
}
<|endoftext|>


<commit_before>// +build !plan9

package cache

import (
	"context"
	"io"
	"path"
	"sync"
	"time"

	"github.com/pkg/errors"
	"github.com/rclone/rclone/fs"
	"github.com/rclone/rclone/fs/hash"
	"github.com/rclone/rclone/lib/readers"
)

const (
	objectInCache       = "Object"
	objectPendingUpload = "TempObject"
)

// Object is a generic file like object that stores basic information about it
type Object struct {
	fs.Object `json:"-"`

	ParentFs      fs.Fs                `json:"-"`        // parent fs
	CacheFs       *Fs                  `json:"-"`        // cache fs
	Name          string               `json:"name"`     // name of the directory
	Dir           string               `json:"dir"`      // abs path of the object
	CacheModTime  int64                `json:"modTime"`  // modification or creation time - IsZero for unknown
	CacheSize     int64                `json:"size"`     // size of directory and contents or -1 if unknown
	CacheStorable bool                 `json:"storable"` // says whether this object can be stored
	CacheType     string               `json:"cacheType"`
	CacheTs       time.Time            `json:"cacheTs"`
	CacheHashes   map[hash.Type]string // all supported hashes cached

	refreshMutex sync.Mutex
}

// NewObject builds one from a generic fs.Object
func NewObject(f *Fs, remote string) *Object {
	fullRemote := path.Join(f.Root(), remote)
	dir, name := path.Split(fullRemote)

	cacheType := objectInCache
	parentFs := f.UnWrap()
	if f.opt.TempWritePath != "" {
		_, err := f.cache.SearchPendingUpload(fullRemote)
		if err == nil { // queued for upload
			cacheType = objectPendingUpload
			parentFs = f.tempFs
			fs.Debugf(fullRemote, "pending upload found")
		}
	}

	co := &Object{
		ParentFs:      parentFs,
		CacheFs:       f,
		Name:          cleanPath(name),
		Dir:           cleanPath(dir),
		CacheModTime:  time.Now().UnixNano(),
		CacheSize:     0,
		CacheStorable: false,
		CacheType:     cacheType,
		CacheTs:       time.Now(),
	}
	return co
}

// ObjectFromOriginal builds one from a generic fs.Object
func ObjectFromOriginal(ctx context.Context, f *Fs, o fs.Object) *Object {
	var co *Object
	fullRemote := cleanPath(path.Join(f.Root(), o.Remote()))
	dir, name := path.Split(fullRemote)

	cacheType := objectInCache
	parentFs := f.UnWrap()
	if f.opt.TempWritePath != "" {
		_, err := f.cache.SearchPendingUpload(fullRemote)
		if err == nil { // queued for upload
			cacheType = objectPendingUpload
			parentFs = f.tempFs
			fs.Debugf(fullRemote, "pending upload found")
		}
	}

	co = &Object{
		ParentFs:  parentFs,
		CacheFs:   f,
		Name:      cleanPath(name),
		Dir:       cleanPath(dir),
		CacheType: cacheType,
		CacheTs:   time.Now(),
	}
	co.updateData(ctx, o)
	return co
}

func (o *Object) updateData(ctx context.Context, source fs.Object) {
	o.Object = source
	o.CacheModTime = source.ModTime(ctx).UnixNano()
	o.CacheSize = source.Size()
	o.CacheStorable = source.Storable()
	o.CacheTs = time.Now()
	o.CacheHashes = make(map[hash.Type]string)
}

// Fs returns its FS info
func (o *Object) Fs() fs.Info {
	return o.CacheFs
}

// String returns a human friendly name for this object
func (o *Object) String() string {
	if o == nil {
		return "<nil>"
	}
	return o.Remote()
}

// Remote returns the remote path
func (o *Object) Remote() string {
	p := path.Join(o.Dir, o.Name)
	return o.CacheFs.cleanRootFromPath(p)
}

// abs returns the absolute path to the object
func (o *Object) abs() string {
	return path.Join(o.Dir, o.Name)
}

// ModTime returns the cached ModTime
func (o *Object) ModTime(ctx context.Context) time.Time {
	_ = o.refresh(ctx)
	return time.Unix(0, o.CacheModTime)
}

// Size returns the cached Size
func (o *Object) Size() int64 {
	_ = o.refresh(context.TODO())
	return o.CacheSize
}

// Storable returns the cached Storable
func (o *Object) Storable() bool {
	_ = o.refresh(context.TODO())
	return o.CacheStorable
}

// refresh will check if the object info is expired and request the info from source if it is
// all these conditions must be true to ignore a refresh
// 1. cache ts didn't expire yet
// 2. is not pending a notification from the wrapped fs
func (o *Object) refresh(ctx context.Context) error {
	isNotified := o.CacheFs.isNotifiedRemote(o.Remote())
	isExpired := time.Now().After(o.CacheTs.Add(time.Duration(o.CacheFs.opt.InfoAge)))
	if !isExpired && !isNotified {
		return nil
	}

	return o.refreshFromSource(ctx, true)
}

// refreshFromSource requests the original FS for the object in case it comes from a cached entry
func (o *Object) refreshFromSource(ctx context.Context, force bool) error {
	o.refreshMutex.Lock()
	defer o.refreshMutex.Unlock()
	var err error
	var liveObject fs.Object

	if o.Object != nil && !force {
		return nil
	}
	if o.isTempFile() {
		liveObject, err = o.ParentFs.NewObject(ctx, o.Remote())
		err = errors.Wrapf(err, "in parent fs %v", o.ParentFs)
	} else {
		liveObject, err = o.CacheFs.Fs.NewObject(ctx, o.Remote())
		err = errors.Wrapf(err, "in cache fs %v", o.CacheFs.Fs)
	}
	if err != nil {
		fs.Errorf(o, "error refreshing object in : %v", err)
		return err
	}
	o.updateData(ctx, liveObject)
	o.persist()

	return nil
}

// SetModTime sets the ModTime of this object
func (o *Object) SetModTime(ctx context.Context, t time.Time) error {
	if err := o.refreshFromSource(ctx, false); err != nil {
		return err
	}

	err := o.Object.SetModTime(ctx, t)
	if err != nil {
		return err
	}

	o.CacheModTime = t.UnixNano()
	o.persist()
	fs.Debugf(o, "updated ModTime: %v", t)

	return nil
}

// Open is used to request a specific part of the file using fs.RangeOption
func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) {
	var err error

	if o.Object == nil {
		err = o.refreshFromSource(ctx, true)
	} else {
		err = o.refresh(ctx)
	}
	if err != nil {
		return nil, err
	}

	cacheReader := NewObjectHandle(ctx, o, o.CacheFs)
	var offset, limit int64 = 0, -1
	for _, option := range options {
		switch x := option.(type) {
		case *fs.SeekOption:
			offset = x.Offset
		case *fs.RangeOption:
			offset, limit = x.Decode(o.Size())
		}
		_, err = cacheReader.Seek(offset, io.SeekStart)
		if err != nil {
			return nil, err
		}
	}

	return readers.NewLimitedReadCloser(cacheReader, limit), nil
}

// Update will change the object data
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
	if err := o.refreshFromSource(ctx, false); err != nil {
		return err
	}
	// pause background uploads if active
	if o.CacheFs.opt.TempWritePath != "" {
		o.CacheFs.backgroundRunner.pause()
		defer o.CacheFs.backgroundRunner.play()
		// don't allow started uploads
		if o.isTempFile() && o.tempFileStartedUpload() {
			return errors.Errorf("%v is currently uploading, can't update", o)
		}
	}
	fs.Debugf(o, "updating object contents with size %v", src.Size())

	// FIXME use reliable upload
	err := o.Object.Update(ctx, in, src, options...)
	if err != nil {
		fs.Errorf(o, "error updating source: %v", err)
		return err
	}

	// deleting cached chunks and info to be replaced with new ones
	_ = o.CacheFs.cache.RemoveObject(o.abs())
	// advertise to ChangeNotify if wrapped doesn't do that
	o.CacheFs.notifyChangeUpstreamIfNeeded(o.Remote(), fs.EntryObject)

	o.CacheModTime = src.ModTime(ctx).UnixNano()
	o.CacheSize = src.Size()
	o.CacheHashes = make(map[hash.Type]string)
	o.CacheTs = time.Now()
	o.persist()

	return nil
}

// Remove deletes the object from both the cache and the source
func (o *Object) Remove(ctx context.Context) error {
	if err := o.refreshFromSource(ctx, false); err != nil {
		return err
	}
	// pause background uploads if active
	if o.CacheFs.opt.TempWritePath != "" {
		o.CacheFs.backgroundRunner.pause()
		defer o.CacheFs.backgroundRunner.play()
		// don't allow started uploads
		if o.isTempFile() && o.tempFileStartedUpload() {
			return errors.Errorf("%v is currently uploading, can't delete", o)
		}
	}
	err := o.Object.Remove(ctx)
	if err != nil {
		return err
	}

	fs.Debugf(o, "removing object")
	_ = o.CacheFs.cache.RemoveObject(o.abs())
	_ = o.CacheFs.cache.removePendingUpload(o.abs())
	parentCd := NewDirectory(o.CacheFs, cleanPath(path.Dir(o.Remote())))
	_ = o.CacheFs.cache.ExpireDir(parentCd)
	// advertise to ChangeNotify if wrapped doesn't do that
	o.CacheFs.notifyChangeUpstreamIfNeeded(parentCd.Remote(), fs.EntryDirectory)

	return nil
}

// Hash requests a hash of the object and stores in the cache
// since it might or might not be called, this is lazy loaded
func (o *Object) Hash(ctx context.Context, ht hash.Type) (string, error) {
	_ = o.refresh(ctx)
	if o.CacheHashes == nil {
		o.CacheHashes = make(map[hash.Type]string)
	}

	cachedHash, found := o.CacheHashes[ht]
	if found {
		return cachedHash, nil
	}
	if err := o.refreshFromSource(ctx, false); err != nil {
		return "", err
	}
	liveHash, err := o.Object.Hash(ctx, ht)
	if err != nil {
		return "", err
	}
	o.CacheHashes[ht] = liveHash

	o.persist()
	fs.Debugf(o, "object hash cached: %v", liveHash)

	return liveHash, nil
}

// persist adds this object to the persistent cache
func (o *Object) persist() *Object {
	err := o.CacheFs.cache.AddObject(o)
	if err != nil {
		fs.Errorf(o, "failed to cache object: %v", err)
	}
	return o
}

func (o *Object) isTempFile() bool {
	_, err := o.CacheFs.cache.SearchPendingUpload(o.abs())
	if err != nil {
		o.CacheType = objectInCache
		return false
	}

	o.CacheType = objectPendingUpload
	return true
}

func (o *Object) tempFileStartedUpload() bool {
	started, err := o.CacheFs.cache.SearchPendingUpload(o.abs())
	if err != nil {
		return false
	}
	return started
}

// UnWrap returns the Object that this Object is wrapping or
// nil if it isn't wrapping anything
func (o *Object) UnWrap() fs.Object {
	return o.Object
}

var (
	_ fs.Object          = (*Object)(nil)
	_ fs.ObjectUnWrapper = (*Object)(nil)
)
<commit_msg>cache: fix `fatal error: concurrent map writes` - fixes #2378<commit_after>// +build !plan9

package cache

import (
	"context"
	"io"
	"path"
	"sync"
	"time"

	"github.com/pkg/errors"
	"github.com/rclone/rclone/fs"
	"github.com/rclone/rclone/fs/hash"
	"github.com/rclone/rclone/lib/readers"
)

const (
	objectInCache       = "Object"
	objectPendingUpload = "TempObject"
)

// Object is a generic file like object that stores basic information about it
type Object struct {
	fs.Object `json:"-"`

	ParentFs      fs.Fs     `json:"-"`        // parent fs
	CacheFs       *Fs       `json:"-"`        // cache fs
	Name          string    `json:"name"`     // name of the directory
	Dir           string    `json:"dir"`      // abs path of the object
	CacheModTime  int64     `json:"modTime"`  // modification or creation time - IsZero for unknown
	CacheSize     int64     `json:"size"`     // size of directory and contents or -1 if unknown
	CacheStorable bool      `json:"storable"` // says whether this object can be stored
	CacheType     string    `json:"cacheType"`
	CacheTs       time.Time `json:"cacheTs"`
	cacheHashesMu sync.Mutex
	CacheHashes   map[hash.Type]string // all supported hashes cached

	refreshMutex sync.Mutex
}

// NewObject builds one from a generic fs.Object
func NewObject(f *Fs, remote string) *Object {
	fullRemote := path.Join(f.Root(), remote)
	dir, name := path.Split(fullRemote)

	cacheType := objectInCache
	parentFs := f.UnWrap()
	if f.opt.TempWritePath != "" {
		_, err := f.cache.SearchPendingUpload(fullRemote)
		if err == nil { // queued for upload
			cacheType = objectPendingUpload
			parentFs = f.tempFs
			fs.Debugf(fullRemote, "pending upload found")
		}
	}

	co := &Object{
		ParentFs:      parentFs,
		CacheFs:       f,
		Name:          cleanPath(name),
		Dir:           cleanPath(dir),
		CacheModTime:  time.Now().UnixNano(),
		CacheSize:     0,
		CacheStorable: false,
		CacheType:     cacheType,
		CacheTs:       time.Now(),
	}
	return co
}

// ObjectFromOriginal builds one from a generic fs.Object
func ObjectFromOriginal(ctx context.Context, f *Fs, o fs.Object) *Object {
	var co *Object
	fullRemote := cleanPath(path.Join(f.Root(), o.Remote()))
	dir, name := path.Split(fullRemote)

	cacheType := objectInCache
	parentFs := f.UnWrap()
	if f.opt.TempWritePath != "" {
		_, err := f.cache.SearchPendingUpload(fullRemote)
		if err == nil { // queued for upload
			cacheType = objectPendingUpload
			parentFs = f.tempFs
			fs.Debugf(fullRemote, "pending upload found")
		}
	}

	co = &Object{
		ParentFs:  parentFs,
		CacheFs:   f,
		Name:      cleanPath(name),
		Dir:       cleanPath(dir),
		CacheType: cacheType,
		CacheTs:   time.Now(),
	}
	co.updateData(ctx, o)
	return co
}

func (o *Object) updateData(ctx context.Context, source fs.Object) {
	o.Object = source
	o.CacheModTime = source.ModTime(ctx).UnixNano()
	o.CacheSize = source.Size()
	o.CacheStorable = source.Storable()
	o.CacheTs = time.Now()
	o.cacheHashesMu.Lock()
	o.CacheHashes = make(map[hash.Type]string)
	o.cacheHashesMu.Unlock()
}

// Fs returns its FS info
func (o *Object) Fs() fs.Info {
	return o.CacheFs
}

// String returns a human friendly name for this object
func (o *Object) String() string {
	if o == nil {
		return "<nil>"
	}
	return o.Remote()
}

// Remote returns the remote path
func (o *Object) Remote() string {
	p := path.Join(o.Dir, o.Name)
	return o.CacheFs.cleanRootFromPath(p)
}

// abs returns the absolute path to the object
func (o *Object) abs() string {
	return path.Join(o.Dir, o.Name)
}

// ModTime returns the cached ModTime
func (o *Object) ModTime(ctx context.Context) time.Time {
	_ = o.refresh(ctx)
	return time.Unix(0, o.CacheModTime)
}

// Size returns the cached Size
func (o *Object) Size() int64 {
	_ = o.refresh(context.TODO())
	return o.CacheSize
}

// Storable returns the cached Storable
func (o *Object) Storable() bool {
	_ = o.refresh(context.TODO())
	return o.CacheStorable
}

// refresh will check if the object info is expired and request the info from source if it is
// all these conditions must be true to ignore a refresh
// 1. cache ts didn't expire yet
// 2. is not pending a notification from the wrapped fs
func (o *Object) refresh(ctx context.Context) error {
	isNotified := o.CacheFs.isNotifiedRemote(o.Remote())
	isExpired := time.Now().After(o.CacheTs.Add(time.Duration(o.CacheFs.opt.InfoAge)))
	if !isExpired && !isNotified {
		return nil
	}

	return o.refreshFromSource(ctx, true)
}

// refreshFromSource requests the original FS for the object in case it comes from a cached entry
func (o *Object) refreshFromSource(ctx context.Context, force bool) error {
	o.refreshMutex.Lock()
	defer o.refreshMutex.Unlock()
	var err error
	var liveObject fs.Object

	if o.Object != nil && !force {
		return nil
	}
	if o.isTempFile() {
		liveObject, err = o.ParentFs.NewObject(ctx, o.Remote())
		err = errors.Wrapf(err, "in parent fs %v", o.ParentFs)
	} else {
		liveObject, err = o.CacheFs.Fs.NewObject(ctx, o.Remote())
		err = errors.Wrapf(err, "in cache fs %v", o.CacheFs.Fs)
	}
	if err != nil {
		fs.Errorf(o, "error refreshing object in : %v", err)
		return err
	}
	o.updateData(ctx, liveObject)
	o.persist()

	return nil
}

// SetModTime sets the ModTime of this object
func (o *Object) SetModTime(ctx context.Context, t time.Time) error {
	if err := o.refreshFromSource(ctx, false); err != nil {
		return err
	}

	err := o.Object.SetModTime(ctx, t)
	if err != nil {
		return err
	}

	o.CacheModTime = t.UnixNano()
	o.persist()
	fs.Debugf(o, "updated ModTime: %v", t)

	return nil
}

// Open is used to request a specific part of the file using fs.RangeOption
func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) {
	var err error

	if o.Object == nil {
		err = o.refreshFromSource(ctx, true)
	} else {
		err = o.refresh(ctx)
	}
	if err != nil {
		return nil, err
	}

	cacheReader := NewObjectHandle(ctx, o, o.CacheFs)
	var offset, limit int64 = 0, -1
	for _, option := range options {
		switch x := option.(type) {
		case *fs.SeekOption:
			offset = x.Offset
		case *fs.RangeOption:
			offset, limit = x.Decode(o.Size())
		}
		_, err = cacheReader.Seek(offset, io.SeekStart)
		if err != nil {
			return nil, err
		}
	}

	return readers.NewLimitedReadCloser(cacheReader, limit), nil
}

// Update will change the object data
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
	if err := o.refreshFromSource(ctx, false); err != nil {
		return err
	}
	// pause background uploads if active
	if o.CacheFs.opt.TempWritePath != "" {
		o.CacheFs.backgroundRunner.pause()
		defer o.CacheFs.backgroundRunner.play()
		// don't allow started uploads
		if o.isTempFile() && o.tempFileStartedUpload() {
			return errors.Errorf("%v is currently uploading, can't update", o)
		}
	}
	fs.Debugf(o, "updating object contents with size %v", src.Size())

	// FIXME use reliable upload
	err := o.Object.Update(ctx, in, src, options...)
	if err != nil {
		fs.Errorf(o, "error updating source: %v", err)
		return err
	}

	// deleting cached chunks and info to be replaced with new ones
	_ = o.CacheFs.cache.RemoveObject(o.abs())
	// advertise to ChangeNotify if wrapped doesn't do that
	o.CacheFs.notifyChangeUpstreamIfNeeded(o.Remote(), fs.EntryObject)

	o.CacheModTime = src.ModTime(ctx).UnixNano()
	o.CacheSize = src.Size()
	o.cacheHashesMu.Lock()
	o.CacheHashes = make(map[hash.Type]string)
	o.cacheHashesMu.Unlock()
	o.CacheTs = time.Now()
	o.persist()

	return nil
}

// Remove deletes the object from both the cache and the source
func (o *Object) Remove(ctx context.Context) error {
	if err := o.refreshFromSource(ctx, false); err != nil {
		return err
	}
	// pause background uploads if active
	if o.CacheFs.opt.TempWritePath != "" {
		o.CacheFs.backgroundRunner.pause()
		defer o.CacheFs.backgroundRunner.play()
		// don't allow started uploads
		if o.isTempFile() && o.tempFileStartedUpload() {
			return errors.Errorf("%v is currently uploading, can't delete", o)
		}
	}
	err := o.Object.Remove(ctx)
	if err != nil {
		return err
	}

	fs.Debugf(o, "removing object")
	_ = o.CacheFs.cache.RemoveObject(o.abs())
	_ = o.CacheFs.cache.removePendingUpload(o.abs())
	parentCd := NewDirectory(o.CacheFs, cleanPath(path.Dir(o.Remote())))
	_ = o.CacheFs.cache.ExpireDir(parentCd)
	// advertise to ChangeNotify if wrapped doesn't do that
	o.CacheFs.notifyChangeUpstreamIfNeeded(parentCd.Remote(), fs.EntryDirectory)

	return nil
}

// Hash requests a hash of the object and stores in the cache
// since it might or might not be called, this is lazy loaded
func (o *Object) Hash(ctx context.Context, ht hash.Type) (string, error) {
	_ = o.refresh(ctx)
	o.cacheHashesMu.Lock()
	if o.CacheHashes == nil {
		o.CacheHashes = make(map[hash.Type]string)
	}
	cachedHash, found := o.CacheHashes[ht]
	o.cacheHashesMu.Unlock()
	if found {
		return cachedHash, nil
	}
	if err := o.refreshFromSource(ctx, false); err != nil {
		return "", err
	}
	liveHash, err := o.Object.Hash(ctx, ht)
	if err != nil {
		return "", err
	}
	o.cacheHashesMu.Lock()
	o.CacheHashes[ht] = liveHash
	o.cacheHashesMu.Unlock()

	o.persist()
	fs.Debugf(o, "object hash cached: %v", liveHash)

	return liveHash, nil
}

// persist adds this object to the persistent cache
func (o *Object) persist() *Object {
	err := o.CacheFs.cache.AddObject(o)
	if err != nil {
		fs.Errorf(o, "failed to cache object: %v", err)
	}
	return o
}

func (o *Object) isTempFile() bool {
	_, err := o.CacheFs.cache.SearchPendingUpload(o.abs())
	if err != nil {
		o.CacheType = objectInCache
		return false
	}

	o.CacheType = objectPendingUpload
	return true
}

func (o *Object) tempFileStartedUpload() bool {
	started, err := o.CacheFs.cache.SearchPendingUpload(o.abs())
	if err != nil {
		return false
	}
	return started
}

// UnWrap returns the Object that this Object is wrapping or
// nil if it isn't wrapping anything
func (o *Object) UnWrap() fs.Object {
	return o.Object
}

var (
	_ fs.Object          = (*Object)(nil)
	_ fs.ObjectUnWrapper = (*Object)(nil)
)
<|endoftext|>