<commit_before>/* Coordination between a collection of chunks. */
package db

import (
	"errors"
	"fmt"
	"github.com/HouzuoGuo/tiedot/chunk"
	"github.com/HouzuoGuo/tiedot/chunkfile"
	"github.com/HouzuoGuo/tiedot/tdlog"
	"math/rand"
	"os"
	"path"
	"path/filepath"
	"strconv"
	"sync"
)

type Col struct {
	BaseDir       string // Collection dir path
	Chunks        []*chunk.ChunkCol
	ChunkMutexes  []*sync.RWMutex // Synchronize access to chunks
	NewChunkMutex sync.Mutex      // Synchronize creation of new chunk
	NumChunks     uint64          // Total number of chunks
}

// Open a collection (made of chunks).
func OpenCol(baseDir string) (col *Col, err error) {
	// Create the directory if it does not yet exist
	if err = os.MkdirAll(baseDir, 0700); err != nil {
		return
	}
	col = &Col{BaseDir: baseDir, NewChunkMutex: sync.Mutex{},
		Chunks: make([]*chunk.ChunkCol, 0), ChunkMutexes: make([]*sync.RWMutex, 0)}
	// Walk the collection directory and look for how many chunks there are
	maxChunk := 0
	walker := func(currPath string, info os.FileInfo, err2 error) error {
		if err2 != nil {
			// log and skip the error
			tdlog.Error(err)
			return nil
		}
		if info.IsDir() {
			// According to the directory name - chunk number, look for how many chunks there are
			if chunkNum, err := strconv.Atoi(info.Name()); err == nil {
				if chunkNum > maxChunk {
					maxChunk = chunkNum
				}
			}
			return nil
		}
		return nil
	}
	if err = filepath.Walk(baseDir, walker); err != nil {
		return
	}
	col.NumChunks = uint64(maxChunk) + 1
	// Open all chunks - there is at least one chunk
	tdlog.Printf("Opening %d chunks in collection %s", maxChunk+1, baseDir)
	for i := 0; i <= maxChunk; i++ {
		var oneChunk *chunk.ChunkCol
		oneChunk, err = chunk.OpenChunk(uint64(i), path.Join(baseDir, strconv.Itoa(i)))
		col.Chunks = append(col.Chunks, oneChunk)
		col.ChunkMutexes = append(col.ChunkMutexes, new(sync.RWMutex))
	}
	return
}

// Create a new chunk.
func (col *Col) CreateNewChunk() {
	col.NewChunkMutex.Lock()
	tdlog.Printf("Going to create a new chunk (number %d) in collection %s", col.NumChunks, col.BaseDir)
	newChunk, err := chunk.OpenChunk(col.NumChunks, path.Join(col.BaseDir, strconv.Itoa(int(col.NumChunks))))
	if err != nil {
		col.NewChunkMutex.Unlock()
		return
	}
	// Make indexes
	for _, path := range col.Chunks[0].HTPaths {
		newChunk.Index(path)
	}
	// Put the new chunk into col structures
	col.Chunks = append(col.Chunks, newChunk)
	col.ChunkMutexes = append(col.ChunkMutexes, new(sync.RWMutex))
	col.NumChunks += 1
	col.NewChunkMutex.Unlock()
}

// Insert a new document, return new document's ID.
func (col *Col) Insert(doc interface{}) (id uint64, err error) {
	// Try to insert the doc into a random chunk
	randChunkNum := uint64(rand.Int63n(int64(col.NumChunks)))
	randChunk := col.Chunks[randChunkNum]
	randChunkMutex := col.ChunkMutexes[randChunkNum]
	randChunkMutex.Lock()
	id, outOfSpace, err := randChunk.Insert(doc)
	if !outOfSpace {
		randChunkMutex.Unlock()
		return
	}
	randChunkMutex.Unlock()
	// If the random chunk was full, try again with the last chunk
	lastChunk := col.Chunks[col.NumChunks-1]
	lastChunkMutex := col.ChunkMutexes[col.NumChunks-1]
	lastChunkMutex.Lock()
	id, outOfSpace, err = lastChunk.Insert(doc)
	if !outOfSpace {
		lastChunkMutex.Unlock()
		return
	}
	lastChunkMutex.Unlock()
	// If the last chunk is full, make a new chunk
	col.CreateNewChunk()
	// Now there is a new chunk, let us try again
	return col.Insert(doc)
}

// Read document at the given ID.
func (col *Col) Read(id uint64, doc interface{}) (err error) {
	chunkNum := id / chunkfile.COL_FILE_SIZE
	if chunkNum >= col.NumChunks {
		return errors.New(fmt.Sprintf("Document %d does not exist in %s - out of bound chunk", id, col.BaseDir))
	}
	chunk := col.Chunks[chunkNum]
	chunkMutex := col.ChunkMutexes[chunkNum]
	chunkMutex.RLock()
	err = chunk.Read(id, doc)
	chunkMutex.RUnlock()
	return
}

// Update a document, return its new ID.
func (col *Col) Update(id uint64, doc interface{}) (newID uint64, err error) {
	chunkNum := id / chunkfile.COL_FILE_SIZE
	if chunkNum >= col.NumChunks {
		err = errors.New(fmt.Sprintf("Document %d does not exist in %s - out of bound chunk", id, col.BaseDir))
		return
	}
	chunk := col.Chunks[chunkNum]
	chunkMutex := col.ChunkMutexes[chunkNum]
	chunkMutex.Lock()
	newID, outOfSpace, err := chunk.Update(id, doc)
	chunkMutex.Unlock()
	if !outOfSpace {
		newID += chunkNum * chunkfile.COL_FILE_SIZE
		return
	}
	tdlog.Printf("Out of space")
	// The chunk does not have enough space for the updated document, let us put it somewhere else
	// The document has already been removed from its original chunk
	return col.Insert(doc)
}

// Delete a document by ID.
func (col *Col) Delete(id uint64) {
	chunkNum := id / chunkfile.COL_FILE_SIZE
	if chunkNum >= col.NumChunks {
		return
	}
	chunk := col.Chunks[chunkNum]
	chunkMutex := col.ChunkMutexes[chunkNum]
	chunkMutex.Lock()
	chunk.Delete(id)
	chunkMutex.Unlock()
}

// Create an index on the path.
func (col *Col) Index(path []string) (err error) {
	// Do not allow new chunk creation for now
	col.NewChunkMutex.Lock()
	defer col.NewChunkMutex.Unlock()
	for _, chunk := range col.Chunks {
		if err = chunk.Index(path); err != nil {
			return
		}
	}
	return
}

// Execute the function (only reads chunk data) for all chunks in parallel.
func (col *Col) ParForeachChunkR(fun func(*chunk.ChunkCol)) {
	numChunks := col.NumChunks
	wg := sync.WaitGroup{}
	for i := uint64(0); i < numChunks; i++ {
		wg.Add(1)
		go func(i uint64) {
			col.ChunkMutexes[i].RLock()
			fun(col.Chunks[i])
			col.ChunkMutexes[i].RUnlock()
			wg.Done()
		}(i)
	}
	wg.Wait()
	return
}

// Execute the function (presumably modifies chunk data) for all chunks in parallel.
func (col *Col) ParForeachChunk(fun func(*chunk.ChunkCol)) {
	numChunks := col.NumChunks
	wg := sync.WaitGroup{}
	for i := uint64(0); i < numChunks; i++ {
		wg.Add(1)
		go func(i uint64) {
			col.ChunkMutexes[i].Lock()
			fun(col.Chunks[i])
			col.ChunkMutexes[i].Unlock()
			wg.Done()
		}(i)
	}
	wg.Wait()
	return
}

// Remove an index.
func (col *Col) Unindex(path []string) (err error) {
	// Do not allow new chunk creation for now
	col.NewChunkMutex.Lock()
	defer col.NewChunkMutex.Unlock()
	for _, chunk := range col.Chunks {
		if err = chunk.Unindex(path); err != nil {
			return
		}
	}
	return
}

// Insert a new document, and assign it a UID.
func (col *Col) InsertWithUID(doc interface{}) (id uint64, uid string, err error) {
	// Try to insert the doc into a random chunk
	randChunkNum := uint64(rand.Int63n(int64(col.NumChunks)))
	randChunk := col.Chunks[randChunkNum]
	randChunkMutexes := col.ChunkMutexes[randChunkNum]
	randChunkMutexes.Lock()
	id, uid, outOfSpace, err := randChunk.InsertWithUID(doc)
	if !outOfSpace {
		randChunkMutexes.Unlock()
		return
	}
	randChunkMutexes.Unlock()
	// If the random chunk was full, try again with the last chunk
	lastChunk := col.Chunks[col.NumChunks-1]
	lastChunkMutexes := col.ChunkMutexes[col.NumChunks-1]
	lastChunkMutexes.Lock()
	id, uid, outOfSpace, err = lastChunk.InsertWithUID(doc)
	if !outOfSpace {
		lastChunkMutexes.Unlock()
		return
	}
	lastChunkMutexes.Unlock()
	// If the last chunk is full, make a new chunk
	col.CreateNewChunk()
	// Now there is a new chunk, let us try again
	return col.InsertWithUID(doc)
}

// Retrieve documentby UID, return its ID.
func (col *Col) ReadByUID(uid string, doc interface{}) (id uint64, err error) {
	var found bool
	col.ParForeachChunkR(func(chunk *chunk.ChunkCol) {
		selfID, selfErr := chunk.ReadByUID(uid, &doc)
		if selfErr == nil {
			found = true
			id = selfID
		}
	})
	if !found {
		return 0, errors.New(fmt.Sprintf("Document %s does not exist in %s", uid, col.BaseDir))
	}
	return
}

// Identify a document using UID and update it, return its new ID.
func (col *Col) UpdateByUID(uid string, doc interface{}) (newID uint64, err error) {
	var found bool
	var outOfSpace bool
	// Attempt to "Update by UID" in each chunk
	col.ParForeachChunk(func(chunk *chunk.ChunkCol) {
		selfNewID, selfOutOfSpace, selfErr := chunk.UpdateByUID(uid, doc)
		if selfErr == nil {
			// Here, this chunk has the document!
			found = true
			outOfSpace = selfOutOfSpace
			newID = selfNewID
		}
	})
	if !found {
		return 0, errors.New(fmt.Sprintf("Document %s does not exist in %s", uid, col.BaseDir))
	}
	// If the original chunk ran out of space, the document is no longer there (deleted by col.Update)
	// So let us put back the updated document
	if outOfSpace {
		return col.Insert(doc)
	}
	return
}

// Give a document (identified by ID) a new UID.
func (col *Col) ReassignUID(id uint64) (newID uint64, newUID string, err error) {
	chunkNum := id / chunkfile.COL_FILE_SIZE
	if chunkNum >= col.NumChunks {
		err = errors.New(fmt.Sprintf("Document %d does not exist in %s - out of bound chunk", id, col.BaseDir))
		return
	}
	chunkDocID := id % chunkfile.COL_FILE_SIZE
	chunk := col.Chunks[chunkNum]
	chunkMutex := col.ChunkMutexes[chunkNum]
	chunkMutex.Lock()
	// Attempt to reassign UID in the original chunk
	newID, newUID, newDoc, outOfSpace, err := chunk.ReassignUID(chunkDocID)
	chunkMutex.Unlock()
	// If the original chunk is full, the document no longer exists in the chunk (deleted by col.Update)
	// So let us put back the new document (with UID)
	if outOfSpace {
		newID, err = col.Insert(newDoc)
	}

	return
}

// Delete a document by UID.
func (col *Col) DeleteByUID(uid string) {
	col.ParForeachChunk(func(chunk *chunk.ChunkCol) {
		chunk.DeleteByUID(uid)
	})
}

/* In parallel, deserialize all documents and invoke the function on each document (Collection Scan).
The function must not write to this collection. */
func (col *Col) ForAll(fun func(id uint64, doc interface{}) bool) {
	col.ParForeachChunkR(func(chunk *chunk.ChunkCol) {
		chunk.ForAll(fun)
	})
}

/* Sequentially, deserialize all documents into the template (pointer to struct) and invoke the function on each document (Collection Scan).
The function must not write to this collection. */
func (col *Col) DeserializeAll(template interface{}, fun func(id uint64) bool) {
	numChunks := col.NumChunks
	for i := uint64(0); i < numChunks; i++ {
		chunk := col.Chunks[i]
		chunkMutex := col.ChunkMutexes[i]
		chunkMutex.RLock()
		chunk.DeserializeAll(template, fun)
		chunkMutex.RUnlock()
	}
}

// Compact the collection and automatically repair any data/index damage.
func (col *Col) Scrub() (recovered uint64) {
	// Do not allow new chunk creation for now
	col.NewChunkMutex.Lock()
	defer col.NewChunkMutex.Unlock()
	for _, chunk := range col.Chunks {
		recovered += chunk.Scrub()
	}
	return
}

// Flush collection data and index files.
func (col *Col) Flush() error {
	// Do not allow new chunk creation for now
	col.NewChunkMutex.Lock()
	defer col.NewChunkMutex.Unlock()
	for _, chunk := range col.Chunks {
		if err := chunk.Flush(); err != nil {
			return err
		}
	}
	return nil
}

// Close the collection.
func (col *Col) Close() {
	for _, chunk := range col.Chunks {
		chunk.Close()
	}
}
<commit_msg>better error handling in chunk creation<commit_after>/* Coordination between a collection of chunks. */
package db

import (
	"errors"
	"fmt"
	"github.com/HouzuoGuo/tiedot/chunk"
	"github.com/HouzuoGuo/tiedot/chunkfile"
	"github.com/HouzuoGuo/tiedot/tdlog"
	"math/rand"
	"os"
	"path"
	"path/filepath"
	"strconv"
	"sync"
)

type Col struct {
	BaseDir       string // Collection dir path
	Chunks        []*chunk.ChunkCol
	ChunkMutexes  []*sync.RWMutex // Synchronize access to chunks
	NewChunkMutex sync.Mutex      // Synchronize creation of new chunk
	NumChunks     uint64          // Total number of chunks
}

// Open a collection (made of chunks).
func OpenCol(baseDir string) (col *Col, err error) {
	// Create the directory if it does not yet exist
	if err = os.MkdirAll(baseDir, 0700); err != nil {
		return
	}
	col = &Col{BaseDir: baseDir, NewChunkMutex: sync.Mutex{},
		Chunks: make([]*chunk.ChunkCol, 0), ChunkMutexes: make([]*sync.RWMutex, 0)}
	// Walk the collection directory and look for how many chunks there are
	maxChunk := 0
	walker := func(currPath string, info os.FileInfo, err2 error) error {
		if err2 != nil {
			// log and skip the error
			tdlog.Error(err)
			return nil
		}
		if info.IsDir() {
			// According to the directory name - chunk number, look for how many chunks there are
			if chunkNum, err := strconv.Atoi(info.Name()); err == nil {
				if chunkNum > maxChunk {
					maxChunk = chunkNum
				}
			}
			return nil
		}
		return nil
	}
	if err = filepath.Walk(baseDir, walker); err != nil {
		return
	}
	col.NumChunks = uint64(maxChunk) + 1
	// Open all chunks - there is at least one chunk
	tdlog.Printf("Opening %d chunks in collection %s", maxChunk+1, baseDir)
	for i := 0; i <= maxChunk; i++ {
		var oneChunk *chunk.ChunkCol
		oneChunk, err = chunk.OpenChunk(uint64(i), path.Join(baseDir, strconv.Itoa(i)))
		col.Chunks = append(col.Chunks, oneChunk)
		col.ChunkMutexes = append(col.ChunkMutexes, new(sync.RWMutex))
	}
	return
}

// Create a new chunk.
func (col *Col) CreateNewChunk() {
	col.NewChunkMutex.Lock()
	defer col.NewChunkMutex.Unlock()
	tdlog.Printf("Going to create a new chunk (number %d) in collection %s", col.NumChunks, col.BaseDir)
	newChunk, err := chunk.OpenChunk(col.NumChunks, path.Join(col.BaseDir, strconv.Itoa(int(col.NumChunks))))
	if err != nil {
		col.NewChunkMutex.Unlock()
		return
	}
	// Make indexes
	for _, path := range col.Chunks[0].HTPaths {
		if path[0] != chunk.UID_PATH {
			if err := newChunk.Index(path); err != nil {
				tdlog.Panicf("Failed to create index %s, error: %v", path, err)
			}
		}
	}
	// Put the new chunk into col structures
	col.Chunks = append(col.Chunks, newChunk)
	col.ChunkMutexes = append(col.ChunkMutexes, new(sync.RWMutex))
	col.NumChunks += 1
}

// Insert a new document, return new document's ID.
func (col *Col) Insert(doc interface{}) (id uint64, err error) {
	// Try to insert the doc into a random chunk
	randChunkNum := uint64(rand.Int63n(int64(col.NumChunks)))
	randChunk := col.Chunks[randChunkNum]
	randChunkMutex := col.ChunkMutexes[randChunkNum]
	randChunkMutex.Lock()
	id, outOfSpace, err := randChunk.Insert(doc)
	if !outOfSpace {
		randChunkMutex.Unlock()
		return
	}
	randChunkMutex.Unlock()
	// If the random chunk was full, try again with the last chunk
	lastChunk := col.Chunks[col.NumChunks-1]
	lastChunkMutex := col.ChunkMutexes[col.NumChunks-1]
	lastChunkMutex.Lock()
	id, outOfSpace, err = lastChunk.Insert(doc)
	if !outOfSpace {
		lastChunkMutex.Unlock()
		return
	}
	lastChunkMutex.Unlock()
	// If the last chunk is full, make a new chunk
	col.CreateNewChunk()
	// Now there is a new chunk, let us try again
	return col.Insert(doc)
}

// Read document at the given ID.
func (col *Col) Read(id uint64, doc interface{}) (err error) {
	chunkNum := id / chunkfile.COL_FILE_SIZE
	if chunkNum >= col.NumChunks {
		return errors.New(fmt.Sprintf("Document %d does not exist in %s - out of bound chunk", id, col.BaseDir))
	}
	chunk := col.Chunks[chunkNum]
	chunkMutex := col.ChunkMutexes[chunkNum]
	chunkMutex.RLock()
	err = chunk.Read(id, doc)
	chunkMutex.RUnlock()
	return
}

// Update a document, return its new ID.
func (col *Col) Update(id uint64, doc interface{}) (newID uint64, err error) {
	chunkNum := id / chunkfile.COL_FILE_SIZE
	if chunkNum >= col.NumChunks {
		err = errors.New(fmt.Sprintf("Document %d does not exist in %s - out of bound chunk", id, col.BaseDir))
		return
	}
	chunk := col.Chunks[chunkNum]
	chunkMutex := col.ChunkMutexes[chunkNum]
	chunkMutex.Lock()
	newID, outOfSpace, err := chunk.Update(id, doc)
	chunkMutex.Unlock()
	if !outOfSpace {
		newID += chunkNum * chunkfile.COL_FILE_SIZE
		return
	}
	tdlog.Printf("Out of space")
	// The chunk does not have enough space for the updated document, let us put it somewhere else
	// The document has already been removed from its original chunk
	return col.Insert(doc)
}

// Delete a document by ID.
func (col *Col) Delete(id uint64) {
	chunkNum := id / chunkfile.COL_FILE_SIZE
	if chunkNum >= col.NumChunks {
		return
	}
	chunk := col.Chunks[chunkNum]
	chunkMutex := col.ChunkMutexes[chunkNum]
	chunkMutex.Lock()
	chunk.Delete(id)
	chunkMutex.Unlock()
}

// Create an index on the path.
func (col *Col) Index(path []string) (err error) {
	// Do not allow new chunk creation for now
	col.NewChunkMutex.Lock()
	defer col.NewChunkMutex.Unlock()
	for _, chunk := range col.Chunks {
		if err = chunk.Index(path); err != nil {
			return
		}
	}
	return
}

// Execute the function (only reads chunk data) for all chunks in parallel.
func (col *Col) ParForeachChunkR(fun func(*chunk.ChunkCol)) {
	numChunks := col.NumChunks
	wg := sync.WaitGroup{}
	for i := uint64(0); i < numChunks; i++ {
		wg.Add(1)
		go func(i uint64) {
			col.ChunkMutexes[i].RLock()
			fun(col.Chunks[i])
			col.ChunkMutexes[i].RUnlock()
			wg.Done()
		}(i)
	}
	wg.Wait()
	return
}

// Execute the function (presumably modifies chunk data) for all chunks in parallel.
func (col *Col) ParForeachChunk(fun func(*chunk.ChunkCol)) {
	numChunks := col.NumChunks
	wg := sync.WaitGroup{}
	for i := uint64(0); i < numChunks; i++ {
		wg.Add(1)
		go func(i uint64) {
			col.ChunkMutexes[i].Lock()
			fun(col.Chunks[i])
			col.ChunkMutexes[i].Unlock()
			wg.Done()
		}(i)
	}
	wg.Wait()
	return
}

// Remove an index.
func (col *Col) Unindex(path []string) (err error) {
	// Do not allow new chunk creation for now
	col.NewChunkMutex.Lock()
	defer col.NewChunkMutex.Unlock()
	for _, chunk := range col.Chunks {
		if err = chunk.Unindex(path); err != nil {
			return
		}
	}
	return
}

// Insert a new document, and assign it a UID.
func (col *Col) InsertWithUID(doc interface{}) (id uint64, uid string, err error) {
	// Try to insert the doc into a random chunk
	randChunkNum := uint64(rand.Int63n(int64(col.NumChunks)))
	randChunk := col.Chunks[randChunkNum]
	randChunkMutexes := col.ChunkMutexes[randChunkNum]
	randChunkMutexes.Lock()
	id, uid, outOfSpace, err := randChunk.InsertWithUID(doc)
	if !outOfSpace {
		randChunkMutexes.Unlock()
		return
	}
	randChunkMutexes.Unlock()
	// If the random chunk was full, try again with the last chunk
	lastChunk := col.Chunks[col.NumChunks-1]
	lastChunkMutexes := col.ChunkMutexes[col.NumChunks-1]
	lastChunkMutexes.Lock()
	id, uid, outOfSpace, err = lastChunk.InsertWithUID(doc)
	if !outOfSpace {
		lastChunkMutexes.Unlock()
		return
	}
	lastChunkMutexes.Unlock()
	// If the last chunk is full, make a new chunk
	col.CreateNewChunk()
	// Now there is a new chunk, let us try again
	return col.InsertWithUID(doc)
}

// Retrieve documentby UID, return its ID.
func (col *Col) ReadByUID(uid string, doc interface{}) (id uint64, err error) {
	var found bool
	col.ParForeachChunkR(func(chunk *chunk.ChunkCol) {
		selfID, selfErr := chunk.ReadByUID(uid, &doc)
		if selfErr == nil {
			found = true
			id = selfID
		}
	})
	if !found {
		return 0, errors.New(fmt.Sprintf("Document %s does not exist in %s", uid, col.BaseDir))
	}
	return
}

// Identify a document using UID and update it, return its new ID.
func (col *Col) UpdateByUID(uid string, doc interface{}) (newID uint64, err error) {
	var found bool
	var outOfSpace bool
	// Attempt to "Update by UID" in each chunk
	col.ParForeachChunk(func(chunk *chunk.ChunkCol) {
		selfNewID, selfOutOfSpace, selfErr := chunk.UpdateByUID(uid, doc)
		if selfErr == nil {
			// Here, this chunk has the document!
			found = true
			outOfSpace = selfOutOfSpace
			newID = selfNewID
		}
	})
	if !found {
		return 0, errors.New(fmt.Sprintf("Document %s does not exist in %s", uid, col.BaseDir))
	}
	// If the original chunk ran out of space, the document is no longer there (deleted by col.Update)
	// So let us put back the updated document
	if outOfSpace {
		return col.Insert(doc)
	}
	return
}

// Give a document (identified by ID) a new UID.
func (col *Col) ReassignUID(id uint64) (newID uint64, newUID string, err error) {
	chunkNum := id / chunkfile.COL_FILE_SIZE
	if chunkNum >= col.NumChunks {
		err = errors.New(fmt.Sprintf("Document %d does not exist in %s - out of bound chunk", id, col.BaseDir))
		return
	}
	chunkDocID := id % chunkfile.COL_FILE_SIZE
	chunk := col.Chunks[chunkNum]
	chunkMutex := col.ChunkMutexes[chunkNum]
	chunkMutex.Lock()
	// Attempt to reassign UID in the original chunk
	newID, newUID, newDoc, outOfSpace, err := chunk.ReassignUID(chunkDocID)
	chunkMutex.Unlock()
	// If the original chunk is full, the document no longer exists in the chunk (deleted by col.Update)
	// So let us put back the new document (with UID)
	if outOfSpace {
		newID, err = col.Insert(newDoc)
	}

	return
}

// Delete a document by UID.
func (col *Col) DeleteByUID(uid string) {
	col.ParForeachChunk(func(chunk *chunk.ChunkCol) {
		chunk.DeleteByUID(uid)
	})
}

/* In parallel, deserialize all documents and invoke the function on each document (Collection Scan).
The function must not write to this collection. */
func (col *Col) ForAll(fun func(id uint64, doc interface{}) bool) {
	col.ParForeachChunkR(func(chunk *chunk.ChunkCol) {
		chunk.ForAll(fun)
	})
}

/* Sequentially, deserialize all documents into the template (pointer to struct) and invoke the function on each document (Collection Scan).
The function must not write to this collection. */
func (col *Col) DeserializeAll(template interface{}, fun func(id uint64) bool) {
	numChunks := col.NumChunks
	for i := uint64(0); i < numChunks; i++ {
		chunk := col.Chunks[i]
		chunkMutex := col.ChunkMutexes[i]
		chunkMutex.RLock()
		chunk.DeserializeAll(template, fun)
		chunkMutex.RUnlock()
	}
}

// Compact the collection and automatically repair any data/index damage.
func (col *Col) Scrub() (recovered uint64) {
	// Do not allow new chunk creation for now
	col.NewChunkMutex.Lock()
	defer col.NewChunkMutex.Unlock()
	for _, chunk := range col.Chunks {
		recovered += chunk.Scrub()
	}
	return
}

// Flush collection data and index files.
func (col *Col) Flush() error {
	// Do not allow new chunk creation for now
	col.NewChunkMutex.Lock()
	defer col.NewChunkMutex.Unlock()
	for _, chunk := range col.Chunks {
		if err := chunk.Flush(); err != nil {
			return err
		}
	}
	return nil
}

// Close the collection.
func (col *Col) Close() {
	for _, chunk := range col.Chunks {
		chunk.Close()
	}
}
<|endoftext|>


<commit_before>package gocsv

import (
	"encoding/csv"
	"errors"
	"fmt"
	"io"
	"reflect"
)

// Decoder .
type Decoder interface {
	getCSVRows() ([][]string, error)
}

// SimpleDecoder .
type SimpleDecoder interface {
	getCSVRow() ([]string, error)
}

type decoder struct {
	in         io.Reader
	csvDecoder *csvDecoder
}

func newDecoder(in io.Reader) *decoder {
	return &decoder{in: in}
}

func (decode *decoder) getCSVRows() ([][]string, error) {
	return getCSVReader(decode.in).ReadAll()
}

func (decode *decoder) getCSVRow() ([]string, error) {
	if decode.csvDecoder == nil {
		decode.csvDecoder = &csvDecoder{getCSVReader(decode.in)}
	}
	return decode.csvDecoder.Read()
}

type CSVReader interface {
	Read() ([]string, error)
	ReadAll() ([][]string, error)
}

type csvDecoder struct {
	CSVReader
}

func (c csvDecoder) getCSVRows() ([][]string, error) {
	return c.ReadAll()
}

func (c csvDecoder) getCSVRow() ([]string, error) {
	return c.Read()
}

func mismatchStructFields(structInfo []fieldInfo, headers []string) []string {
	missing := make([]string, 0)
	if len(structInfo) == 0 {
		return missing
	}

	headerMap := make(map[string]struct{}, len(headers))
	for idx := range headers {
		headerMap[headers[idx]] = struct{}{}
	}

	for _, info := range structInfo {
		found := false
		for _, key := range info.keys {
			if _, ok := headerMap[key]; ok {
				found = true
				break
			}
		}
		if !found {
			missing = append(missing, info.keys...)
		}
	}
	return missing
}

func mismatchHeaderFields(structInfo []fieldInfo, headers []string) []string {
	missing := make([]string, 0)
	if len(headers) == 0 {
		return missing
	}

	keyMap := make(map[string]struct{}, 0)
	for _, info := range structInfo {
		for _, key := range info.keys {
			keyMap[key] = struct{}{}
		}
	}

	for _, header := range headers {
		if _, ok := keyMap[header]; !ok {
			missing = append(missing, header)
		}
	}
	return missing
}

func maybeMissingStructFields(structInfo []fieldInfo, headers []string) error {
	missing := mismatchStructFields(structInfo, headers)
	if len(missing) != 0 {
		return fmt.Errorf("found unmatched struct field with tags %v", missing)
	}
	return nil
}

// Check that no header name is repeated twice
func maybeDoubleHeaderNames(headers []string) error {
	headerMap := make(map[string]bool, len(headers))
	for _, v := range headers {
		if _, ok := headerMap[v]; ok {
			return fmt.Errorf("Repeated header name: %v", v)
		}
		headerMap[v] = true
	}
	return nil
}

func readTo(decoder Decoder, out interface{}) error {
	outValue, outType := getConcreteReflectValueAndType(out) // Get the concrete type (not pointer) (Slice<?> or Array<?>)
	if err := ensureOutType(outType); err != nil {
		return err
	}
	outInnerWasPointer, outInnerType := getConcreteContainerInnerType(outType) // Get the concrete inner type (not pointer) (Container<"?">)
	if err := ensureOutInnerType(outInnerType); err != nil {
		return err
	}
	csvRows, err := decoder.getCSVRows() // Get the CSV csvRows
	if err != nil {
		return err
	}
	if len(csvRows) == 0 {
		return errors.New("empty csv file given")
	}
	if err := ensureOutCapacity(&outValue, len(csvRows)); err != nil { // Ensure the container is big enough to hold the CSV content
		return err
	}
	outInnerStructInfo := getStructInfo(outInnerType) // Get the inner struct info to get CSV annotations
	if len(outInnerStructInfo.Fields) == 0 {
		return errors.New("no csv struct tags found")
	}

	headers := csvRows[0]
	body := csvRows[1:]

	csvHeadersLabels := make(map[int]*fieldInfo, len(outInnerStructInfo.Fields)) // Used to store the correspondance header <-> position in CSV

	headerCount := map[string]int{}
	for i, csvColumnHeader := range headers {
		curHeaderCount := headerCount[csvColumnHeader]
		if fieldInfo := getCSVFieldPosition(csvColumnHeader, outInnerStructInfo, curHeaderCount); fieldInfo != nil {
			csvHeadersLabels[i] = fieldInfo
			if ShouldAlignDuplicateHeadersWithStructFieldOrder {
				curHeaderCount++
				headerCount[csvColumnHeader] = curHeaderCount
			}
		}
	}

	if FailIfUnmatchedStructTags {
		if err := maybeMissingStructFields(outInnerStructInfo.Fields, headers); err != nil {
			return err
		}
	}
	if FailIfDoubleHeaderNames {
		if err := maybeDoubleHeaderNames(headers); err != nil {
			return err
		}
	}

	for i, csvRow := range body {
		outInner := createNewOutInner(outInnerWasPointer, outInnerType)
		for j, csvColumnContent := range csvRow {
			if fieldInfo, ok := csvHeadersLabels[j]; ok { // Position found accordingly to header name
				if err := setInnerField(&outInner, outInnerWasPointer, fieldInfo.IndexChain, csvColumnContent, fieldInfo.omitEmpty); err != nil { // Set field of struct
					return &csv.ParseError{
						Line:   i + 2, //add 2 to account for the header & 0-indexing of arrays
						Column: j + 1,
						Err:    err,
					}
				}
			}
		}
		outValue.Index(i).Set(outInner)
	}
	return nil
}

func readEach(decoder SimpleDecoder, c interface{}) error {
	headers, err := decoder.getCSVRow()
	if err != nil {
		return err
	}
	outValue, outType := getConcreteReflectValueAndType(c) // Get the concrete type (not pointer) (Slice<?> or Array<?>)
	if err := ensureOutType(outType); err != nil {
		return err
	}
	defer outValue.Close()
	outInnerWasPointer, outInnerType := getConcreteContainerInnerType(outType) // Get the concrete inner type (not pointer) (Container<"?">)
	if err := ensureOutInnerType(outInnerType); err != nil {
		return err
	}
	outInnerStructInfo := getStructInfo(outInnerType) // Get the inner struct info to get CSV annotations
	if len(outInnerStructInfo.Fields) == 0 {
		return errors.New("no csv struct tags found")
	}
	csvHeadersLabels := make(map[int]*fieldInfo, len(outInnerStructInfo.Fields)) // Used to store the correspondance header <-> position in CSV
	headerCount := map[string]int{}
	for i, csvColumnHeader := range headers {
		curHeaderCount := headerCount[csvColumnHeader]
		if fieldInfo := getCSVFieldPosition(csvColumnHeader, outInnerStructInfo, curHeaderCount); fieldInfo != nil {
			csvHeadersLabels[i] = fieldInfo
			if ShouldAlignDuplicateHeadersWithStructFieldOrder {
				curHeaderCount++
				headerCount[csvColumnHeader] = curHeaderCount
			}
		}
	}
	if err := maybeMissingStructFields(outInnerStructInfo.Fields, headers); err != nil {
		if FailIfUnmatchedStructTags {
			return err
		}
	}
	if FailIfDoubleHeaderNames {
		if err := maybeDoubleHeaderNames(headers); err != nil {
			return err
		}
	}
	i := 0
	for {
		line, err := decoder.getCSVRow()
		if err == io.EOF {
			break
		} else if err != nil {
			return err
		}
		outInner := createNewOutInner(outInnerWasPointer, outInnerType)
		for j, csvColumnContent := range line {
			if fieldInfo, ok := csvHeadersLabels[j]; ok { // Position found accordingly to header name
				if err := setInnerField(&outInner, outInnerWasPointer, fieldInfo.IndexChain, csvColumnContent, fieldInfo.omitEmpty); err != nil { // Set field of struct
					return &csv.ParseError{
						Line:   i + 2, //add 2 to account for the header & 0-indexing of arrays
						Column: j + 1,
						Err:    err,
					}
				}
			}
		}
		outValue.Send(outInner)
		i++
	}
	return nil
}

func readToWithoutHeaders(decoder Decoder, out interface{}) error {
	outValue, outType := getConcreteReflectValueAndType(out) // Get the concrete type (not pointer) (Slice<?> or Array<?>)
	if err := ensureOutType(outType); err != nil {
		return err
	}
	outInnerWasPointer, outInnerType := getConcreteContainerInnerType(outType) // Get the concrete inner type (not pointer) (Container<"?">)
	if err := ensureOutInnerType(outInnerType); err != nil {
		return err
	}
	csvRows, err := decoder.getCSVRows() // Get the CSV csvRows
	if err != nil {
		return err
	}
	if len(csvRows) == 0 {
		return errors.New("empty csv file given")
	}
	if err := ensureOutCapacity(&outValue, len(csvRows)+1); err != nil { // Ensure the container is big enough to hold the CSV content
		return err
	}
	outInnerStructInfo := getStructInfo(outInnerType) // Get the inner struct info to get CSV annotations
	if len(outInnerStructInfo.Fields) == 0 {
		return errors.New("no csv struct tags found")
	}

	for i, csvRow := range csvRows {
		outInner := createNewOutInner(outInnerWasPointer, outInnerType)
		for j, csvColumnContent := range csvRow {
			if err := setInnerField(&outInner, outInnerWasPointer, []int{j}, csvColumnContent, false); err != nil { // Set field of struct
				return &csv.ParseError{
					Line:   i + 2, //add 2 to account for the header & 0-indexing of arrays
					Column: j + 1,
					Err:    err,
				}
			}
		}
		outValue.Index(i).Set(outInner)
	}

	return nil
}

// Check if the outType is an array or a slice
func ensureOutType(outType reflect.Type) error {
	switch outType.Kind() {
	case reflect.Slice:
		fallthrough
	case reflect.Chan:
		fallthrough
	case reflect.Array:
		return nil
	}
	return fmt.Errorf("cannot use " + outType.String() + ", only slice or array supported")
}

// Check if the outInnerType is of type struct
func ensureOutInnerType(outInnerType reflect.Type) error {
	switch outInnerType.Kind() {
	case reflect.Struct:
		return nil
	}
	return fmt.Errorf("cannot use " + outInnerType.String() + ", only struct supported")
}

func ensureOutCapacity(out *reflect.Value, csvLen int) error {
	switch out.Kind() {
	case reflect.Array:
		if out.Len() < csvLen-1 { // Array is not big enough to hold the CSV content (arrays are not addressable)
			return fmt.Errorf("array capacity problem: cannot store %d %s in %s", csvLen-1, out.Type().Elem().String(), out.Type().String())
		}
	case reflect.Slice:
		if !out.CanAddr() && out.Len() < csvLen-1 { // Slice is not big enough tho hold the CSV content and is not addressable
			return fmt.Errorf("slice capacity problem and is not addressable (did you forget &?)")
		} else if out.CanAddr() && out.Len() < csvLen-1 {
			out.Set(reflect.MakeSlice(out.Type(), csvLen-1, csvLen-1)) // Slice is not big enough, so grows it
		}
	}
	return nil
}

func getCSVFieldPosition(key string, structInfo *structInfo, curHeaderCount int) *fieldInfo {
	matchedFieldCount := 0
	for _, field := range structInfo.Fields {
		if field.matchesKey(key) {
			if matchedFieldCount >= curHeaderCount {
				return &field
			} else {
				matchedFieldCount++
			}
		}
	}
	return nil
}

func createNewOutInner(outInnerWasPointer bool, outInnerType reflect.Type) reflect.Value {
	if outInnerWasPointer {
		return reflect.New(outInnerType)
	}
	return reflect.New(outInnerType).Elem()
}

func setInnerField(outInner *reflect.Value, outInnerWasPointer bool, index []int, value string, omitEmpty bool) error {
	oi := *outInner
	if outInnerWasPointer {
		oi = outInner.Elem()
	}
	return setField(oi.FieldByIndex(index), value, omitEmpty)
}
<commit_msg>Fix line error<commit_after>package gocsv

import (
	"encoding/csv"
	"errors"
	"fmt"
	"io"
	"reflect"
)

// Decoder .
type Decoder interface {
	getCSVRows() ([][]string, error)
}

// SimpleDecoder .
type SimpleDecoder interface {
	getCSVRow() ([]string, error)
}

type decoder struct {
	in         io.Reader
	csvDecoder *csvDecoder
}

func newDecoder(in io.Reader) *decoder {
	return &decoder{in: in}
}

func (decode *decoder) getCSVRows() ([][]string, error) {
	return getCSVReader(decode.in).ReadAll()
}

func (decode *decoder) getCSVRow() ([]string, error) {
	if decode.csvDecoder == nil {
		decode.csvDecoder = &csvDecoder{getCSVReader(decode.in)}
	}
	return decode.csvDecoder.Read()
}

type CSVReader interface {
	Read() ([]string, error)
	ReadAll() ([][]string, error)
}

type csvDecoder struct {
	CSVReader
}

func (c csvDecoder) getCSVRows() ([][]string, error) {
	return c.ReadAll()
}

func (c csvDecoder) getCSVRow() ([]string, error) {
	return c.Read()
}

func mismatchStructFields(structInfo []fieldInfo, headers []string) []string {
	missing := make([]string, 0)
	if len(structInfo) == 0 {
		return missing
	}

	headerMap := make(map[string]struct{}, len(headers))
	for idx := range headers {
		headerMap[headers[idx]] = struct{}{}
	}

	for _, info := range structInfo {
		found := false
		for _, key := range info.keys {
			if _, ok := headerMap[key]; ok {
				found = true
				break
			}
		}
		if !found {
			missing = append(missing, info.keys...)
		}
	}
	return missing
}

func mismatchHeaderFields(structInfo []fieldInfo, headers []string) []string {
	missing := make([]string, 0)
	if len(headers) == 0 {
		return missing
	}

	keyMap := make(map[string]struct{}, 0)
	for _, info := range structInfo {
		for _, key := range info.keys {
			keyMap[key] = struct{}{}
		}
	}

	for _, header := range headers {
		if _, ok := keyMap[header]; !ok {
			missing = append(missing, header)
		}
	}
	return missing
}

func maybeMissingStructFields(structInfo []fieldInfo, headers []string) error {
	missing := mismatchStructFields(structInfo, headers)
	if len(missing) != 0 {
		return fmt.Errorf("found unmatched struct field with tags %v", missing)
	}
	return nil
}

// Check that no header name is repeated twice
func maybeDoubleHeaderNames(headers []string) error {
	headerMap := make(map[string]bool, len(headers))
	for _, v := range headers {
		if _, ok := headerMap[v]; ok {
			return fmt.Errorf("Repeated header name: %v", v)
		}
		headerMap[v] = true
	}
	return nil
}

func readTo(decoder Decoder, out interface{}) error {
	outValue, outType := getConcreteReflectValueAndType(out) // Get the concrete type (not pointer) (Slice<?> or Array<?>)
	if err := ensureOutType(outType); err != nil {
		return err
	}
	outInnerWasPointer, outInnerType := getConcreteContainerInnerType(outType) // Get the concrete inner type (not pointer) (Container<"?">)
	if err := ensureOutInnerType(outInnerType); err != nil {
		return err
	}
	csvRows, err := decoder.getCSVRows() // Get the CSV csvRows
	if err != nil {
		return err
	}
	if len(csvRows) == 0 {
		return errors.New("empty csv file given")
	}
	if err := ensureOutCapacity(&outValue, len(csvRows)); err != nil { // Ensure the container is big enough to hold the CSV content
		return err
	}
	outInnerStructInfo := getStructInfo(outInnerType) // Get the inner struct info to get CSV annotations
	if len(outInnerStructInfo.Fields) == 0 {
		return errors.New("no csv struct tags found")
	}

	headers := csvRows[0]
	body := csvRows[1:]

	csvHeadersLabels := make(map[int]*fieldInfo, len(outInnerStructInfo.Fields)) // Used to store the correspondance header <-> position in CSV

	headerCount := map[string]int{}
	for i, csvColumnHeader := range headers {
		curHeaderCount := headerCount[csvColumnHeader]
		if fieldInfo := getCSVFieldPosition(csvColumnHeader, outInnerStructInfo, curHeaderCount); fieldInfo != nil {
			csvHeadersLabels[i] = fieldInfo
			if ShouldAlignDuplicateHeadersWithStructFieldOrder {
				curHeaderCount++
				headerCount[csvColumnHeader] = curHeaderCount
			}
		}
	}

	if FailIfUnmatchedStructTags {
		if err := maybeMissingStructFields(outInnerStructInfo.Fields, headers); err != nil {
			return err
		}
	}
	if FailIfDoubleHeaderNames {
		if err := maybeDoubleHeaderNames(headers); err != nil {
			return err
		}
	}

	for i, csvRow := range body {
		outInner := createNewOutInner(outInnerWasPointer, outInnerType)
		for j, csvColumnContent := range csvRow {
			if fieldInfo, ok := csvHeadersLabels[j]; ok { // Position found accordingly to header name
				if err := setInnerField(&outInner, outInnerWasPointer, fieldInfo.IndexChain, csvColumnContent, fieldInfo.omitEmpty); err != nil { // Set field of struct
					return &csv.ParseError{
						Line:   i + 2, //add 2 to account for the header & 0-indexing of arrays
						Column: j + 1,
						Err:    err,
					}
				}
			}
		}
		outValue.Index(i).Set(outInner)
	}
	return nil
}

func readEach(decoder SimpleDecoder, c interface{}) error {
	headers, err := decoder.getCSVRow()
	if err != nil {
		return err
	}
	outValue, outType := getConcreteReflectValueAndType(c) // Get the concrete type (not pointer) (Slice<?> or Array<?>)
	if err := ensureOutType(outType); err != nil {
		return err
	}
	defer outValue.Close()
	outInnerWasPointer, outInnerType := getConcreteContainerInnerType(outType) // Get the concrete inner type (not pointer) (Container<"?">)
	if err := ensureOutInnerType(outInnerType); err != nil {
		return err
	}
	outInnerStructInfo := getStructInfo(outInnerType) // Get the inner struct info to get CSV annotations
	if len(outInnerStructInfo.Fields) == 0 {
		return errors.New("no csv struct tags found")
	}
	csvHeadersLabels := make(map[int]*fieldInfo, len(outInnerStructInfo.Fields)) // Used to store the correspondance header <-> position in CSV
	headerCount := map[string]int{}
	for i, csvColumnHeader := range headers {
		curHeaderCount := headerCount[csvColumnHeader]
		if fieldInfo := getCSVFieldPosition(csvColumnHeader, outInnerStructInfo, curHeaderCount); fieldInfo != nil {
			csvHeadersLabels[i] = fieldInfo
			if ShouldAlignDuplicateHeadersWithStructFieldOrder {
				curHeaderCount++
				headerCount[csvColumnHeader] = curHeaderCount
			}
		}
	}
	if err := maybeMissingStructFields(outInnerStructInfo.Fields, headers); err != nil {
		if FailIfUnmatchedStructTags {
			return err
		}
	}
	if FailIfDoubleHeaderNames {
		if err := maybeDoubleHeaderNames(headers); err != nil {
			return err
		}
	}
	i := 0
	for {
		line, err := decoder.getCSVRow()
		if err == io.EOF {
			break
		} else if err != nil {
			return err
		}
		outInner := createNewOutInner(outInnerWasPointer, outInnerType)
		for j, csvColumnContent := range line {
			if fieldInfo, ok := csvHeadersLabels[j]; ok { // Position found accordingly to header name
				if err := setInnerField(&outInner, outInnerWasPointer, fieldInfo.IndexChain, csvColumnContent, fieldInfo.omitEmpty); err != nil { // Set field of struct
					return &csv.ParseError{
						Line:   i + 2, //add 2 to account for the header & 0-indexing of arrays
						Column: j + 1,
						Err:    err,
					}
				}
			}
		}
		outValue.Send(outInner)
		i++
	}
	return nil
}

func readToWithoutHeaders(decoder Decoder, out interface{}) error {
	outValue, outType := getConcreteReflectValueAndType(out) // Get the concrete type (not pointer) (Slice<?> or Array<?>)
	if err := ensureOutType(outType); err != nil {
		return err
	}
	outInnerWasPointer, outInnerType := getConcreteContainerInnerType(outType) // Get the concrete inner type (not pointer) (Container<"?">)
	if err := ensureOutInnerType(outInnerType); err != nil {
		return err
	}
	csvRows, err := decoder.getCSVRows() // Get the CSV csvRows
	if err != nil {
		return err
	}
	if len(csvRows) == 0 {
		return errors.New("empty csv file given")
	}
	if err := ensureOutCapacity(&outValue, len(csvRows)+1); err != nil { // Ensure the container is big enough to hold the CSV content
		return err
	}
	outInnerStructInfo := getStructInfo(outInnerType) // Get the inner struct info to get CSV annotations
	if len(outInnerStructInfo.Fields) == 0 {
		return errors.New("no csv struct tags found")
	}

	for i, csvRow := range csvRows {
		outInner := createNewOutInner(outInnerWasPointer, outInnerType)
		for j, csvColumnContent := range csvRow {
			if err := setInnerField(&outInner, outInnerWasPointer, []int{j}, csvColumnContent, false); err != nil { // Set field of struct
				return &csv.ParseError{
					Line:   i + 1, //add 2 to account for the header & 0-indexing of arrays
					Column: j + 1,
					Err:    err,
				}
			}
		}
		outValue.Index(i).Set(outInner)
	}

	return nil
}

// Check if the outType is an array or a slice
func ensureOutType(outType reflect.Type) error {
	switch outType.Kind() {
	case reflect.Slice:
		fallthrough
	case reflect.Chan:
		fallthrough
	case reflect.Array:
		return nil
	}
	return fmt.Errorf("cannot use " + outType.String() + ", only slice or array supported")
}

// Check if the outInnerType is of type struct
func ensureOutInnerType(outInnerType reflect.Type) error {
	switch outInnerType.Kind() {
	case reflect.Struct:
		return nil
	}
	return fmt.Errorf("cannot use " + outInnerType.String() + ", only struct supported")
}

func ensureOutCapacity(out *reflect.Value, csvLen int) error {
	switch out.Kind() {
	case reflect.Array:
		if out.Len() < csvLen-1 { // Array is not big enough to hold the CSV content (arrays are not addressable)
			return fmt.Errorf("array capacity problem: cannot store %d %s in %s", csvLen-1, out.Type().Elem().String(), out.Type().String())
		}
	case reflect.Slice:
		if !out.CanAddr() && out.Len() < csvLen-1 { // Slice is not big enough tho hold the CSV content and is not addressable
			return fmt.Errorf("slice capacity problem and is not addressable (did you forget &?)")
		} else if out.CanAddr() && out.Len() < csvLen-1 {
			out.Set(reflect.MakeSlice(out.Type(), csvLen-1, csvLen-1)) // Slice is not big enough, so grows it
		}
	}
	return nil
}

func getCSVFieldPosition(key string, structInfo *structInfo, curHeaderCount int) *fieldInfo {
	matchedFieldCount := 0
	for _, field := range structInfo.Fields {
		if field.matchesKey(key) {
			if matchedFieldCount >= curHeaderCount {
				return &field
			} else {
				matchedFieldCount++
			}
		}
	}
	return nil
}

func createNewOutInner(outInnerWasPointer bool, outInnerType reflect.Type) reflect.Value {
	if outInnerWasPointer {
		return reflect.New(outInnerType)
	}
	return reflect.New(outInnerType).Elem()
}

func setInnerField(outInner *reflect.Value, outInnerWasPointer bool, index []int, value string, omitEmpty bool) error {
	oi := *outInner
	if outInnerWasPointer {
		oi = outInner.Elem()
	}
	return setField(oi.FieldByIndex(index), value, omitEmpty)
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bigquery

// TODO(mcgreevy): support dry-run mode when creating jobs.

import (
	"fmt"

	"google.golang.org/api/option"
	"google.golang.org/api/transport"

	"golang.org/x/net/context"
	bq "google.golang.org/api/bigquery/v2"
)

const prodAddr = "https://www.googleapis.com/bigquery/v2/"

// A Source is a source of data for the Copy function.
type Source interface {
	implementsSource()
}

// A Destination is a destination of data for the Copy function.
type Destination interface {
	implementsDestination()
}

// An Option is an optional argument to Copy.
type Option interface {
	implementsOption()
}

// A ReadSource is a source of data for the Read function.
type ReadSource interface {
	implementsReadSource()
}

// A ReadOption is an optional argument to Read.
type ReadOption interface {
	customizeRead(conf *pagingConf)
}

const Scope = "https://www.googleapis.com/auth/bigquery"
const userAgent = "gcloud-golang-bigquery/20160429"

// Client may be used to perform BigQuery operations.
type Client struct {
	service   service
	projectID string
}

// NewClient constructs a new Client which can perform BigQuery operations.
// Operations performed via the client are billed to the specified GCP project.
func NewClient(ctx context.Context, projectID string, opts ...option.ClientOption) (*Client, error) {
	o := []option.ClientOption{
		option.WithEndpoint(prodAddr),
		option.WithScopes(Scope),
		option.WithUserAgent(userAgent),
	}
	o = append(o, opts...)
	httpClient, endpoint, err := transport.NewHTTPClient(ctx, o...)
	if err != nil {
		return nil, fmt.Errorf("dialing: %v", err)
	}

	s, err := newBigqueryService(httpClient, endpoint)
	if err != nil {
		return nil, fmt.Errorf("constructing bigquery client: %v", err)
	}

	c := &Client{
		service:   s,
		projectID: projectID,
	}
	return c, nil
}

// initJobProto creates and returns a bigquery Job proto.
// The proto is customized using any jobOptions in options.
// The list of Options is returned with the jobOptions removed.
func initJobProto(projectID string, options []Option) (*bq.Job, []Option) {
	job := &bq.Job{}

	var other []Option
	for _, opt := range options {
		if o, ok := opt.(jobOption); ok {
			o.customizeJob(job, projectID)
		} else {
			other = append(other, opt)
		}
	}
	return job, other
}

// Copy starts a BigQuery operation to copy data from a Source to a Destination.
func (c *Client) Copy(ctx context.Context, dst Destination, src Source, options ...Option) (*Job, error) {
	switch dst := dst.(type) {
	case *Table:
		switch src := src.(type) {
		case *GCSReference:
			return c.load(ctx, dst, src, options)
		case *Table:
			return c.cp(ctx, dst, Tables{src}, options)
		case Tables:
			return c.cp(ctx, dst, src, options)
		case *Query:
			return c.query(ctx, dst, src, options)
		}
	case *GCSReference:
		if src, ok := src.(*Table); ok {
			return c.extract(ctx, dst, src, options)
		}
	}
	return nil, fmt.Errorf("no Copy operation matches dst/src pair: dst: %T ; src: %T", dst, src)
}

// Query creates a query with string q. You may optionally set
// DefaultProjectID and DefaultDatasetID on the returned query before using it.
func (c *Client) Query(q string) *Query {
	return &Query{Q: q, client: c}
}

// Read submits a query for execution and returns the results via an Iterator.
//
// Read uses a temporary table to hold the results of the query job.
//
// For more control over how a query is performed, don't use this method but
// instead pass the Query as a Source to Client.Copy, and call Read on the
// resulting Job.
func (q *Query) Read(ctx context.Context, options ...ReadOption) (*Iterator, error) {
	dest := &Table{}
	job, err := q.client.Copy(ctx, dest, q, WriteTruncate)
	if err != nil {
		return nil, err
	}
	return job.Read(ctx, options...)
}

// executeQuery submits a query for execution and returns the results via an Iterator.
func (c *Client) executeQuery(ctx context.Context, q *Query, options ...ReadOption) (*Iterator, error) {
	dest := &Table{}
	job, err := c.Copy(ctx, dest, q, WriteTruncate)
	if err != nil {
		return nil, err
	}

	return c.Read(ctx, job, options...)
}

// Dataset creates a handle to a BigQuery dataset in the client's project.
func (c *Client) Dataset(id string) *Dataset {
	return c.DatasetInProject(c.projectID, id)
}

// DatasetInProject creates a handle to a BigQuery dataset in the specified project.
func (c *Client) DatasetInProject(projectID, datasetID string) *Dataset {
	return &Dataset{
		projectID: projectID,
		id:        datasetID,
		service:   c.service,
	}
}
<commit_msg>bigquery: add Client.Close<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bigquery

// TODO(mcgreevy): support dry-run mode when creating jobs.

import (
	"fmt"

	"google.golang.org/api/option"
	"google.golang.org/api/transport"

	"golang.org/x/net/context"
	bq "google.golang.org/api/bigquery/v2"
)

const prodAddr = "https://www.googleapis.com/bigquery/v2/"

// A Source is a source of data for the Copy function.
type Source interface {
	implementsSource()
}

// A Destination is a destination of data for the Copy function.
type Destination interface {
	implementsDestination()
}

// An Option is an optional argument to Copy.
type Option interface {
	implementsOption()
}

// A ReadSource is a source of data for the Read function.
type ReadSource interface {
	implementsReadSource()
}

// A ReadOption is an optional argument to Read.
type ReadOption interface {
	customizeRead(conf *pagingConf)
}

const Scope = "https://www.googleapis.com/auth/bigquery"
const userAgent = "gcloud-golang-bigquery/20160429"

// Client may be used to perform BigQuery operations.
type Client struct {
	service   service
	projectID string
}

// NewClient constructs a new Client which can perform BigQuery operations.
// Operations performed via the client are billed to the specified GCP project.
func NewClient(ctx context.Context, projectID string, opts ...option.ClientOption) (*Client, error) {
	o := []option.ClientOption{
		option.WithEndpoint(prodAddr),
		option.WithScopes(Scope),
		option.WithUserAgent(userAgent),
	}
	o = append(o, opts...)
	httpClient, endpoint, err := transport.NewHTTPClient(ctx, o...)
	if err != nil {
		return nil, fmt.Errorf("dialing: %v", err)
	}

	s, err := newBigqueryService(httpClient, endpoint)
	if err != nil {
		return nil, fmt.Errorf("constructing bigquery client: %v", err)
	}

	c := &Client{
		service:   s,
		projectID: projectID,
	}
	return c, nil
}

// Close closes any resources held by the client.
// Close should be called when the client is no longer needed.
// It need not be called at program exit.
func (c *Client) Close() error {
	return nil
}

// initJobProto creates and returns a bigquery Job proto.
// The proto is customized using any jobOptions in options.
// The list of Options is returned with the jobOptions removed.
func initJobProto(projectID string, options []Option) (*bq.Job, []Option) {
	job := &bq.Job{}

	var other []Option
	for _, opt := range options {
		if o, ok := opt.(jobOption); ok {
			o.customizeJob(job, projectID)
		} else {
			other = append(other, opt)
		}
	}
	return job, other
}

// Copy starts a BigQuery operation to copy data from a Source to a Destination.
func (c *Client) Copy(ctx context.Context, dst Destination, src Source, options ...Option) (*Job, error) {
	switch dst := dst.(type) {
	case *Table:
		switch src := src.(type) {
		case *GCSReference:
			return c.load(ctx, dst, src, options)
		case *Table:
			return c.cp(ctx, dst, Tables{src}, options)
		case Tables:
			return c.cp(ctx, dst, src, options)
		case *Query:
			return c.query(ctx, dst, src, options)
		}
	case *GCSReference:
		if src, ok := src.(*Table); ok {
			return c.extract(ctx, dst, src, options)
		}
	}
	return nil, fmt.Errorf("no Copy operation matches dst/src pair: dst: %T ; src: %T", dst, src)
}

// Query creates a query with string q. You may optionally set
// DefaultProjectID and DefaultDatasetID on the returned query before using it.
func (c *Client) Query(q string) *Query {
	return &Query{Q: q, client: c}
}

// Read submits a query for execution and returns the results via an Iterator.
//
// Read uses a temporary table to hold the results of the query job.
//
// For more control over how a query is performed, don't use this method but
// instead pass the Query as a Source to Client.Copy, and call Read on the
// resulting Job.
func (q *Query) Read(ctx context.Context, options ...ReadOption) (*Iterator, error) {
	dest := &Table{}
	job, err := q.client.Copy(ctx, dest, q, WriteTruncate)
	if err != nil {
		return nil, err
	}
	return job.Read(ctx, options...)
}

// executeQuery submits a query for execution and returns the results via an Iterator.
func (c *Client) executeQuery(ctx context.Context, q *Query, options ...ReadOption) (*Iterator, error) {
	dest := &Table{}
	job, err := c.Copy(ctx, dest, q, WriteTruncate)
	if err != nil {
		return nil, err
	}

	return c.Read(ctx, job, options...)
}

// Dataset creates a handle to a BigQuery dataset in the client's project.
func (c *Client) Dataset(id string) *Dataset {
	return c.DatasetInProject(c.projectID, id)
}

// DatasetInProject creates a handle to a BigQuery dataset in the specified project.
func (c *Client) DatasetInProject(projectID, datasetID string) *Dataset {
	return &Dataset{
		projectID: projectID,
		id:        datasetID,
		service:   c.service,
	}
}
<|endoftext|>


<commit_before>package defect

import (
	"reflect"
	"runtime"
	"strconv"
	"testing"
)

// Similar to errors.New(), but can be a const
type Error string

var _ error = Error("")

func (e Error) Error() string { return string(e) }

// Asserts that obtained and expected are equal.  If they are not, sets the test as an error,
// prints the optional message, followed by the values of obtained and expected.
func Equal(t *testing.T, obtained, expected interface{}, message ...interface{}) {
	if obtained != expected {
		t.Error(append(message, []interface{}{
			"\n",
			lineAndFile(2),
			"obtained:", obtained, "\n",
			"expected:", expected, "\n"}...)...)
	}
}

// Asserts that obtained and expected are equal using reflect.DeepEqual.  If they are not,
// sets the test as an error, prints the optional message, followed by the values of obtained
// and expected.
func DeepEqual(t *testing.T, obtained, expected interface{}, message ...interface{}) {
	if !reflect.DeepEqual(obtained, expected) {
		t.Error(append(message, []interface{}{
			"\n",
			"obtained:", obtained, "\n",
			"expected:", expected, "\n"}...)...)
	}
}

func lineAndFile(skip int) string {
	if _, file, line, ok := runtime.Caller(skip); ok {
		return file + ":" + strconv.Itoa(line) + "\n"
	}
	return ""
}
<commit_msg>Only display the file name<commit_after>package defect

import (
	"path/filepath"
	"reflect"
	"runtime"
	"strconv"
	"testing"
)

// Similar to errors.New(), but can be a const
type Error string

var _ error = Error("")

func (e Error) Error() string { return string(e) }

// Asserts that obtained and expected are equal.  If they are not, sets the test as an error,
// prints the optional message, followed by the values of obtained and expected.
func Equal(t *testing.T, obtained, expected interface{}, message ...interface{}) {
	if obtained != expected {
		t.Error(append(message, []interface{}{
			"\n",
			lineAndFile(2),
			"obtained:", obtained, "\n",
			"expected:", expected, "\n"}...)...)
	}
}

// Asserts that obtained and expected are equal using reflect.DeepEqual.  If they are not,
// sets the test as an error, prints the optional message, followed by the values of obtained
// and expected.
func DeepEqual(t *testing.T, obtained, expected interface{}, message ...interface{}) {
	if !reflect.DeepEqual(obtained, expected) {
		t.Error(append(message, []interface{}{
			"\n",
			lineAndFile(2),
			"obtained:", obtained, "\n",
			"expected:", expected, "\n"}...)...)
	}
}

func lineAndFile(skip int) string {
	if _, file, line, ok := runtime.Caller(skip); ok {
		return filepath.Base(file) + ":" + strconv.Itoa(line) + "\n"
	}
	return ""
}
<|endoftext|>


<commit_before>// Copyright 2015 - António Meireles  <antonio.meireles@reformi.st>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package main

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/blang/semver"
	"github.com/spf13/cobra"
)

var (
	rmCmd = &cobra.Command{
		Use:     "rm",
		Aliases: []string{"rmi"},
		Short:   "Removes one or more CoreOS images from local fs",
		PreRunE: defaultPreRunE,
		RunE:    rmCommand,
	}
)

func defaultPreRunE(cmd *cobra.Command, args []string) (err error) {
	if len(args) != 0 {
		return fmt.Errorf("Incorrect usage. " +
			"This command doesn't accept any arguments.")
	}
	vipre.BindPFlags(cmd.Flags())
	return err
}

func rmCommand(cmd *cobra.Command, args []string) (err error) {
	var (
		channel = vipre.GetString("channel")
		version = vipre.GetString("version")
		ll      map[string]semver.Versions
		l       semver.Versions
	)

	if ll, err = localImages(); err != nil {
		return err
	}
	l = ll[channel]

	if vipre.GetBool("old") {
		for _, v := range l[0 : l.Len()-1] {
			if err = os.RemoveAll(filepath.Join(SessionContext.imageDir,
				channel, "/", v.String())); err != nil {
				return
			}
		}
		return
	}

	if version == "latest" {
		version = l[l.Len()-1].String()
	}

	target := filepath.Join(SessionContext.imageDir, channel, "/", version)
	if _, err = os.Stat(target); err != nil {
		return err
	}
	return os.RemoveAll(target)
}

func init() {
	rmCmd.Flags().String("channel", "alpha", "CoreOS channel")
	rmCmd.Flags().String("version", "latest", "CoreOS version")
	rmCmd.Flags().Bool("old", false, "removes outdated images")
	RootCmd.AddCommand(rmCmd)
}
<commit_msg>avert panic on rm when there's nothing to delete<commit_after>// Copyright 2015 - António Meireles  <antonio.meireles@reformi.st>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package main

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/blang/semver"
	"github.com/spf13/cobra"
)

var (
	rmCmd = &cobra.Command{
		Use:     "rm",
		Aliases: []string{"rmi"},
		Short:   "Removes one or more CoreOS images from local fs",
		PreRunE: defaultPreRunE,
		RunE:    rmCommand,
	}
)

func defaultPreRunE(cmd *cobra.Command, args []string) (err error) {
	if len(args) != 0 {
		return fmt.Errorf("Incorrect usage. " +
			"This command doesn't accept any arguments.")
	}
	vipre.BindPFlags(cmd.Flags())
	return err
}

func rmCommand(cmd *cobra.Command, args []string) (err error) {
	var (
		channel = vipre.GetString("channel")
		version = vipre.GetString("version")
		ll      map[string]semver.Versions
		l       semver.Versions
	)

	if ll, err = localImages(); err != nil {
		return err
	}
	l = ll[channel]

	if vipre.GetBool("old") {
		for _, v := range l[0 : l.Len()-1] {
			if err = os.RemoveAll(filepath.Join(SessionContext.imageDir,
				channel, "/", v.String())); err != nil {
				return
			}
		}
		return
	}

	if version == "latest" {
		if l.Len() > 0 {
			version = l[l.Len()-1].String()
		} else {
			log.Println("nothing to delete")
			return
		}
	}

	target := filepath.Join(SessionContext.imageDir, channel, "/", version)
	if _, err = os.Stat(target); err != nil {
		log.Printf("%s/%s not found\n", channel, version)
		return nil
	}
	return os.RemoveAll(target)
}

func init() {
	rmCmd.Flags().String("channel", "alpha", "CoreOS channel")
	rmCmd.Flags().String("version", "latest", "CoreOS version")
	rmCmd.Flags().Bool("old", false, "removes outdated images")
	RootCmd.AddCommand(rmCmd)
}
<|endoftext|>


<commit_before>// +build ignore

// Cross compile rclone - in go because I hate bash ;-)

package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"regexp"
	"runtime"
	"sort"
	"strings"
	"sync"
	"text/template"
	"time"

	"github.com/coreos/go-semver/semver"
)

var (
	// Flags
	debug       = flag.Bool("d", false, "Print commands instead of running them.")
	parallel    = flag.Int("parallel", runtime.NumCPU(), "Number of commands to run in parallel.")
	copyAs      = flag.String("release", "", "Make copies of the releases with this name")
	gitLog      = flag.String("git-log", "", "git log to include as well")
	include     = flag.String("include", "^.*$", "os/arch regexp to include")
	exclude     = flag.String("exclude", "^$", "os/arch regexp to exclude")
	cgo         = flag.Bool("cgo", false, "Use cgo for the build")
	noClean     = flag.Bool("no-clean", false, "Don't clean the build directory before running.")
	tags        = flag.String("tags", "", "Space separated list of build tags")
	compileOnly = flag.Bool("compile-only", false, "Just build the binary, not the zip.")
)

// GOOS/GOARCH pairs we build for
//
// If the GOARCH contains a - it is a synthetic arch with more parameters
var osarches = []string{
	"windows/386",
	"windows/amd64",
	"darwin/amd64",
	"linux/386",
	"linux/amd64",
	"linux/arm",
	"linux/arm-v7",
	"linux/arm64",
	"linux/mips",
	"linux/mipsle",
	"freebsd/386",
	"freebsd/amd64",
	"freebsd/arm",
	"freebsd/arm-v7",
	"netbsd/386",
	"netbsd/amd64",
	"netbsd/arm",
	"netbsd/arm-v7",
	"openbsd/386",
	"openbsd/amd64",
	"plan9/386",
	"plan9/amd64",
	"solaris/amd64",
	"js/wasm",
}

// Special environment flags for a given arch
var archFlags = map[string][]string{
	"386":    {"GO386=387"},
	"mips":   {"GOMIPS=softfloat"},
	"mipsle": {"GOMIPS=softfloat"},
	"arm-v7": {"GOARM=7"},
}

// runEnv - run a shell command with env
func runEnv(args, env []string) error {
	if *debug {
		args = append([]string{"echo"}, args...)
	}
	cmd := exec.Command(args[0], args[1:]...)
	if env != nil {
		cmd.Env = append(os.Environ(), env...)
	}
	if *debug {
		log.Printf("args = %v, env = %v\n", args, cmd.Env)
	}
	out, err := cmd.CombinedOutput()
	if err != nil {
		log.Print("----------------------------")
		log.Printf("Failed to run %v: %v", args, err)
		log.Printf("Command output was:\n%s", out)
		log.Print("----------------------------")
	}
	return err
}

// run a shell command
func run(args ...string) {
	err := runEnv(args, nil)
	if err != nil {
		log.Fatalf("Exiting after error: %v", err)
	}
}

// chdir or die
func chdir(dir string) {
	err := os.Chdir(dir)
	if err != nil {
		log.Fatalf("Couldn't cd into %q: %v", dir, err)
	}
}

// substitute data from go template file in to file out
func substitute(inFile, outFile string, data interface{}) {
	t, err := template.ParseFiles(inFile)
	if err != nil {
		log.Fatalf("Failed to read template file %q: %v %v", inFile, err)
	}
	out, err := os.Create(outFile)
	if err != nil {
		log.Fatalf("Failed to create output file %q: %v %v", outFile, err)
	}
	defer func() {
		err := out.Close()
		if err != nil {
			log.Fatalf("Failed to close output file %q: %v %v", outFile, err)
		}
	}()
	err = t.Execute(out, data)
	if err != nil {
		log.Fatalf("Failed to substitute template file %q: %v %v", inFile, err)
	}
}

// build the zip package return its name
func buildZip(dir string) string {
	// Now build the zip
	run("cp", "-a", "../MANUAL.txt", filepath.Join(dir, "README.txt"))
	run("cp", "-a", "../MANUAL.html", filepath.Join(dir, "README.html"))
	run("cp", "-a", "../rclone.1", dir)
	if *gitLog != "" {
		run("cp", "-a", *gitLog, dir)
	}
	zip := dir + ".zip"
	run("zip", "-r9", zip, dir)
	return zip
}

// Build .deb and .rpm packages
//
// It returns a list of artifacts it has made
func buildDebAndRpm(dir, version, goarch string) []string {
	// Make internal version number acceptable to .deb and .rpm
	pkgVersion := version[1:]
	pkgVersion = strings.Replace(pkgVersion, "β", "-beta", -1)
	pkgVersion = strings.Replace(pkgVersion, "-", ".", -1)

	// Make nfpm.yaml from the template
	substitute("../bin/nfpm.yaml", path.Join(dir, "nfpm.yaml"), map[string]string{
		"Version": pkgVersion,
		"Arch":    goarch,
	})

	// build them
	var artifacts []string
	for _, pkg := range []string{".deb", ".rpm"} {
		artifact := dir + pkg
		run("bash", "-c", "cd "+dir+" && nfpm -f nfpm.yaml pkg -t ../"+artifact)
		artifacts = append(artifacts, artifact)
	}

	return artifacts
}

// generate system object (syso) file to be picked up by a following go build for embedding icon and version info resources into windows executable
func buildWindowsResourceSyso(goarch string, versionTag string) string {
	type M map[string]interface{}
	version := strings.TrimPrefix(versionTag, "v")
	semanticVersion := semver.New(version)

	// Build json input to goversioninfo utility
	bs, err := json.Marshal(M{
		"FixedFileInfo": M{
			"FileVersion": M{
				"Major": semanticVersion.Major,
				"Minor": semanticVersion.Minor,
				"Patch": semanticVersion.Patch,
			},
			"ProductVersion": M{
				"Major": semanticVersion.Major,
				"Minor": semanticVersion.Minor,
				"Patch": semanticVersion.Patch,
			},
		},
		"StringFileInfo": M{
			"CompanyName":      "https://rclone.org",
			"ProductName":      "Rclone",
			"FileDescription":  "Rsync for cloud storage",
			"InternalName":     "rclone",
			"OriginalFilename": "rclone.exe",
			"LegalCopyright":   "The Rclone Authors",
			"FileVersion":      version,
			"ProductVersion":   version,
		},
		"IconPath": "../graphics/logo/ico/logo_symbol_color.ico",
	})
	if err != nil {
		log.Printf("Failed to build version info json: %v", err)
		return ""
	}

	// Write json to temporary file that will only be used by the goversioninfo command executed below.
	jsonPath, err := filepath.Abs("versioninfo_windows_" + goarch + ".json") // Appending goos and goarch as suffix to avoid any race conditions
	if err != nil {
		log.Printf("Failed to resolve path: %v", err)
		return ""
	}
	err = ioutil.WriteFile(jsonPath, bs, 0644)
	if err != nil {
		log.Printf("Failed to write %s: %v", jsonPath, err)
		return ""
	}
	defer func() {
		if err := os.Remove(jsonPath); err != nil {
			if !os.IsNotExist(err) {
				log.Printf("Warning: Couldn't remove generated %s: %v. Please remove it manually.", jsonPath, err)
			}
		}
	}()

	// Execute goversioninfo utility using the json file as input.
	// It will produce a system object (syso) file that a following go build should pick up.
	sysoPath, err := filepath.Abs("../resource_windows_" + goarch + ".syso") // Appending goos and goarch as suffix to avoid any race conditions, and also it is recognized by go build and avoids any builds for other systems considering it
	if err != nil {
		log.Printf("Failed to resolve path: %v", err)
		return ""
	}
	args := []string{
		"goversioninfo",
		"-o",
		sysoPath,
	}
	if goarch == "amd64" {
		args = append(args, "-64") // Make the syso a 64-bit coff file
	}
	args = append(args, jsonPath)
	err = runEnv(args, nil)
	if err != nil {
		return ""
	}

	return sysoPath
}

// delete generated system object (syso) resource file
func cleanupResourceSyso(sysoFilePath string) {
	if sysoFilePath == "" {
		return
	}
	if err := os.Remove(sysoFilePath); err != nil {
		if !os.IsNotExist(err) {
			log.Printf("Warning: Couldn't remove generated %s: %v. Please remove it manually.", sysoFilePath, err)
		}
	}
}

// Trip a version suffix off the arch if present
func stripVersion(goarch string) string {
	i := strings.Index(goarch, "-")
	if i < 0 {
		return goarch
	}
	return goarch[:i]
}

// build the binary in dir returning success or failure
func compileArch(version, goos, goarch, dir string) bool {
	log.Printf("Compiling %s/%s", goos, goarch)
	output := filepath.Join(dir, "rclone")
	if goos == "windows" {
		output += ".exe"
		sysoPath := buildWindowsResourceSyso(goarch, version)
		if sysoPath == "" {
			log.Printf("Warning: Windows binaries will not have file information embedded")
		}
		defer cleanupResourceSyso(sysoPath)
	}
	err := os.MkdirAll(dir, 0777)
	if err != nil {
		log.Fatalf("Failed to mkdir: %v", err)
	}
	args := []string{
		"go", "build",
		"--ldflags", "-s -X github.com/rclone/rclone/fs.Version=" + version,
		"-trimpath",
		"-i",
		"-o", output,
		"-tags", *tags,
		"..",
	}
	env := []string{
		"GOOS=" + goos,
		"GOARCH=" + stripVersion(goarch),
	}
	if !*cgo {
		env = append(env, "CGO_ENABLED=0")
	} else {
		env = append(env, "CGO_ENABLED=1")
	}
	if flags, ok := archFlags[goarch]; ok {
		env = append(env, flags...)
	}
	err = runEnv(args, env)
	if err != nil {
		log.Printf("Error compiling %s/%s: %v", goos, goarch, err)
		return false
	}
	if !*compileOnly {
		if goos != "js" {
			artifacts := []string{buildZip(dir)}
			// build a .deb and .rpm if appropriate
			if goos == "linux" {
				artifacts = append(artifacts, buildDebAndRpm(dir, version, goarch)...)
			}
			if *copyAs != "" {
				for _, artifact := range artifacts {
					run("ln", artifact, strings.Replace(artifact, "-"+version, "-"+*copyAs, 1))
				}
			}
		}
		// tidy up
		run("rm", "-rf", dir)
	}
	log.Printf("Done compiling %s/%s", goos, goarch)
	return true
}

func compile(version string) {
	start := time.Now()
	wg := new(sync.WaitGroup)
	run := make(chan func(), *parallel)
	for i := 0; i < *parallel; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for f := range run {
				f()
			}
		}()
	}
	includeRe, err := regexp.Compile(*include)
	if err != nil {
		log.Fatalf("Bad -include regexp: %v", err)
	}
	excludeRe, err := regexp.Compile(*exclude)
	if err != nil {
		log.Fatalf("Bad -exclude regexp: %v", err)
	}
	compiled := 0
	var failuresMu sync.Mutex
	var failures []string
	for _, osarch := range osarches {
		if excludeRe.MatchString(osarch) || !includeRe.MatchString(osarch) {
			continue
		}
		parts := strings.Split(osarch, "/")
		if len(parts) != 2 {
			log.Fatalf("Bad osarch %q", osarch)
		}
		goos, goarch := parts[0], parts[1]
		userGoos := goos
		if goos == "darwin" {
			userGoos = "osx"
		}
		dir := filepath.Join("rclone-" + version + "-" + userGoos + "-" + goarch)
		run <- func() {
			if !compileArch(version, goos, goarch, dir) {
				failuresMu.Lock()
				failures = append(failures, goos+"/"+goarch)
				failuresMu.Unlock()
			}
		}
		compiled++
	}
	close(run)
	wg.Wait()
	log.Printf("Compiled %d arches in %v", compiled, time.Since(start))
	if len(failures) > 0 {
		sort.Strings(failures)
		log.Printf("%d compile failures:\n  %s\n", len(failures), strings.Join(failures, "\n  "))
		os.Exit(1)
	}
}

func main() {
	flag.Parse()
	args := flag.Args()
	if len(args) != 1 {
		log.Fatalf("Syntax: %s <version>", os.Args[0])
	}
	version := args[0]
	if !*noClean {
		run("rm", "-rf", "build")
		run("mkdir", "build")
	}
	chdir("build")
	err := ioutil.WriteFile("version.txt", []byte(fmt.Sprintf("rclone %s\n", version)), 0666)
	if err != nil {
		log.Fatalf("Couldn't write version.txt: %v", err)
	}
	compile(version)
}
<commit_msg>build: explicitly set ARM version to fix build #4553<commit_after>// +build ignore

// Cross compile rclone - in go because I hate bash ;-)

package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"regexp"
	"runtime"
	"sort"
	"strings"
	"sync"
	"text/template"
	"time"

	"github.com/coreos/go-semver/semver"
)

var (
	// Flags
	debug       = flag.Bool("d", false, "Print commands instead of running them.")
	parallel    = flag.Int("parallel", runtime.NumCPU(), "Number of commands to run in parallel.")
	copyAs      = flag.String("release", "", "Make copies of the releases with this name")
	gitLog      = flag.String("git-log", "", "git log to include as well")
	include     = flag.String("include", "^.*$", "os/arch regexp to include")
	exclude     = flag.String("exclude", "^$", "os/arch regexp to exclude")
	cgo         = flag.Bool("cgo", false, "Use cgo for the build")
	noClean     = flag.Bool("no-clean", false, "Don't clean the build directory before running.")
	tags        = flag.String("tags", "", "Space separated list of build tags")
	compileOnly = flag.Bool("compile-only", false, "Just build the binary, not the zip.")
)

// GOOS/GOARCH pairs we build for
//
// If the GOARCH contains a - it is a synthetic arch with more parameters
var osarches = []string{
	"windows/386",
	"windows/amd64",
	"darwin/amd64",
	"linux/386",
	"linux/amd64",
	"linux/arm",
	"linux/arm-v7",
	"linux/arm64",
	"linux/mips",
	"linux/mipsle",
	"freebsd/386",
	"freebsd/amd64",
	"freebsd/arm",
	"freebsd/arm-v7",
	"netbsd/386",
	"netbsd/amd64",
	"netbsd/arm",
	"netbsd/arm-v7",
	"openbsd/386",
	"openbsd/amd64",
	"plan9/386",
	"plan9/amd64",
	"solaris/amd64",
	"js/wasm",
}

// Special environment flags for a given arch
var archFlags = map[string][]string{
	"386":    {"GO386=387"},
	"mips":   {"GOMIPS=softfloat"},
	"mipsle": {"GOMIPS=softfloat"},
	"arm":    {"GOARM=6"},
	"arm-v7": {"GOARM=7"},
}

// runEnv - run a shell command with env
func runEnv(args, env []string) error {
	if *debug {
		args = append([]string{"echo"}, args...)
	}
	cmd := exec.Command(args[0], args[1:]...)
	if env != nil {
		cmd.Env = append(os.Environ(), env...)
	}
	if *debug {
		log.Printf("args = %v, env = %v\n", args, cmd.Env)
	}
	out, err := cmd.CombinedOutput()
	if err != nil {
		log.Print("----------------------------")
		log.Printf("Failed to run %v: %v", args, err)
		log.Printf("Command output was:\n%s", out)
		log.Print("----------------------------")
	}
	return err
}

// run a shell command
func run(args ...string) {
	err := runEnv(args, nil)
	if err != nil {
		log.Fatalf("Exiting after error: %v", err)
	}
}

// chdir or die
func chdir(dir string) {
	err := os.Chdir(dir)
	if err != nil {
		log.Fatalf("Couldn't cd into %q: %v", dir, err)
	}
}

// substitute data from go template file in to file out
func substitute(inFile, outFile string, data interface{}) {
	t, err := template.ParseFiles(inFile)
	if err != nil {
		log.Fatalf("Failed to read template file %q: %v %v", inFile, err)
	}
	out, err := os.Create(outFile)
	if err != nil {
		log.Fatalf("Failed to create output file %q: %v %v", outFile, err)
	}
	defer func() {
		err := out.Close()
		if err != nil {
			log.Fatalf("Failed to close output file %q: %v %v", outFile, err)
		}
	}()
	err = t.Execute(out, data)
	if err != nil {
		log.Fatalf("Failed to substitute template file %q: %v %v", inFile, err)
	}
}

// build the zip package return its name
func buildZip(dir string) string {
	// Now build the zip
	run("cp", "-a", "../MANUAL.txt", filepath.Join(dir, "README.txt"))
	run("cp", "-a", "../MANUAL.html", filepath.Join(dir, "README.html"))
	run("cp", "-a", "../rclone.1", dir)
	if *gitLog != "" {
		run("cp", "-a", *gitLog, dir)
	}
	zip := dir + ".zip"
	run("zip", "-r9", zip, dir)
	return zip
}

// Build .deb and .rpm packages
//
// It returns a list of artifacts it has made
func buildDebAndRpm(dir, version, goarch string) []string {
	// Make internal version number acceptable to .deb and .rpm
	pkgVersion := version[1:]
	pkgVersion = strings.Replace(pkgVersion, "β", "-beta", -1)
	pkgVersion = strings.Replace(pkgVersion, "-", ".", -1)

	// Make nfpm.yaml from the template
	substitute("../bin/nfpm.yaml", path.Join(dir, "nfpm.yaml"), map[string]string{
		"Version": pkgVersion,
		"Arch":    goarch,
	})

	// build them
	var artifacts []string
	for _, pkg := range []string{".deb", ".rpm"} {
		artifact := dir + pkg
		run("bash", "-c", "cd "+dir+" && nfpm -f nfpm.yaml pkg -t ../"+artifact)
		artifacts = append(artifacts, artifact)
	}

	return artifacts
}

// generate system object (syso) file to be picked up by a following go build for embedding icon and version info resources into windows executable
func buildWindowsResourceSyso(goarch string, versionTag string) string {
	type M map[string]interface{}
	version := strings.TrimPrefix(versionTag, "v")
	semanticVersion := semver.New(version)

	// Build json input to goversioninfo utility
	bs, err := json.Marshal(M{
		"FixedFileInfo": M{
			"FileVersion": M{
				"Major": semanticVersion.Major,
				"Minor": semanticVersion.Minor,
				"Patch": semanticVersion.Patch,
			},
			"ProductVersion": M{
				"Major": semanticVersion.Major,
				"Minor": semanticVersion.Minor,
				"Patch": semanticVersion.Patch,
			},
		},
		"StringFileInfo": M{
			"CompanyName":      "https://rclone.org",
			"ProductName":      "Rclone",
			"FileDescription":  "Rsync for cloud storage",
			"InternalName":     "rclone",
			"OriginalFilename": "rclone.exe",
			"LegalCopyright":   "The Rclone Authors",
			"FileVersion":      version,
			"ProductVersion":   version,
		},
		"IconPath": "../graphics/logo/ico/logo_symbol_color.ico",
	})
	if err != nil {
		log.Printf("Failed to build version info json: %v", err)
		return ""
	}

	// Write json to temporary file that will only be used by the goversioninfo command executed below.
	jsonPath, err := filepath.Abs("versioninfo_windows_" + goarch + ".json") // Appending goos and goarch as suffix to avoid any race conditions
	if err != nil {
		log.Printf("Failed to resolve path: %v", err)
		return ""
	}
	err = ioutil.WriteFile(jsonPath, bs, 0644)
	if err != nil {
		log.Printf("Failed to write %s: %v", jsonPath, err)
		return ""
	}
	defer func() {
		if err := os.Remove(jsonPath); err != nil {
			if !os.IsNotExist(err) {
				log.Printf("Warning: Couldn't remove generated %s: %v. Please remove it manually.", jsonPath, err)
			}
		}
	}()

	// Execute goversioninfo utility using the json file as input.
	// It will produce a system object (syso) file that a following go build should pick up.
	sysoPath, err := filepath.Abs("../resource_windows_" + goarch + ".syso") // Appending goos and goarch as suffix to avoid any race conditions, and also it is recognized by go build and avoids any builds for other systems considering it
	if err != nil {
		log.Printf("Failed to resolve path: %v", err)
		return ""
	}
	args := []string{
		"goversioninfo",
		"-o",
		sysoPath,
	}
	if goarch == "amd64" {
		args = append(args, "-64") // Make the syso a 64-bit coff file
	}
	args = append(args, jsonPath)
	err = runEnv(args, nil)
	if err != nil {
		return ""
	}

	return sysoPath
}

// delete generated system object (syso) resource file
func cleanupResourceSyso(sysoFilePath string) {
	if sysoFilePath == "" {
		return
	}
	if err := os.Remove(sysoFilePath); err != nil {
		if !os.IsNotExist(err) {
			log.Printf("Warning: Couldn't remove generated %s: %v. Please remove it manually.", sysoFilePath, err)
		}
	}
}

// Trip a version suffix off the arch if present
func stripVersion(goarch string) string {
	i := strings.Index(goarch, "-")
	if i < 0 {
		return goarch
	}
	return goarch[:i]
}

// build the binary in dir returning success or failure
func compileArch(version, goos, goarch, dir string) bool {
	log.Printf("Compiling %s/%s into %s", goos, goarch, dir)
	output := filepath.Join(dir, "rclone")
	if goos == "windows" {
		output += ".exe"
		sysoPath := buildWindowsResourceSyso(goarch, version)
		if sysoPath == "" {
			log.Printf("Warning: Windows binaries will not have file information embedded")
		}
		defer cleanupResourceSyso(sysoPath)
	}
	err := os.MkdirAll(dir, 0777)
	if err != nil {
		log.Fatalf("Failed to mkdir: %v", err)
	}
	args := []string{
		"go", "build",
		"--ldflags", "-s -X github.com/rclone/rclone/fs.Version=" + version,
		"-trimpath",
		"-i",
		"-o", output,
		"-tags", *tags,
		"..",
	}
	env := []string{
		"GOOS=" + goos,
		"GOARCH=" + stripVersion(goarch),
	}
	if !*cgo {
		env = append(env, "CGO_ENABLED=0")
	} else {
		env = append(env, "CGO_ENABLED=1")
	}
	if flags, ok := archFlags[goarch]; ok {
		env = append(env, flags...)
	}
	err = runEnv(args, env)
	if err != nil {
		log.Printf("Error compiling %s/%s: %v", goos, goarch, err)
		return false
	}
	if !*compileOnly {
		if goos != "js" {
			artifacts := []string{buildZip(dir)}
			// build a .deb and .rpm if appropriate
			if goos == "linux" {
				artifacts = append(artifacts, buildDebAndRpm(dir, version, goarch)...)
			}
			if *copyAs != "" {
				for _, artifact := range artifacts {
					run("ln", artifact, strings.Replace(artifact, "-"+version, "-"+*copyAs, 1))
				}
			}
		}
		// tidy up
		run("rm", "-rf", dir)
	}
	log.Printf("Done compiling %s/%s", goos, goarch)
	return true
}

func compile(version string) {
	start := time.Now()
	wg := new(sync.WaitGroup)
	run := make(chan func(), *parallel)
	for i := 0; i < *parallel; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for f := range run {
				f()
			}
		}()
	}
	includeRe, err := regexp.Compile(*include)
	if err != nil {
		log.Fatalf("Bad -include regexp: %v", err)
	}
	excludeRe, err := regexp.Compile(*exclude)
	if err != nil {
		log.Fatalf("Bad -exclude regexp: %v", err)
	}
	compiled := 0
	var failuresMu sync.Mutex
	var failures []string
	for _, osarch := range osarches {
		if excludeRe.MatchString(osarch) || !includeRe.MatchString(osarch) {
			continue
		}
		parts := strings.Split(osarch, "/")
		if len(parts) != 2 {
			log.Fatalf("Bad osarch %q", osarch)
		}
		goos, goarch := parts[0], parts[1]
		userGoos := goos
		if goos == "darwin" {
			userGoos = "osx"
		}
		dir := filepath.Join("rclone-" + version + "-" + userGoos + "-" + goarch)
		run <- func() {
			if !compileArch(version, goos, goarch, dir) {
				failuresMu.Lock()
				failures = append(failures, goos+"/"+goarch)
				failuresMu.Unlock()
			}
		}
		compiled++
	}
	close(run)
	wg.Wait()
	log.Printf("Compiled %d arches in %v", compiled, time.Since(start))
	if len(failures) > 0 {
		sort.Strings(failures)
		log.Printf("%d compile failures:\n  %s\n", len(failures), strings.Join(failures, "\n  "))
		os.Exit(1)
	}
}

func main() {
	flag.Parse()
	args := flag.Args()
	if len(args) != 1 {
		log.Fatalf("Syntax: %s <version>", os.Args[0])
	}
	version := args[0]
	if !*noClean {
		run("rm", "-rf", "build")
		run("mkdir", "build")
	}
	chdir("build")
	err := ioutil.WriteFile("version.txt", []byte(fmt.Sprintf("rclone %s\n", version)), 0666)
	if err != nil {
		log.Fatalf("Couldn't write version.txt: %v", err)
	}
	compile(version)
}
<|endoftext|>


<commit_before>package mount

import (
	"context"
	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/mount/meta_cache"
	"github.com/chrislusf/seaweedfs/weed/pb"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/pb/mount_pb"
	"github.com/chrislusf/seaweedfs/weed/storage/types"
	"github.com/chrislusf/seaweedfs/weed/util"
	"github.com/chrislusf/seaweedfs/weed/util/chunk_cache"
	"github.com/chrislusf/seaweedfs/weed/util/grace"
	"github.com/chrislusf/seaweedfs/weed/wdclient"
	"github.com/hanwen/go-fuse/v2/fuse"
	"google.golang.org/grpc"
	"math/rand"
	"os"
	"path"
	"path/filepath"
	"time"

	"github.com/hanwen/go-fuse/v2/fs"
)

type Option struct {
	MountDirectory     string
	FilerAddresses     []pb.ServerAddress
	filerIndex         int
	GrpcDialOption     grpc.DialOption
	FilerMountRootPath string
	Collection         string
	Replication        string
	TtlSec             int32
	DiskType           types.DiskType
	ChunkSizeLimit     int64
	ConcurrentWriters  int
	CacheDir           string
	CacheSizeMB        int64
	DataCenter         string
	Umask              os.FileMode
	Quota              int64
	DisableXAttr       bool

	MountUid         uint32
	MountGid         uint32
	MountMode        os.FileMode
	MountCtime       time.Time
	MountMtime       time.Time
	MountParentInode uint64

	VolumeServerAccess string // how to access volume servers
	Cipher             bool   // whether encrypt data on volume server
	UidGidMapper       *meta_cache.UidGidMapper

	uniqueCacheDir         string
	uniqueCacheTempPageDir string
}

type WFS struct {
	// https://dl.acm.org/doi/fullHtml/10.1145/3310148
	// follow https://github.com/hanwen/go-fuse/blob/master/fuse/api.go
	fuse.RawFileSystem
	mount_pb.UnimplementedSeaweedMountServer
	fs.Inode
	option            *Option
	metaCache         *meta_cache.MetaCache
	stats             statsCache
	chunkCache        *chunk_cache.TieredChunkCache
	signature         int32
	concurrentWriters *util.LimitedConcurrentExecutor
	inodeToPath       *InodeToPath
	fhmap             *FileHandleToInode
	dhmap             *DirectoryHandleToInode
	fuseServer        *fuse.Server
	IsOverQuota       bool
}

func NewSeaweedFileSystem(option *Option) *WFS {
	wfs := &WFS{
		RawFileSystem: fuse.NewDefaultRawFileSystem(),
		option:        option,
		signature:     util.RandomInt32(),
		inodeToPath:   NewInodeToPath(util.FullPath(option.FilerMountRootPath)),
		fhmap:         NewFileHandleToInode(),
		dhmap:         NewDirectoryHandleToInode(),
	}

	wfs.option.filerIndex = rand.Intn(len(option.FilerAddresses))
	wfs.option.setupUniqueCacheDirectory()
	if option.CacheSizeMB > 0 {
		wfs.chunkCache = chunk_cache.NewTieredChunkCache(256, option.getUniqueCacheDir(), option.CacheSizeMB, 1024*1024)
	}

	wfs.metaCache = meta_cache.NewMetaCache(path.Join(option.getUniqueCacheDir(), "meta"), option.UidGidMapper,
		util.FullPath(option.FilerMountRootPath),
		func(path util.FullPath) {
			wfs.inodeToPath.MarkChildrenCached(path)
		}, func(path util.FullPath) bool {
			return wfs.inodeToPath.IsChildrenCached(path)
		}, func(filePath util.FullPath, entry *filer_pb.Entry) {
		})
	grace.OnInterrupt(func() {
		wfs.metaCache.Shutdown()
		os.RemoveAll(option.getUniqueCacheDir())
	})

	if wfs.option.ConcurrentWriters > 0 {
		wfs.concurrentWriters = util.NewLimitedConcurrentExecutor(wfs.option.ConcurrentWriters)
	}
	return wfs
}

func (wfs *WFS) StartBackgroundTasks() {
	startTime := time.Now()
	go meta_cache.SubscribeMetaEvents(wfs.metaCache, wfs.signature, wfs, wfs.option.FilerMountRootPath, startTime.UnixNano())
	go wfs.loopCheckQuota()
}

func (wfs *WFS) String() string {
	return "seaweedfs"
}

func (wfs *WFS) Init(server *fuse.Server) {
	wfs.fuseServer = server
}

func (wfs *WFS) maybeReadEntry(inode uint64) (path util.FullPath, fh *FileHandle, entry *filer_pb.Entry, status fuse.Status) {
	path, status = wfs.inodeToPath.GetPath(inode)
	if status != fuse.OK {
		return
	}
	var found bool
	if fh, found = wfs.fhmap.FindFileHandle(inode); found {
		entry = fh.GetEntry()
		if entry != nil && fh.entry.Attributes == nil {
			entry.Attributes = &filer_pb.FuseAttributes{}
		}
		return path, fh, entry, fuse.OK
	}
	entry, status = wfs.maybeLoadEntry(path)
	if status == fuse.OK && entry.FileMode()&os.ModeSymlink != 0 {
		target := filepath.Join(string(path), "../"+entry.Attributes.SymlinkTarget)
		entry, status = wfs.maybeLoadEntry(util.FullPath(target))
	}
	return
}

func (wfs *WFS) maybeLoadEntry(fullpath util.FullPath) (*filer_pb.Entry, fuse.Status) {

	// glog.V(3).Infof("read entry cache miss %s", fullpath)
	dir, name := fullpath.DirAndName()

	// return a valid entry for the mount root
	if string(fullpath) == wfs.option.FilerMountRootPath {
		return &filer_pb.Entry{
			Name:        name,
			IsDirectory: true,
			Attributes: &filer_pb.FuseAttributes{
				Mtime:    wfs.option.MountMtime.Unix(),
				FileMode: uint32(wfs.option.MountMode),
				Uid:      wfs.option.MountUid,
				Gid:      wfs.option.MountGid,
				Crtime:   wfs.option.MountCtime.Unix(),
			},
		}, fuse.OK
	}

	// read from async meta cache
	meta_cache.EnsureVisited(wfs.metaCache, wfs, util.FullPath(dir))
	cachedEntry, cacheErr := wfs.metaCache.FindEntry(context.Background(), fullpath)
	if cacheErr == filer_pb.ErrNotFound {
		return nil, fuse.ENOENT
	}
	return cachedEntry.ToProtoEntry(), fuse.OK
}

func (wfs *WFS) LookupFn() wdclient.LookupFileIdFunctionType {
	if wfs.option.VolumeServerAccess == "filerProxy" {
		return func(fileId string) (targetUrls []string, err error) {
			return []string{"http://" + wfs.getCurrentFiler().ToHttpAddress() + "/?proxyChunkId=" + fileId}, nil
		}
	}
	return filer.LookupFn(wfs)
}

func (wfs *WFS) getCurrentFiler() pb.ServerAddress {
	return wfs.option.FilerAddresses[wfs.option.filerIndex]
}

func (option *Option) setupUniqueCacheDirectory() {
	cacheUniqueId := util.Md5String([]byte(option.MountDirectory + string(option.FilerAddresses[0]) + option.FilerMountRootPath + util.Version()))[0:8]
	option.uniqueCacheDir = path.Join(option.CacheDir, cacheUniqueId)
	option.uniqueCacheTempPageDir = filepath.Join(option.uniqueCacheDir, "swap")
	os.MkdirAll(option.uniqueCacheTempPageDir, os.FileMode(0777)&^option.Umask)
}

func (option *Option) getTempFilePageDir() string {
	return option.uniqueCacheTempPageDir
}

func (option *Option) getUniqueCacheDir() string {
	return option.uniqueCacheDir
}
<commit_msg>set status to fuse.OK<commit_after>package mount

import (
	"context"
	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/mount/meta_cache"
	"github.com/chrislusf/seaweedfs/weed/pb"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/pb/mount_pb"
	"github.com/chrislusf/seaweedfs/weed/storage/types"
	"github.com/chrislusf/seaweedfs/weed/util"
	"github.com/chrislusf/seaweedfs/weed/util/chunk_cache"
	"github.com/chrislusf/seaweedfs/weed/util/grace"
	"github.com/chrislusf/seaweedfs/weed/wdclient"
	"github.com/hanwen/go-fuse/v2/fuse"
	"google.golang.org/grpc"
	"math/rand"
	"os"
	"path"
	"path/filepath"
	"time"

	"github.com/hanwen/go-fuse/v2/fs"
)

type Option struct {
	MountDirectory     string
	FilerAddresses     []pb.ServerAddress
	filerIndex         int
	GrpcDialOption     grpc.DialOption
	FilerMountRootPath string
	Collection         string
	Replication        string
	TtlSec             int32
	DiskType           types.DiskType
	ChunkSizeLimit     int64
	ConcurrentWriters  int
	CacheDir           string
	CacheSizeMB        int64
	DataCenter         string
	Umask              os.FileMode
	Quota              int64
	DisableXAttr       bool

	MountUid         uint32
	MountGid         uint32
	MountMode        os.FileMode
	MountCtime       time.Time
	MountMtime       time.Time
	MountParentInode uint64

	VolumeServerAccess string // how to access volume servers
	Cipher             bool   // whether encrypt data on volume server
	UidGidMapper       *meta_cache.UidGidMapper

	uniqueCacheDir         string
	uniqueCacheTempPageDir string
}

type WFS struct {
	// https://dl.acm.org/doi/fullHtml/10.1145/3310148
	// follow https://github.com/hanwen/go-fuse/blob/master/fuse/api.go
	fuse.RawFileSystem
	mount_pb.UnimplementedSeaweedMountServer
	fs.Inode
	option            *Option
	metaCache         *meta_cache.MetaCache
	stats             statsCache
	chunkCache        *chunk_cache.TieredChunkCache
	signature         int32
	concurrentWriters *util.LimitedConcurrentExecutor
	inodeToPath       *InodeToPath
	fhmap             *FileHandleToInode
	dhmap             *DirectoryHandleToInode
	fuseServer        *fuse.Server
	IsOverQuota       bool
}

func NewSeaweedFileSystem(option *Option) *WFS {
	wfs := &WFS{
		RawFileSystem: fuse.NewDefaultRawFileSystem(),
		option:        option,
		signature:     util.RandomInt32(),
		inodeToPath:   NewInodeToPath(util.FullPath(option.FilerMountRootPath)),
		fhmap:         NewFileHandleToInode(),
		dhmap:         NewDirectoryHandleToInode(),
	}

	wfs.option.filerIndex = rand.Intn(len(option.FilerAddresses))
	wfs.option.setupUniqueCacheDirectory()
	if option.CacheSizeMB > 0 {
		wfs.chunkCache = chunk_cache.NewTieredChunkCache(256, option.getUniqueCacheDir(), option.CacheSizeMB, 1024*1024)
	}

	wfs.metaCache = meta_cache.NewMetaCache(path.Join(option.getUniqueCacheDir(), "meta"), option.UidGidMapper,
		util.FullPath(option.FilerMountRootPath),
		func(path util.FullPath) {
			wfs.inodeToPath.MarkChildrenCached(path)
		}, func(path util.FullPath) bool {
			return wfs.inodeToPath.IsChildrenCached(path)
		}, func(filePath util.FullPath, entry *filer_pb.Entry) {
		})
	grace.OnInterrupt(func() {
		wfs.metaCache.Shutdown()
		os.RemoveAll(option.getUniqueCacheDir())
	})

	if wfs.option.ConcurrentWriters > 0 {
		wfs.concurrentWriters = util.NewLimitedConcurrentExecutor(wfs.option.ConcurrentWriters)
	}
	return wfs
}

func (wfs *WFS) StartBackgroundTasks() {
	startTime := time.Now()
	go meta_cache.SubscribeMetaEvents(wfs.metaCache, wfs.signature, wfs, wfs.option.FilerMountRootPath, startTime.UnixNano())
	go wfs.loopCheckQuota()
}

func (wfs *WFS) String() string {
	return "seaweedfs"
}

func (wfs *WFS) Init(server *fuse.Server) {
	wfs.fuseServer = server
}

func (wfs *WFS) maybeReadEntry(inode uint64) (path util.FullPath, fh *FileHandle, entry *filer_pb.Entry, status fuse.Status) {
	path, status = wfs.inodeToPath.GetPath(inode)
	if status != fuse.OK {
		return
	}
	var found bool
	if fh, found = wfs.fhmap.FindFileHandle(inode); found {
		entry = fh.GetEntry()
		if entry != nil && fh.entry.Attributes == nil {
			entry.Attributes = &filer_pb.FuseAttributes{}
		}
		status = fuse.OK
	} else {
		entry, status = wfs.maybeLoadEntry(path)
	}
	if status == fuse.OK && entry.FileMode()&os.ModeSymlink != 0 {
		target := filepath.Join(string(path), "../"+entry.Attributes.SymlinkTarget)
		entry, status = wfs.maybeLoadEntry(util.FullPath(target))
	}
	return
}

func (wfs *WFS) maybeLoadEntry(fullpath util.FullPath) (*filer_pb.Entry, fuse.Status) {

	// glog.V(3).Infof("read entry cache miss %s", fullpath)
	dir, name := fullpath.DirAndName()

	// return a valid entry for the mount root
	if string(fullpath) == wfs.option.FilerMountRootPath {
		return &filer_pb.Entry{
			Name:        name,
			IsDirectory: true,
			Attributes: &filer_pb.FuseAttributes{
				Mtime:    wfs.option.MountMtime.Unix(),
				FileMode: uint32(wfs.option.MountMode),
				Uid:      wfs.option.MountUid,
				Gid:      wfs.option.MountGid,
				Crtime:   wfs.option.MountCtime.Unix(),
			},
		}, fuse.OK
	}

	// read from async meta cache
	meta_cache.EnsureVisited(wfs.metaCache, wfs, util.FullPath(dir))
	cachedEntry, cacheErr := wfs.metaCache.FindEntry(context.Background(), fullpath)
	if cacheErr == filer_pb.ErrNotFound {
		return nil, fuse.ENOENT
	}
	return cachedEntry.ToProtoEntry(), fuse.OK
}

func (wfs *WFS) LookupFn() wdclient.LookupFileIdFunctionType {
	if wfs.option.VolumeServerAccess == "filerProxy" {
		return func(fileId string) (targetUrls []string, err error) {
			return []string{"http://" + wfs.getCurrentFiler().ToHttpAddress() + "/?proxyChunkId=" + fileId}, nil
		}
	}
	return filer.LookupFn(wfs)
}

func (wfs *WFS) getCurrentFiler() pb.ServerAddress {
	return wfs.option.FilerAddresses[wfs.option.filerIndex]
}

func (option *Option) setupUniqueCacheDirectory() {
	cacheUniqueId := util.Md5String([]byte(option.MountDirectory + string(option.FilerAddresses[0]) + option.FilerMountRootPath + util.Version()))[0:8]
	option.uniqueCacheDir = path.Join(option.CacheDir, cacheUniqueId)
	option.uniqueCacheTempPageDir = filepath.Join(option.uniqueCacheDir, "swap")
	os.MkdirAll(option.uniqueCacheTempPageDir, os.FileMode(0777)&^option.Umask)
}

func (option *Option) getTempFilePageDir() string {
	return option.uniqueCacheTempPageDir
}

func (option *Option) getUniqueCacheDir() string {
	return option.uniqueCacheDir
}
<|endoftext|>