<commit_before>package dispatcher

import (
	"fmt"

	"github.com/docker/swarmkit/api"
	"github.com/docker/swarmkit/api/equality"
	"github.com/docker/swarmkit/api/validation"
	"github.com/docker/swarmkit/identity"
	"github.com/docker/swarmkit/manager/drivers"
	"github.com/docker/swarmkit/manager/state/store"
	"github.com/sirupsen/logrus"
)

type typeAndID struct {
	id      string
	objType api.ResourceType
}

type assignmentSet struct {
	dp                   *drivers.DriverProvider
	tasksMap             map[string]*api.Task
	tasksUsingDependency map[typeAndID]map[string]struct{}
	changes              map[typeAndID]*api.AssignmentChange
	log                  *logrus.Entry
}

func newAssignmentSet(log *logrus.Entry, dp *drivers.DriverProvider) *assignmentSet {
	return &assignmentSet{
		dp:                   dp,
		changes:              make(map[typeAndID]*api.AssignmentChange),
		tasksMap:             make(map[string]*api.Task),
		tasksUsingDependency: make(map[typeAndID]map[string]struct{}),
		log:                  log,
	}
}

func assignSecret(a *assignmentSet, readTx store.ReadTx, mapKey typeAndID, t *api.Task) {
	if _, exists := a.tasksUsingDependency[mapKey]; !exists {
		a.tasksUsingDependency[mapKey] = make(map[string]struct{})
	}
	secret, doNotReuse, err := a.secret(readTx, t, mapKey.id)
	if err != nil {
		a.log.WithFields(logrus.Fields{
			"resource.type": "secret",
			"secret.id":     mapKey.id,
			"error":         err,
		}).Debug("failed to fetch secret")
		return
	}
	// If the secret should not be reused for other tasks, give it a unique ID for the task to allow different values for different tasks.
	if doNotReuse {
		// Give the secret a new ID and mark it as internal
		originalSecretID := secret.ID
		taskSpecificID := identity.CombineTwoIDs(originalSecretID, t.ID)
		secret.ID = taskSpecificID
		secret.Internal = true
		// Create a new mapKey with the new ID and insert it into the dependencies map for the task.
		// This will make the changes map contain an entry with the new ID rather than the original one.
		mapKey = typeAndID{objType: mapKey.objType, id: secret.ID}
		a.tasksUsingDependency[mapKey] = make(map[string]struct{})
		a.tasksUsingDependency[mapKey][t.ID] = struct{}{}
	}
	a.changes[mapKey] = &api.AssignmentChange{
		Assignment: &api.Assignment{
			Item: &api.Assignment_Secret{
				Secret: secret,
			},
		},
		Action: api.AssignmentChange_AssignmentActionUpdate,
	}
}

func assignConfig(a *assignmentSet, readTx store.ReadTx, mapKey typeAndID) {
	a.tasksUsingDependency[mapKey] = make(map[string]struct{})
	config := store.GetConfig(readTx, mapKey.id)
	if config == nil {
		a.log.WithFields(logrus.Fields{
			"resource.type": "config",
			"config.id":     mapKey.id,
		}).Debug("config not found")
		return
	}
	a.changes[mapKey] = &api.AssignmentChange{
		Assignment: &api.Assignment{
			Item: &api.Assignment_Config{
				Config: config,
			},
		},
		Action: api.AssignmentChange_AssignmentActionUpdate,
	}
}

func (a *assignmentSet) addTaskDependencies(readTx store.ReadTx, t *api.Task) {
	for _, resourceRef := range t.Spec.ResourceReferences {
		mapKey := typeAndID{objType: resourceRef.ResourceType, id: resourceRef.ResourceID}
		if len(a.tasksUsingDependency[mapKey]) == 0 {
			switch resourceRef.ResourceType {
			case api.ResourceType_SECRET:
				assignSecret(a, readTx, mapKey, t)
			case api.ResourceType_CONFIG:
				assignConfig(a, readTx, mapKey)
			default:
				a.log.WithField(
					"resource.type", resourceRef.ResourceType,
				).Debug("invalid resource type for a task dependency, skipping")
				continue
			}
		}
		a.tasksUsingDependency[mapKey][t.ID] = struct{}{}
	}

	var secrets []*api.SecretReference
	container := t.Spec.GetContainer()
	if container != nil {
		secrets = container.Secrets
	}

	for _, secretRef := range secrets {
		secretID := secretRef.SecretID
		mapKey := typeAndID{objType: api.ResourceType_SECRET, id: secretID}

		if _, exists := a.tasksUsingDependency[mapKey][t.ID]; !exists {
			assignSecret(a, readTx, mapKey, t)
		}
		a.tasksUsingDependency[mapKey][t.ID] = struct{}{}
	}

	var configs []*api.ConfigReference
	if container != nil {
		configs = container.Configs
	}
	for _, configRef := range configs {
		configID := configRef.ConfigID
		mapKey := typeAndID{objType: api.ResourceType_CONFIG, id: configID}

		if len(a.tasksUsingDependency[mapKey]) == 0 {
			assignConfig(a, readTx, mapKey)
		}
		a.tasksUsingDependency[mapKey][t.ID] = struct{}{}
	}
}

func (a *assignmentSet) releaseDependency(mapKey typeAndID, assignment *api.Assignment, taskID string) bool {
	delete(a.tasksUsingDependency[mapKey], taskID)
	if len(a.tasksUsingDependency[mapKey]) != 0 {
		return false
	}
	// No tasks are using the dependency anymore
	delete(a.tasksUsingDependency, mapKey)
	a.changes[mapKey] = &api.AssignmentChange{
		Assignment: assignment,
		Action:     api.AssignmentChange_AssignmentActionRemove,
	}
	return true
}

func (a *assignmentSet) releaseTaskDependencies(t *api.Task) bool {
	var modified bool

	for _, resourceRef := range t.Spec.ResourceReferences {
		var assignment *api.Assignment
		switch resourceRef.ResourceType {
		case api.ResourceType_SECRET:
			assignment = &api.Assignment{
				Item: &api.Assignment_Secret{
					Secret: &api.Secret{ID: resourceRef.ResourceID},
				},
			}
		case api.ResourceType_CONFIG:
			assignment = &api.Assignment{
				Item: &api.Assignment_Config{
					Config: &api.Config{ID: resourceRef.ResourceID},
				},
			}
		default:
			a.log.WithField(
				"resource.type", resourceRef.ResourceType,
			).Debug("invalid resource type for a task dependency, skipping")
			continue
		}

		mapKey := typeAndID{objType: resourceRef.ResourceType, id: resourceRef.ResourceID}
		if a.releaseDependency(mapKey, assignment, t.ID) {
			modified = true
		}
	}

	container := t.Spec.GetContainer()

	var secrets []*api.SecretReference
	if container != nil {
		secrets = container.Secrets
	}

	for _, secretRef := range secrets {
		secretID := secretRef.SecretID
		mapKey := typeAndID{objType: api.ResourceType_SECRET, id: secretID}
		assignment := &api.Assignment{
			Item: &api.Assignment_Secret{
				Secret: &api.Secret{ID: secretID},
			},
		}
		if a.releaseDependency(mapKey, assignment, t.ID) {
			modified = true
		}
	}

	var configs []*api.ConfigReference
	if container != nil {
		configs = container.Configs
	}

	for _, configRef := range configs {
		configID := configRef.ConfigID
		mapKey := typeAndID{objType: api.ResourceType_CONFIG, id: configID}
		assignment := &api.Assignment{
			Item: &api.Assignment_Config{
				Config: &api.Config{ID: configID},
			},
		}
		if a.releaseDependency(mapKey, assignment, t.ID) {
			modified = true
		}
	}

	return modified
}

func (a *assignmentSet) addOrUpdateTask(readTx store.ReadTx, t *api.Task) bool {
	// We only care about tasks that are ASSIGNED or higher.
	if t.Status.State < api.TaskStateAssigned {
		return false
	}

	if oldTask, exists := a.tasksMap[t.ID]; exists {
		// States ASSIGNED and below are set by the orchestrator/scheduler,
		// not the agent, so tasks in these states need to be sent to the
		// agent even if nothing else has changed.
		if equality.TasksEqualStable(oldTask, t) && t.Status.State > api.TaskStateAssigned {
			// this update should not trigger a task change for the agent
			a.tasksMap[t.ID] = t
			// If this task got updated to a final state, let's release
			// the dependencies that are being used by the task
			if t.Status.State > api.TaskStateRunning {
				// If releasing the dependencies caused us to
				// remove something from the assignment set,
				// mark one modification.
				return a.releaseTaskDependencies(t)
			}
			return false
		}
	} else if t.Status.State <= api.TaskStateRunning {
		// If this task wasn't part of the assignment set before, and it's <= RUNNING
		// add the dependencies it references to the assignment.
		// Task states > RUNNING are worker reported only, are never created in
		// a > RUNNING state.
		a.addTaskDependencies(readTx, t)
	}
	a.tasksMap[t.ID] = t
	a.changes[typeAndID{objType: api.ResourceType_TASK, id: t.ID}] = &api.AssignmentChange{
		Assignment: &api.Assignment{
			Item: &api.Assignment_Task{
				Task: t,
			},
		},
		Action: api.AssignmentChange_AssignmentActionUpdate,
	}
	return true
}

func (a *assignmentSet) removeTask(t *api.Task) bool {
	if _, exists := a.tasksMap[t.ID]; !exists {
		return false
	}

	a.changes[typeAndID{objType: api.ResourceType_TASK, id: t.ID}] = &api.AssignmentChange{
		Assignment: &api.Assignment{
			Item: &api.Assignment_Task{
				Task: &api.Task{ID: t.ID},
			},
		},
		Action: api.AssignmentChange_AssignmentActionRemove,
	}

	delete(a.tasksMap, t.ID)

	// Release the dependencies being used by this task.
	// Ignoring the return here. We will always mark this as a
	// modification, since a task is being removed.
	a.releaseTaskDependencies(t)
	return true
}

func (a *assignmentSet) message() api.AssignmentsMessage {
	var message api.AssignmentsMessage
	for _, change := range a.changes {
		message.Changes = append(message.Changes, change)
	}

	// The the set of changes is reinitialized to prepare for formation
	// of the next message.
	a.changes = make(map[typeAndID]*api.AssignmentChange)

	return message
}

// secret populates the secret value from raft store. For external secrets, the value is populated
// from the secret driver. The function returns: a secret object; an indication of whether the value
// is to be reused across tasks; and an error if the secret is not found in the store, if the secret
// driver responds with one or if the payload does not pass validation.
func (a *assignmentSet) secret(readTx store.ReadTx, task *api.Task, secretID string) (*api.Secret, bool, error) {
	secret := store.GetSecret(readTx, secretID)
	if secret == nil {
		return nil, false, fmt.Errorf("secret not found")
	}
	if secret.Spec.Driver == nil {
		return secret, false, nil
	}
	d, err := a.dp.NewSecretDriver(secret.Spec.Driver)
	if err != nil {
		return nil, false, err
	}
	value, doNotReuse, err := d.Get(&secret.Spec, task)
	if err != nil {
		return nil, false, err
	}
	if err := validation.ValidateSecretPayload(value); err != nil {
		return nil, false, err
	}
	// Assign the secret
	secret.Spec.Data = value
	return secret, doNotReuse, nil
}
<commit_msg>Add comment on difference in dependency map check<commit_after>package dispatcher

import (
	"fmt"

	"github.com/docker/swarmkit/api"
	"github.com/docker/swarmkit/api/equality"
	"github.com/docker/swarmkit/api/validation"
	"github.com/docker/swarmkit/identity"
	"github.com/docker/swarmkit/manager/drivers"
	"github.com/docker/swarmkit/manager/state/store"
	"github.com/sirupsen/logrus"
)

type typeAndID struct {
	id      string
	objType api.ResourceType
}

type assignmentSet struct {
	dp                   *drivers.DriverProvider
	tasksMap             map[string]*api.Task
	tasksUsingDependency map[typeAndID]map[string]struct{}
	changes              map[typeAndID]*api.AssignmentChange
	log                  *logrus.Entry
}

func newAssignmentSet(log *logrus.Entry, dp *drivers.DriverProvider) *assignmentSet {
	return &assignmentSet{
		dp:                   dp,
		changes:              make(map[typeAndID]*api.AssignmentChange),
		tasksMap:             make(map[string]*api.Task),
		tasksUsingDependency: make(map[typeAndID]map[string]struct{}),
		log:                  log,
	}
}

func assignSecret(a *assignmentSet, readTx store.ReadTx, mapKey typeAndID, t *api.Task) {
	if _, exists := a.tasksUsingDependency[mapKey]; !exists {
		a.tasksUsingDependency[mapKey] = make(map[string]struct{})
	}
	secret, doNotReuse, err := a.secret(readTx, t, mapKey.id)
	if err != nil {
		a.log.WithFields(logrus.Fields{
			"resource.type": "secret",
			"secret.id":     mapKey.id,
			"error":         err,
		}).Debug("failed to fetch secret")
		return
	}
	// If the secret should not be reused for other tasks, give it a unique ID for the task to allow different values for different tasks.
	if doNotReuse {
		// Give the secret a new ID and mark it as internal
		originalSecretID := secret.ID
		taskSpecificID := identity.CombineTwoIDs(originalSecretID, t.ID)
		secret.ID = taskSpecificID
		secret.Internal = true
		// Create a new mapKey with the new ID and insert it into the dependencies map for the task.
		// This will make the changes map contain an entry with the new ID rather than the original one.
		mapKey = typeAndID{objType: mapKey.objType, id: secret.ID}
		a.tasksUsingDependency[mapKey] = make(map[string]struct{})
		a.tasksUsingDependency[mapKey][t.ID] = struct{}{}
	}
	a.changes[mapKey] = &api.AssignmentChange{
		Assignment: &api.Assignment{
			Item: &api.Assignment_Secret{
				Secret: secret,
			},
		},
		Action: api.AssignmentChange_AssignmentActionUpdate,
	}
}

func assignConfig(a *assignmentSet, readTx store.ReadTx, mapKey typeAndID) {
	a.tasksUsingDependency[mapKey] = make(map[string]struct{})
	config := store.GetConfig(readTx, mapKey.id)
	if config == nil {
		a.log.WithFields(logrus.Fields{
			"resource.type": "config",
			"config.id":     mapKey.id,
		}).Debug("config not found")
		return
	}
	a.changes[mapKey] = &api.AssignmentChange{
		Assignment: &api.Assignment{
			Item: &api.Assignment_Config{
				Config: config,
			},
		},
		Action: api.AssignmentChange_AssignmentActionUpdate,
	}
}

func (a *assignmentSet) addTaskDependencies(readTx store.ReadTx, t *api.Task) {
	for _, resourceRef := range t.Spec.ResourceReferences {
		mapKey := typeAndID{objType: resourceRef.ResourceType, id: resourceRef.ResourceID}
		if len(a.tasksUsingDependency[mapKey]) == 0 {
			switch resourceRef.ResourceType {
			case api.ResourceType_SECRET:
				assignSecret(a, readTx, mapKey, t)
			case api.ResourceType_CONFIG:
				assignConfig(a, readTx, mapKey)
			default:
				a.log.WithField(
					"resource.type", resourceRef.ResourceType,
				).Debug("invalid resource type for a task dependency, skipping")
				continue
			}
		}
		a.tasksUsingDependency[mapKey][t.ID] = struct{}{}
	}

	var secrets []*api.SecretReference
	container := t.Spec.GetContainer()
	if container != nil {
		secrets = container.Secrets
	}

	for _, secretRef := range secrets {
		secretID := secretRef.SecretID
		mapKey := typeAndID{objType: api.ResourceType_SECRET, id: secretID}

		// This checks for the presence of each task in the dependency map for the
		// secret. This is currently only done for secrets since the other types of
		// dependencies do not support driver plugins. Arguably, the same task would
		// not have the same secret as a dependency more than once, but this check
		// makes sure the task only gets the secret assigned once.
		if _, exists := a.tasksUsingDependency[mapKey][t.ID]; !exists {
			assignSecret(a, readTx, mapKey, t)
		}
		a.tasksUsingDependency[mapKey][t.ID] = struct{}{}
	}

	var configs []*api.ConfigReference
	if container != nil {
		configs = container.Configs
	}
	for _, configRef := range configs {
		configID := configRef.ConfigID
		mapKey := typeAndID{objType: api.ResourceType_CONFIG, id: configID}

		if len(a.tasksUsingDependency[mapKey]) == 0 {
			assignConfig(a, readTx, mapKey)
		}
		a.tasksUsingDependency[mapKey][t.ID] = struct{}{}
	}
}

func (a *assignmentSet) releaseDependency(mapKey typeAndID, assignment *api.Assignment, taskID string) bool {
	delete(a.tasksUsingDependency[mapKey], taskID)
	if len(a.tasksUsingDependency[mapKey]) != 0 {
		return false
	}
	// No tasks are using the dependency anymore
	delete(a.tasksUsingDependency, mapKey)
	a.changes[mapKey] = &api.AssignmentChange{
		Assignment: assignment,
		Action:     api.AssignmentChange_AssignmentActionRemove,
	}
	return true
}

func (a *assignmentSet) releaseTaskDependencies(t *api.Task) bool {
	var modified bool

	for _, resourceRef := range t.Spec.ResourceReferences {
		var assignment *api.Assignment
		switch resourceRef.ResourceType {
		case api.ResourceType_SECRET:
			assignment = &api.Assignment{
				Item: &api.Assignment_Secret{
					Secret: &api.Secret{ID: resourceRef.ResourceID},
				},
			}
		case api.ResourceType_CONFIG:
			assignment = &api.Assignment{
				Item: &api.Assignment_Config{
					Config: &api.Config{ID: resourceRef.ResourceID},
				},
			}
		default:
			a.log.WithField(
				"resource.type", resourceRef.ResourceType,
			).Debug("invalid resource type for a task dependency, skipping")
			continue
		}

		mapKey := typeAndID{objType: resourceRef.ResourceType, id: resourceRef.ResourceID}
		if a.releaseDependency(mapKey, assignment, t.ID) {
			modified = true
		}
	}

	container := t.Spec.GetContainer()

	var secrets []*api.SecretReference
	if container != nil {
		secrets = container.Secrets
	}

	for _, secretRef := range secrets {
		secretID := secretRef.SecretID
		mapKey := typeAndID{objType: api.ResourceType_SECRET, id: secretID}
		assignment := &api.Assignment{
			Item: &api.Assignment_Secret{
				Secret: &api.Secret{ID: secretID},
			},
		}
		if a.releaseDependency(mapKey, assignment, t.ID) {
			modified = true
		}
	}

	var configs []*api.ConfigReference
	if container != nil {
		configs = container.Configs
	}

	for _, configRef := range configs {
		configID := configRef.ConfigID
		mapKey := typeAndID{objType: api.ResourceType_CONFIG, id: configID}
		assignment := &api.Assignment{
			Item: &api.Assignment_Config{
				Config: &api.Config{ID: configID},
			},
		}
		if a.releaseDependency(mapKey, assignment, t.ID) {
			modified = true
		}
	}

	return modified
}

func (a *assignmentSet) addOrUpdateTask(readTx store.ReadTx, t *api.Task) bool {
	// We only care about tasks that are ASSIGNED or higher.
	if t.Status.State < api.TaskStateAssigned {
		return false
	}

	if oldTask, exists := a.tasksMap[t.ID]; exists {
		// States ASSIGNED and below are set by the orchestrator/scheduler,
		// not the agent, so tasks in these states need to be sent to the
		// agent even if nothing else has changed.
		if equality.TasksEqualStable(oldTask, t) && t.Status.State > api.TaskStateAssigned {
			// this update should not trigger a task change for the agent
			a.tasksMap[t.ID] = t
			// If this task got updated to a final state, let's release
			// the dependencies that are being used by the task
			if t.Status.State > api.TaskStateRunning {
				// If releasing the dependencies caused us to
				// remove something from the assignment set,
				// mark one modification.
				return a.releaseTaskDependencies(t)
			}
			return false
		}
	} else if t.Status.State <= api.TaskStateRunning {
		// If this task wasn't part of the assignment set before, and it's <= RUNNING
		// add the dependencies it references to the assignment.
		// Task states > RUNNING are worker reported only, are never created in
		// a > RUNNING state.
		a.addTaskDependencies(readTx, t)
	}
	a.tasksMap[t.ID] = t
	a.changes[typeAndID{objType: api.ResourceType_TASK, id: t.ID}] = &api.AssignmentChange{
		Assignment: &api.Assignment{
			Item: &api.Assignment_Task{
				Task: t,
			},
		},
		Action: api.AssignmentChange_AssignmentActionUpdate,
	}
	return true
}

func (a *assignmentSet) removeTask(t *api.Task) bool {
	if _, exists := a.tasksMap[t.ID]; !exists {
		return false
	}

	a.changes[typeAndID{objType: api.ResourceType_TASK, id: t.ID}] = &api.AssignmentChange{
		Assignment: &api.Assignment{
			Item: &api.Assignment_Task{
				Task: &api.Task{ID: t.ID},
			},
		},
		Action: api.AssignmentChange_AssignmentActionRemove,
	}

	delete(a.tasksMap, t.ID)

	// Release the dependencies being used by this task.
	// Ignoring the return here. We will always mark this as a
	// modification, since a task is being removed.
	a.releaseTaskDependencies(t)
	return true
}

func (a *assignmentSet) message() api.AssignmentsMessage {
	var message api.AssignmentsMessage
	for _, change := range a.changes {
		message.Changes = append(message.Changes, change)
	}

	// The the set of changes is reinitialized to prepare for formation
	// of the next message.
	a.changes = make(map[typeAndID]*api.AssignmentChange)

	return message
}

// secret populates the secret value from raft store. For external secrets, the value is populated
// from the secret driver. The function returns: a secret object; an indication of whether the value
// is to be reused across tasks; and an error if the secret is not found in the store, if the secret
// driver responds with one or if the payload does not pass validation.
func (a *assignmentSet) secret(readTx store.ReadTx, task *api.Task, secretID string) (*api.Secret, bool, error) {
	secret := store.GetSecret(readTx, secretID)
	if secret == nil {
		return nil, false, fmt.Errorf("secret not found")
	}
	if secret.Spec.Driver == nil {
		return secret, false, nil
	}
	d, err := a.dp.NewSecretDriver(secret.Spec.Driver)
	if err != nil {
		return nil, false, err
	}
	value, doNotReuse, err := d.Get(&secret.Spec, task)
	if err != nil {
		return nil, false, err
	}
	if err := validation.ValidateSecretPayload(value); err != nil {
		return nil, false, err
	}
	// Assign the secret
	secret.Spec.Data = value
	return secret, doNotReuse, nil
}
<|endoftext|>


<commit_before>// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
//
// This software (Documize Community Edition) is licensed under
// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
//
// You can operate outside the AGPL restrictions by purchasing
// Documize Enterprise Edition and obtaining a commercial license
// by contacting <sales@documize.com>.
//
// https://documize.com

package attachment

import (
	"bytes"
	"database/sql"
	"fmt"
	"io"
	"mime"
	"net/http"

	"github.com/documize/community/core/env"
	"github.com/documize/community/core/request"
	"github.com/documize/community/core/response"
	"github.com/documize/community/core/secrets"
	"github.com/documize/community/core/uniqueid"
	"github.com/documize/community/domain"
	"github.com/documize/community/domain/organization"
	"github.com/documize/community/domain/permission"
	indexer "github.com/documize/community/domain/search"
	"github.com/documize/community/model/attachment"
	"github.com/documize/community/model/audit"
	"github.com/documize/community/model/workflow"
	uuid "github.com/nu7hatch/gouuid"
)

// Handler contains the runtime information such as logging and database.
type Handler struct {
	Runtime *env.Runtime
	Store   *domain.Store
	Indexer indexer.Indexer
}

// Download is the end-point that responds to a request for a particular attachment
// by sending the requested file to the client.
func (h *Handler) Download(w http.ResponseWriter, r *http.Request) {
	method := "attachment.Download"
	ctx := domain.GetRequestContext(r)
	ctx.Subdomain = organization.GetSubdomainFromHost(r)

	a, err := h.Store.Attachment.GetAttachment(ctx, request.Param(r, "orgID"), request.Param(r, "attachmentID"))

	if err == sql.ErrNoRows {
		response.WriteNotFoundError(w, method, request.Param(r, "fileID"))
		return
	}
	if err != nil {
		h.Runtime.Log.Error("get attachment", err)
		response.WriteServerError(w, method, err)
		return
	}

	typ := mime.TypeByExtension("." + a.Extension)
	if typ == "" {
		typ = "application/octet-stream"
	}

	w.Header().Set("Content-Type", typ)
	w.Header().Set("Content-Disposition", `Attachment; filename="`+a.Filename+`" ; `+`filename*="`+a.Filename+`"`)
	w.Header().Set("Content-Length", fmt.Sprintf("%d", len(a.Data)))
	w.WriteHeader(http.StatusOK)

	_, err = w.Write(a.Data)
	if err != nil {
		h.Runtime.Log.Error("write attachment", err)
		return
	}

	h.Store.Audit.Record(ctx, audit.EventTypeAttachmentDownload)
}

// Get is an end-point that returns all of the attachments of a particular documentID.
func (h *Handler) Get(w http.ResponseWriter, r *http.Request) {
	method := "attachment.GetAttachments"
	ctx := domain.GetRequestContext(r)

	documentID := request.Param(r, "documentID")
	if len(documentID) == 0 {
		response.WriteMissingDataError(w, method, "documentID")
		return
	}

	if !permission.CanViewDocument(ctx, *h.Store, documentID) {
		response.WriteForbiddenError(w)
		return
	}

	a, err := h.Store.Attachment.GetAttachments(ctx, documentID)
	if err != nil && err != sql.ErrNoRows {
		h.Runtime.Log.Error("get attachment", err)
		response.WriteServerError(w, method, err)
		return
	}

	if len(a) == 0 {
		a = []attachment.Attachment{}
	}

	response.WriteJSON(w, a)
}

// Delete is an endpoint that deletes a particular document attachment.
func (h *Handler) Delete(w http.ResponseWriter, r *http.Request) {
	method := "attachment.DeleteAttachment"
	ctx := domain.GetRequestContext(r)

	documentID := request.Param(r, "documentID")
	if len(documentID) == 0 {
		response.WriteMissingDataError(w, method, "documentID")
		return
	}

	attachmentID := request.Param(r, "attachmentID")
	if len(attachmentID) == 0 {
		response.WriteMissingDataError(w, method, "attachmentID")
		return
	}

	if !permission.CanChangeDocument(ctx, *h.Store, documentID) {
		response.WriteForbiddenError(w)
		return
	}

	var err error
	ctx.Transaction, err = h.Runtime.Db.Beginx()
	if err != nil {
		h.Runtime.Log.Error("transaction", err)
		response.WriteServerError(w, method, err)
		return
	}

	_, err = h.Store.Attachment.Delete(ctx, attachmentID)
	if err != nil {
		ctx.Transaction.Rollback()
		response.WriteServerError(w, method, err)
		h.Runtime.Log.Error("delete attachment", err)
		return
	}

	// Mark references to this document as orphaned
	err = h.Store.Link.MarkOrphanAttachmentLink(ctx, attachmentID)
	if err != nil {
		ctx.Transaction.Rollback()
		response.WriteServerError(w, method, err)
		h.Runtime.Log.Error("delete attachment links", err)
		return
	}

	ctx.Transaction.Commit()

	h.Store.Audit.Record(ctx, audit.EventTypeAttachmentDelete)

	a, _ := h.Store.Attachment.GetAttachments(ctx, documentID)
	d, _ := h.Store.Document.Get(ctx, documentID)

	if d.Lifecycle == workflow.LifecycleLive {
		go h.Indexer.IndexDocument(ctx, d, a)
	} else {
		go h.Indexer.DeleteDocument(ctx, d.RefID)
	}

	response.WriteEmpty(w)
}

// Add stores files against a document.
func (h *Handler) Add(w http.ResponseWriter, r *http.Request) {
	method := "attachment.Add"
	ctx := domain.GetRequestContext(r)

	documentID := request.Param(r, "documentID")
	if len(documentID) == 0 {
		response.WriteMissingDataError(w, method, "documentID")
		return
	}

	if !permission.CanChangeDocument(ctx, *h.Store, documentID) {
		response.WriteForbiddenError(w)
		return
	}

	filedata, filename, err := r.FormFile("attachment")
	if err != nil {
		response.WriteMissingDataError(w, method, "attachment")
		return
	}

	b := new(bytes.Buffer)
	_, err = io.Copy(b, filedata)
	if err != nil {
		response.WriteServerError(w, method, err)
		h.Runtime.Log.Error("add attachment", err)
		return
	}

	var job = "some-uuid"
	newUUID, err := uuid.NewV4()
	if err != nil {
		h.Runtime.Log.Error("uuid", err)
		response.WriteServerError(w, method, err)
		return
	}
	job = newUUID.String()

	var a attachment.Attachment
	refID := uniqueid.Generate()
	a.RefID = refID
	a.DocumentID = documentID
	a.Job = job
	random := secrets.GenerateSalt()
	a.FileID = random[0:9]
	a.Filename = filename.Filename
	a.Data = b.Bytes()

	ctx.Transaction, err = h.Runtime.Db.Beginx()
	if err != nil {
		response.WriteServerError(w, method, err)
		h.Runtime.Log.Error("transaction", err)
		return
	}

	err = h.Store.Attachment.Add(ctx, a)
	if err != nil {
		ctx.Transaction.Rollback()
		response.WriteServerError(w, method, err)
		h.Runtime.Log.Error("add attachment", err)
		return
	}

	ctx.Transaction.Commit()

	h.Store.Audit.Record(ctx, audit.EventTypeAttachmentAdd)

	all, _ := h.Store.Attachment.GetAttachments(ctx, documentID)
	d, _ := h.Store.Document.Get(ctx, documentID)

	if d.Lifecycle == workflow.LifecycleLive {
		go h.Indexer.IndexDocument(ctx, d, all)
	} else {
		go h.Indexer.DeleteDocument(ctx, d.RefID)
	}

	response.WriteEmpty(w)
}
<commit_msg>Comment fix for attachment endpoint<commit_after>// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
//
// This software (Documize Community Edition) is licensed under
// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
//
// You can operate outside the AGPL restrictions by purchasing
// Documize Enterprise Edition and obtaining a commercial license
// by contacting <sales@documize.com>.
//
// https://documize.com

package attachment

import (
	"bytes"
	"database/sql"
	"fmt"
	"io"
	"mime"
	"net/http"

	"github.com/documize/community/core/env"
	"github.com/documize/community/core/request"
	"github.com/documize/community/core/response"
	"github.com/documize/community/core/secrets"
	"github.com/documize/community/core/uniqueid"
	"github.com/documize/community/domain"
	"github.com/documize/community/domain/organization"
	"github.com/documize/community/domain/permission"
	indexer "github.com/documize/community/domain/search"
	"github.com/documize/community/model/attachment"
	"github.com/documize/community/model/audit"
	"github.com/documize/community/model/workflow"
	uuid "github.com/nu7hatch/gouuid"
)

// Handler contains the runtime information such as logging and database.
type Handler struct {
	Runtime *env.Runtime
	Store   *domain.Store
	Indexer indexer.Indexer
}

// Download sends requested file to the client/browser.
func (h *Handler) Download(w http.ResponseWriter, r *http.Request) {
	method := "attachment.Download"
	ctx := domain.GetRequestContext(r)
	ctx.Subdomain = organization.GetSubdomainFromHost(r)

	a, err := h.Store.Attachment.GetAttachment(ctx, request.Param(r, "orgID"), request.Param(r, "attachmentID"))

	if err == sql.ErrNoRows {
		response.WriteNotFoundError(w, method, request.Param(r, "fileID"))
		return
	}
	if err != nil {
		h.Runtime.Log.Error("get attachment", err)
		response.WriteServerError(w, method, err)
		return
	}

	typ := mime.TypeByExtension("." + a.Extension)
	if typ == "" {
		typ = "application/octet-stream"
	}

	w.Header().Set("Content-Type", typ)
	w.Header().Set("Content-Disposition", `Attachment; filename="`+a.Filename+`" ; `+`filename*="`+a.Filename+`"`)
	w.Header().Set("Content-Length", fmt.Sprintf("%d", len(a.Data)))
	w.WriteHeader(http.StatusOK)

	_, err = w.Write(a.Data)
	if err != nil {
		h.Runtime.Log.Error("write attachment", err)
		return
	}

	h.Store.Audit.Record(ctx, audit.EventTypeAttachmentDownload)
}

// Get is an end-point that returns all of the attachments of a particular documentID.
func (h *Handler) Get(w http.ResponseWriter, r *http.Request) {
	method := "attachment.GetAttachments"
	ctx := domain.GetRequestContext(r)

	documentID := request.Param(r, "documentID")
	if len(documentID) == 0 {
		response.WriteMissingDataError(w, method, "documentID")
		return
	}

	if !permission.CanViewDocument(ctx, *h.Store, documentID) {
		response.WriteForbiddenError(w)
		return
	}

	a, err := h.Store.Attachment.GetAttachments(ctx, documentID)
	if err != nil && err != sql.ErrNoRows {
		h.Runtime.Log.Error("get attachment", err)
		response.WriteServerError(w, method, err)
		return
	}

	if len(a) == 0 {
		a = []attachment.Attachment{}
	}

	response.WriteJSON(w, a)
}

// Delete is an endpoint that deletes a particular document attachment.
func (h *Handler) Delete(w http.ResponseWriter, r *http.Request) {
	method := "attachment.DeleteAttachment"
	ctx := domain.GetRequestContext(r)

	documentID := request.Param(r, "documentID")
	if len(documentID) == 0 {
		response.WriteMissingDataError(w, method, "documentID")
		return
	}

	attachmentID := request.Param(r, "attachmentID")
	if len(attachmentID) == 0 {
		response.WriteMissingDataError(w, method, "attachmentID")
		return
	}

	if !permission.CanChangeDocument(ctx, *h.Store, documentID) {
		response.WriteForbiddenError(w)
		return
	}

	var err error
	ctx.Transaction, err = h.Runtime.Db.Beginx()
	if err != nil {
		h.Runtime.Log.Error("transaction", err)
		response.WriteServerError(w, method, err)
		return
	}

	_, err = h.Store.Attachment.Delete(ctx, attachmentID)
	if err != nil {
		ctx.Transaction.Rollback()
		response.WriteServerError(w, method, err)
		h.Runtime.Log.Error("delete attachment", err)
		return
	}

	// Mark references to this document as orphaned
	err = h.Store.Link.MarkOrphanAttachmentLink(ctx, attachmentID)
	if err != nil {
		ctx.Transaction.Rollback()
		response.WriteServerError(w, method, err)
		h.Runtime.Log.Error("delete attachment links", err)
		return
	}

	ctx.Transaction.Commit()

	h.Store.Audit.Record(ctx, audit.EventTypeAttachmentDelete)

	a, _ := h.Store.Attachment.GetAttachments(ctx, documentID)
	d, _ := h.Store.Document.Get(ctx, documentID)

	if d.Lifecycle == workflow.LifecycleLive {
		go h.Indexer.IndexDocument(ctx, d, a)
	} else {
		go h.Indexer.DeleteDocument(ctx, d.RefID)
	}

	response.WriteEmpty(w)
}

// Add stores files against a document.
func (h *Handler) Add(w http.ResponseWriter, r *http.Request) {
	method := "attachment.Add"
	ctx := domain.GetRequestContext(r)

	documentID := request.Param(r, "documentID")
	if len(documentID) == 0 {
		response.WriteMissingDataError(w, method, "documentID")
		return
	}

	if !permission.CanChangeDocument(ctx, *h.Store, documentID) {
		response.WriteForbiddenError(w)
		return
	}

	filedata, filename, err := r.FormFile("attachment")
	if err != nil {
		response.WriteMissingDataError(w, method, "attachment")
		return
	}

	b := new(bytes.Buffer)
	_, err = io.Copy(b, filedata)
	if err != nil {
		response.WriteServerError(w, method, err)
		h.Runtime.Log.Error("add attachment", err)
		return
	}

	var job = "some-uuid"
	newUUID, err := uuid.NewV4()
	if err != nil {
		h.Runtime.Log.Error("uuid", err)
		response.WriteServerError(w, method, err)
		return
	}
	job = newUUID.String()

	var a attachment.Attachment
	refID := uniqueid.Generate()
	a.RefID = refID
	a.DocumentID = documentID
	a.Job = job
	random := secrets.GenerateSalt()
	a.FileID = random[0:9]
	a.Filename = filename.Filename
	a.Data = b.Bytes()

	ctx.Transaction, err = h.Runtime.Db.Beginx()
	if err != nil {
		response.WriteServerError(w, method, err)
		h.Runtime.Log.Error("transaction", err)
		return
	}

	err = h.Store.Attachment.Add(ctx, a)
	if err != nil {
		ctx.Transaction.Rollback()
		response.WriteServerError(w, method, err)
		h.Runtime.Log.Error("add attachment", err)
		return
	}

	ctx.Transaction.Commit()

	h.Store.Audit.Record(ctx, audit.EventTypeAttachmentAdd)

	all, _ := h.Store.Attachment.GetAttachments(ctx, documentID)
	d, _ := h.Store.Document.Get(ctx, documentID)

	if d.Lifecycle == workflow.LifecycleLive {
		go h.Indexer.IndexDocument(ctx, d, all)
	} else {
		go h.Indexer.DeleteDocument(ctx, d.RefID)
	}

	response.WriteEmpty(w)
}
<|endoftext|>


<commit_before>package weed_server

import (
	"net"
	"strings"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb"
	"github.com/chrislusf/seaweedfs/weed/storage"
	"github.com/chrislusf/seaweedfs/weed/topology"
	"google.golang.org/grpc/peer"
)

func (ms MasterServer) SendHeartbeat(stream pb.Seaweed_SendHeartbeatServer) error {
	var dn *topology.DataNode
	t := ms.Topo
	for {
		heartbeat, err := stream.Recv()
		if err == nil {
			if dn == nil {
				t.Sequence.SetMax(heartbeat.MaxFileKey)
				if heartbeat.Ip == "" {
					if pr, ok := peer.FromContext(stream.Context()); ok {
						if pr.Addr != net.Addr(nil) {
							heartbeat.Ip = pr.Addr.String()[0:strings.LastIndex(pr.Addr.String(), ":")]
							glog.V(0).Infof("remote IP address is detected as %v", heartbeat.Ip)
						}
					}
				}
				dcName, rackName := t.Configuration.Locate(heartbeat.Ip, heartbeat.DataCenter, heartbeat.Rack)
				dc := t.GetOrCreateDataCenter(dcName)
				rack := dc.GetOrCreateRack(rackName)
				dn = rack.GetOrCreateDataNode(heartbeat.Ip,
					int(heartbeat.Port), heartbeat.PublicUrl,
					int(heartbeat.MaxVolumeCount))
				glog.V(0).Infof("added volume server %v:%d", heartbeat.GetIp(), heartbeat.GetPort())
				if err := stream.Send(&pb.HeartbeatResponse{
					VolumeSizeLimit: uint64(ms.volumeSizeLimitMB) * 1024 * 1024,
					SecretKey:       string(ms.guard.SecretKey),
				}); err != nil {
					return err
				}
			}

			var volumeInfos []storage.VolumeInfo
			for _, v := range heartbeat.Volumes {
				if vi, err := storage.NewVolumeInfo(v); err == nil {
					volumeInfos = append(volumeInfos, vi)
				} else {
					glog.V(0).Infof("Fail to convert joined volume information: %v", err)
				}
			}
			deletedVolumes := dn.UpdateVolumes(volumeInfos)
			for _, v := range volumeInfos {
				t.RegisterVolumeLayout(v, dn)
			}
			for _, v := range deletedVolumes {
				t.UnRegisterVolumeLayout(v, dn)
			}

		} else {
			glog.V(0).Infof("lost volume server %s:%d", dn.Ip, dn.Port)
			if dn != nil {
				t.UnRegisterDataNode(dn)
			}
			return err
		}

		// tell the volume servers about the leader
		newLeader, err := t.Leader()
		if err == nil {
			if err := stream.Send(&pb.HeartbeatResponse{
				Leader: newLeader,
			}); err != nil {
				return err
			}
		}
	}
}
<commit_msg>avoid nil<commit_after>package weed_server

import (
	"net"
	"strings"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb"
	"github.com/chrislusf/seaweedfs/weed/storage"
	"github.com/chrislusf/seaweedfs/weed/topology"
	"google.golang.org/grpc/peer"
)

func (ms MasterServer) SendHeartbeat(stream pb.Seaweed_SendHeartbeatServer) error {
	var dn *topology.DataNode
	t := ms.Topo
	for {
		heartbeat, err := stream.Recv()
		if err == nil {
			if dn == nil {
				t.Sequence.SetMax(heartbeat.MaxFileKey)
				if heartbeat.Ip == "" {
					if pr, ok := peer.FromContext(stream.Context()); ok {
						if pr.Addr != net.Addr(nil) {
							heartbeat.Ip = pr.Addr.String()[0:strings.LastIndex(pr.Addr.String(), ":")]
							glog.V(0).Infof("remote IP address is detected as %v", heartbeat.Ip)
						}
					}
				}
				dcName, rackName := t.Configuration.Locate(heartbeat.Ip, heartbeat.DataCenter, heartbeat.Rack)
				dc := t.GetOrCreateDataCenter(dcName)
				rack := dc.GetOrCreateRack(rackName)
				dn = rack.GetOrCreateDataNode(heartbeat.Ip,
					int(heartbeat.Port), heartbeat.PublicUrl,
					int(heartbeat.MaxVolumeCount))
				glog.V(0).Infof("added volume server %v:%d", heartbeat.GetIp(), heartbeat.GetPort())
				if err := stream.Send(&pb.HeartbeatResponse{
					VolumeSizeLimit: uint64(ms.volumeSizeLimitMB) * 1024 * 1024,
					SecretKey:       string(ms.guard.SecretKey),
				}); err != nil {
					return err
				}
			}

			var volumeInfos []storage.VolumeInfo
			for _, v := range heartbeat.Volumes {
				if vi, err := storage.NewVolumeInfo(v); err == nil {
					volumeInfos = append(volumeInfos, vi)
				} else {
					glog.V(0).Infof("Fail to convert joined volume information: %v", err)
				}
			}
			deletedVolumes := dn.UpdateVolumes(volumeInfos)
			for _, v := range volumeInfos {
				t.RegisterVolumeLayout(v, dn)
			}
			for _, v := range deletedVolumes {
				t.UnRegisterVolumeLayout(v, dn)
			}

		} else {
			if dn != nil {
				glog.V(0).Infof("lost volume server %s:%d", dn.Ip, dn.Port)
				t.UnRegisterDataNode(dn)
			}
			return err
		}

		// tell the volume servers about the leader
		newLeader, err := t.Leader()
		if err == nil {
			if err := stream.Send(&pb.HeartbeatResponse{
				Leader: newLeader,
			}); err != nil {
				return err
			}
		}
	}
}
<|endoftext|>


<commit_before>package storage

import (
	"fmt"
	"os"
	"sync/atomic"

	"github.com/hashicorp/golang-lru"

	"github.com/chrislusf/seaweedfs/weed/util"
	"github.com/chrislusf/seaweedfs/weed/glog"
)

var (
	bytesCache *lru.Cache
	bytesPool  *util.BytesPool
)

/*
There are one level of caching, and one level of pooling.

In pooling, all []byte are fetched and returned to the pool bytesPool.

In caching, the string~[]byte mapping is cached
*/
func init() {
	bytesPool = util.NewBytesPool()
	bytesCache, _ = lru.NewWithEvict(50, func(key interface{}, value interface{}) {
		value.(*Block).decreaseReference()
	})
}

type Block struct {
	Bytes    []byte
	refCount int32
}

func (block *Block) decreaseReference() {
	if atomic.AddInt32(&block.refCount, -1) == 0 {
		bytesPool.Put(block.Bytes)
	}
}
func (block *Block) increaseReference() {
	atomic.AddInt32(&block.refCount, 1)
}

// get bytes from the LRU cache of []byte first, then from the bytes pool
// when []byte in LRU cache is evicted, it will be put back to the bytes pool
func getBytesForFileBlock(r *os.File, offset int64, readSize int) (dataSlice []byte, block *Block, err error) {
	//Skip the cache if we are looking for a block that is too big to fit in the cache (defaulting to 10MB)
	cacheable := readSize <= (1024*1024*10)
	if !cacheable {
		glog.V(4).Infoln("Block too big to keep in cache. Size:", readSize)
	}
	cacheKey := string("")
	if cacheable {
	// check cache, return if found
		cacheKey = fmt.Sprintf("%d:%d:%d", r.Fd(), offset >> 3, readSize)
	if obj, found := bytesCache.Get(cacheKey); found {
		glog.V(4).Infoln("Found block in cache. Size:", readSize)
		block = obj.(*Block)
		block.increaseReference()
		dataSlice = block.Bytes[0:readSize]
		return dataSlice, block, nil
		}
	}

	// get the []byte from pool
	b := bytesPool.Get(readSize)
	// refCount = 2, one by the bytesCache, one by the actual needle object
	refCount := int32(1)
	if cacheable {
		refCount = 2
	}
	block = &Block{Bytes: b, refCount: refCount}
	dataSlice = block.Bytes[0:readSize]
	_, err = r.ReadAt(dataSlice, offset)
	if cacheable {
	bytesCache.Add(cacheKey, block)
	}
	return dataSlice, block, err
}

func (n *Needle) ReleaseMemory() {
	if n.rawBlock != nil {
		n.rawBlock.decreaseReference()
	}
}
func ReleaseBytes(b []byte) {
	bytesPool.Put(b)
}
<commit_msg>Revert "Changing needle_byte_cache so that it doesn't grow so big when larger files are added."<commit_after>package storage

import (
	"fmt"
	"os"
	"sync/atomic"

	"github.com/hashicorp/golang-lru"

	"github.com/chrislusf/seaweedfs/weed/util"
)

var (
	bytesCache *lru.Cache
	bytesPool  *util.BytesPool
)

/*
There are one level of caching, and one level of pooling.

In pooling, all []byte are fetched and returned to the pool bytesPool.

In caching, the string~[]byte mapping is cached
*/
func init() {
	bytesPool = util.NewBytesPool()
	bytesCache, _ = lru.NewWithEvict(512, func(key interface{}, value interface{}) {
		value.(*Block).decreaseReference()
	})
}

type Block struct {
	Bytes    []byte
	refCount int32
}

func (block *Block) decreaseReference() {
	if atomic.AddInt32(&block.refCount, -1) == 0 {
		bytesPool.Put(block.Bytes)
	}
}
func (block *Block) increaseReference() {
	atomic.AddInt32(&block.refCount, 1)
}

// get bytes from the LRU cache of []byte first, then from the bytes pool
// when []byte in LRU cache is evicted, it will be put back to the bytes pool
func getBytesForFileBlock(r *os.File, offset int64, readSize int) (dataSlice []byte, block *Block, err error) {
	// check cache, return if found
	cacheKey := fmt.Sprintf("%d:%d:%d", r.Fd(), offset>>3, readSize)
	if obj, found := bytesCache.Get(cacheKey); found {
		block = obj.(*Block)
		block.increaseReference()
		dataSlice = block.Bytes[0:readSize]
		return dataSlice, block, nil
	}

	// get the []byte from pool
	b := bytesPool.Get(readSize)
	// refCount = 2, one by the bytesCache, one by the actual needle object
	block = &Block{Bytes: b, refCount: 2}
	dataSlice = block.Bytes[0:readSize]
	_, err = r.ReadAt(dataSlice, offset)
	bytesCache.Add(cacheKey, block)
	return dataSlice, block, err
}

func (n *Needle) ReleaseMemory() {
	if n.rawBlock != nil {
		n.rawBlock.decreaseReference()
	}
}
func ReleaseBytes(b []byte) {
	bytesPool.Put(b)
}
<|endoftext|>


<commit_before>package containerstore_test

import (
	"crypto/rand"
	"crypto/rsa"
	"crypto/x509"
	"encoding/pem"
	"io"
	"io/ioutil"
	"math/big"
	"net"
	"os"
	"path/filepath"
	"time"

	"code.cloudfoundry.org/clock/fakeclock"
	"code.cloudfoundry.org/executor"
	"code.cloudfoundry.org/executor/depot/containerstore"
	"code.cloudfoundry.org/garden"
	"code.cloudfoundry.org/lager"
	"code.cloudfoundry.org/lager/lagertest"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("CredManager", func() {
	var (
		credManager containerstore.CredManager
		CaCert      *x509.Certificate
		privateKey  *rsa.PrivateKey
		tmpdir      string
		logger      lager.Logger
		clock       *fakeclock.FakeClock
	)

	BeforeEach(func() {
		var err error
		tmpdir, err = ioutil.TempDir("", "credsmanager")
		Expect(err).ToNot(HaveOccurred())

		logger = lagertest.NewTestLogger("credmanager")
		// Truncate and set to UTC time because of parsing time from certificate
		// and only has second granularity
		clock = fakeclock.NewFakeClock(time.Now().UTC().Truncate(time.Second))

		CaCert, privateKey = createIntermediateCert()
		credManager = containerstore.NewCredManager(tmpdir, rand.Reader, clock, CaCert, privateKey, "containerpath")
	})

	AfterEach(func() {
		os.RemoveAll(tmpdir)
	})

	Context("CreateCredDir", func() {
		It("returns a valid directory path", func() {
			mount, _, err := credManager.CreateCredDir(logger, executor.Container{Guid: "guid"})
			Expect(err).To(Succeed())

			Expect(mount).To(HaveLen(1))
			Expect(mount[0].SrcPath).To(BeADirectory())
			Expect(mount[0].DstPath).To(Equal("containerpath"))
			Expect(mount[0].Mode).To(Equal(garden.BindMountModeRO))
			Expect(mount[0].Origin).To(Equal(garden.BindMountOriginHost))
		})

		It("returns CF_INSTANCE_CERT and CF_INSTANCE_KEY environment variable values", func() {
			_, envVariables, err := credManager.CreateCredDir(logger, executor.Container{Guid: "guid"})
			Expect(err).To(Succeed())

			Expect(envVariables).To(HaveLen(2))
			values := map[string]string{}
			values[envVariables[0].Name] = envVariables[0].Value
			values[envVariables[1].Name] = envVariables[1].Value
			Expect(values).To(Equal(map[string]string{
				"CF_INSTANCE_CERT": "containerpath/instance.crt",
				"CF_INSTANCE_KEY":  "containerpath/instance.key",
			}))
		})

		Context("when making directory fails", func() {
			BeforeEach(func() {
				os.Chmod(tmpdir, 0400)
			})

			It("returns an error", func() {
				_, _, err := credManager.CreateCredDir(logger, executor.Container{Guid: "somefailure"})
				Expect(err).To(HaveOccurred())
			})
		})
	})

	Context("GenerateCreds", func() {
		var container executor.Container

		BeforeEach(func() {
			container = executor.Container{
				Guid:       "container-guid",
				InternalIP: "127.0.0.1",
			}
			_, _, err := credManager.CreateCredDir(logger, container)
			Expect(err).NotTo(HaveOccurred())
		})

		It("puts private key into container directory", func() {
			err := credManager.GenerateCreds(logger, container)
			Expect(err).NotTo(HaveOccurred())

			certPath := filepath.Join(tmpdir, container.Guid)

			keyFile := filepath.Join(certPath, "instance.key")
			data, err := ioutil.ReadFile(keyFile)
			Expect(err).NotTo(HaveOccurred())

			block, rest := pem.Decode(data)
			Expect(block).NotTo(BeNil())
			Expect(rest).To(BeEmpty())

			Expect(block.Type).To(Equal("RSA PRIVATE KEY"))
			key, err := x509.ParsePKCS1PrivateKey(block.Bytes)
			Expect(err).NotTo(HaveOccurred())

			var bits int
			for _, p := range key.Primes {
				bits += p.BitLen()
			}
			Expect(bits).To(Equal(2048))
		})

		Context("when generating private key fails", func() {
			BeforeEach(func() {
				reader := io.LimitReader(rand.Reader, 0)
				credManager = containerstore.NewCredManager(tmpdir, reader, clock, CaCert, privateKey, "")
			})

			It("returns an error", func() {
				err := credManager.GenerateCreds(logger, container)
				Expect(err).To(MatchError("EOF"))
			})
		})

		It("signs and puts the certificate into container directory", func() {
			err := credManager.GenerateCreds(logger, container)
			Expect(err).NotTo(HaveOccurred())

			certPath := filepath.Join(tmpdir, container.Guid)

			certFile := filepath.Join(certPath, "instance.crt")
			Expect(certFile).To(BeARegularFile())
		})

		Describe("the cert", func() {
			var (
				cert *x509.Certificate
				rest []byte
			)

			BeforeEach(func() {
				err := credManager.GenerateCreds(logger, container)
				Expect(err).NotTo(HaveOccurred())
				certFile := filepath.Join(tmpdir, container.Guid, "instance.crt")
				data, err := ioutil.ReadFile(certFile)
				Expect(err).NotTo(HaveOccurred())
				var block *pem.Block
				block, rest = pem.Decode(data)
				Expect(err).NotTo(HaveOccurred())
				Expect(block).NotTo(BeNil())
				Expect(block.Type).To(Equal("CERTIFICATE"))
				certs, err := x509.ParseCertificates(block.Bytes)
				Expect(err).NotTo(HaveOccurred())
				Expect(certs).To(HaveLen(1))
				cert = certs[0]
			})

			It("has the container ip", func() {
				ip := net.ParseIP(container.InternalIP)
				Expect(cert.IPAddresses).To(ContainElement(ip.To4()))
			})

			It("signed by the rep intermediate CA", func() {
				CaCertPool := x509.NewCertPool()
				CaCertPool.AddCert(CaCert)
				verifyOpts := x509.VerifyOptions{Roots: CaCertPool}
				Expect(cert.CheckSignatureFrom(CaCert)).To(Succeed())
				_, err := cert.Verify(verifyOpts)
				Expect(err).NotTo(HaveOccurred())
			})

			It("common name should be set to the container guid", func() {
				Expect(cert.Subject.CommonName).To(Equal(container.Guid))
			})

			It("expires in the next 24 hours", func() {
				Expect(cert.NotAfter).To(Equal(clock.Now().Add(24 * time.Hour)))
			})

			It("not before is set to current timestamp", func() {
				Expect(cert.NotBefore).To(Equal(clock.Now()))
			})

			It("sets the serial number to the container guid", func() {
				expected := big.NewInt(0)
				expected.SetBytes([]byte(container.Guid))

				Expect(expected).To(Equal(cert.SerialNumber))
			})

			It("has the rep intermediate CA", func() {
				block, rest := pem.Decode(rest)
				Expect(block).NotTo(BeNil())
				Expect(rest).To(BeEmpty())
				Expect(block.Type).To(Equal("CERTIFICATE"))
				certs, err := x509.ParseCertificates(block.Bytes)
				Expect(err).NotTo(HaveOccurred())
				Expect(certs).To(HaveLen(1))
				cert = certs[0]
				Expect(cert).To(Equal(CaCert))
			})
		})
	})

	Context("RemoveCreds", func() {
		var container executor.Container

		BeforeEach(func() {
			container = executor.Container{
				Guid:       "container-guid",
				InternalIP: "127.0.0.1",
			}
		})

		It("removes container credentials from the filesystem", func() {
			certMount, _, err := credManager.CreateCredDir(logger, container)
			Expect(err).NotTo(HaveOccurred())
			Expect(certMount[0].SrcPath).To(BeADirectory())

			credManager.RemoveCreds(logger, container)
			Expect(certMount[0].SrcPath).ToNot(BeADirectory())
		})
	})
})

func createIntermediateCert() (*x509.Certificate, *rsa.PrivateKey) {
	privateKey, err := rsa.GenerateKey(rand.Reader, 2048)
	Expect(err).NotTo(HaveOccurred())

	template := &x509.Certificate{
		IsCA: true,
		BasicConstraintsValid: true,
		SerialNumber:          big.NewInt(1),
		NotAfter:              time.Now().Add(36 * time.Hour),
	}
	certBytes, err := x509.CreateCertificate(rand.Reader, template, template, privateKey.Public(), privateKey)
	Expect(err).NotTo(HaveOccurred())

	certs, err := x509.ParseCertificates(certBytes)
	Expect(err).NotTo(HaveOccurred())
	Expect(certs).To(HaveLen(1))
	return certs[0], privateKey
}
<commit_msg>Chmod doesn't work on Windows<commit_after>package containerstore_test

import (
	"crypto/rand"
	"crypto/rsa"
	"crypto/x509"
	"encoding/pem"
	"io"
	"io/ioutil"
	"math/big"
	"net"
	"os"
	"path/filepath"
	"runtime"
	"time"

	"code.cloudfoundry.org/clock/fakeclock"
	"code.cloudfoundry.org/executor"
	"code.cloudfoundry.org/executor/depot/containerstore"
	"code.cloudfoundry.org/garden"
	"code.cloudfoundry.org/lager"
	"code.cloudfoundry.org/lager/lagertest"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("CredManager", func() {
	var (
		credManager containerstore.CredManager
		CaCert      *x509.Certificate
		privateKey  *rsa.PrivateKey
		tmpdir      string
		logger      lager.Logger
		clock       *fakeclock.FakeClock
	)

	BeforeEach(func() {
		var err error
		tmpdir, err = ioutil.TempDir("", "credsmanager")
		Expect(err).ToNot(HaveOccurred())

		logger = lagertest.NewTestLogger("credmanager")
		// Truncate and set to UTC time because of parsing time from certificate
		// and only has second granularity
		clock = fakeclock.NewFakeClock(time.Now().UTC().Truncate(time.Second))

		CaCert, privateKey = createIntermediateCert()
		credManager = containerstore.NewCredManager(tmpdir, rand.Reader, clock, CaCert, privateKey, "containerpath")
	})

	AfterEach(func() {
		os.RemoveAll(tmpdir)
	})

	Context("CreateCredDir", func() {
		It("returns a valid directory path", func() {
			mount, _, err := credManager.CreateCredDir(logger, executor.Container{Guid: "guid"})
			Expect(err).To(Succeed())

			Expect(mount).To(HaveLen(1))
			Expect(mount[0].SrcPath).To(BeADirectory())
			Expect(mount[0].DstPath).To(Equal("containerpath"))
			Expect(mount[0].Mode).To(Equal(garden.BindMountModeRO))
			Expect(mount[0].Origin).To(Equal(garden.BindMountOriginHost))
		})

		It("returns CF_INSTANCE_CERT and CF_INSTANCE_KEY environment variable values", func() {
			_, envVariables, err := credManager.CreateCredDir(logger, executor.Container{Guid: "guid"})
			Expect(err).To(Succeed())

			Expect(envVariables).To(HaveLen(2))
			values := map[string]string{}
			values[envVariables[0].Name] = envVariables[0].Value
			values[envVariables[1].Name] = envVariables[1].Value
			Expect(values).To(Equal(map[string]string{
				"CF_INSTANCE_CERT": "containerpath/instance.crt",
				"CF_INSTANCE_KEY":  "containerpath/instance.key",
			}))
		})

		Context("when making directory fails", func() {
			BeforeEach(func() {
				if runtime.GOOS == "windows" {
					Skip("Chmod does not work on windows")
				}

				os.Chmod(tmpdir, 0400)
			})

			It("returns an error", func() {
				_, _, err := credManager.CreateCredDir(logger, executor.Container{Guid: "somefailure"})
				Expect(err).To(HaveOccurred())
			})
		})
	})

	Context("GenerateCreds", func() {
		var container executor.Container

		BeforeEach(func() {
			container = executor.Container{
				Guid:       "container-guid",
				InternalIP: "127.0.0.1",
			}
			_, _, err := credManager.CreateCredDir(logger, container)
			Expect(err).NotTo(HaveOccurred())
		})

		It("puts private key into container directory", func() {
			err := credManager.GenerateCreds(logger, container)
			Expect(err).NotTo(HaveOccurred())

			certPath := filepath.Join(tmpdir, container.Guid)

			keyFile := filepath.Join(certPath, "instance.key")
			data, err := ioutil.ReadFile(keyFile)
			Expect(err).NotTo(HaveOccurred())

			block, rest := pem.Decode(data)
			Expect(block).NotTo(BeNil())
			Expect(rest).To(BeEmpty())

			Expect(block.Type).To(Equal("RSA PRIVATE KEY"))
			key, err := x509.ParsePKCS1PrivateKey(block.Bytes)
			Expect(err).NotTo(HaveOccurred())

			var bits int
			for _, p := range key.Primes {
				bits += p.BitLen()
			}
			Expect(bits).To(Equal(2048))
		})

		Context("when generating private key fails", func() {
			BeforeEach(func() {
				reader := io.LimitReader(rand.Reader, 0)
				credManager = containerstore.NewCredManager(tmpdir, reader, clock, CaCert, privateKey, "")
			})

			It("returns an error", func() {
				err := credManager.GenerateCreds(logger, container)
				Expect(err).To(MatchError("EOF"))
			})
		})

		It("signs and puts the certificate into container directory", func() {
			err := credManager.GenerateCreds(logger, container)
			Expect(err).NotTo(HaveOccurred())

			certPath := filepath.Join(tmpdir, container.Guid)

			certFile := filepath.Join(certPath, "instance.crt")
			Expect(certFile).To(BeARegularFile())
		})

		Describe("the cert", func() {
			var (
				cert *x509.Certificate
				rest []byte
			)

			BeforeEach(func() {
				err := credManager.GenerateCreds(logger, container)
				Expect(err).NotTo(HaveOccurred())
				certFile := filepath.Join(tmpdir, container.Guid, "instance.crt")
				data, err := ioutil.ReadFile(certFile)
				Expect(err).NotTo(HaveOccurred())
				var block *pem.Block
				block, rest = pem.Decode(data)
				Expect(err).NotTo(HaveOccurred())
				Expect(block).NotTo(BeNil())
				Expect(block.Type).To(Equal("CERTIFICATE"))
				certs, err := x509.ParseCertificates(block.Bytes)
				Expect(err).NotTo(HaveOccurred())
				Expect(certs).To(HaveLen(1))
				cert = certs[0]
			})

			It("has the container ip", func() {
				ip := net.ParseIP(container.InternalIP)
				Expect(cert.IPAddresses).To(ContainElement(ip.To4()))
			})

			It("signed by the rep intermediate CA", func() {
				CaCertPool := x509.NewCertPool()
				CaCertPool.AddCert(CaCert)
				verifyOpts := x509.VerifyOptions{Roots: CaCertPool}
				Expect(cert.CheckSignatureFrom(CaCert)).To(Succeed())
				_, err := cert.Verify(verifyOpts)
				Expect(err).NotTo(HaveOccurred())
			})

			It("common name should be set to the container guid", func() {
				Expect(cert.Subject.CommonName).To(Equal(container.Guid))
			})

			It("expires in the next 24 hours", func() {
				Expect(cert.NotAfter).To(Equal(clock.Now().Add(24 * time.Hour)))
			})

			It("not before is set to current timestamp", func() {
				Expect(cert.NotBefore).To(Equal(clock.Now()))
			})

			It("sets the serial number to the container guid", func() {
				expected := big.NewInt(0)
				expected.SetBytes([]byte(container.Guid))

				Expect(expected).To(Equal(cert.SerialNumber))
			})

			It("has the rep intermediate CA", func() {
				block, rest := pem.Decode(rest)
				Expect(block).NotTo(BeNil())
				Expect(rest).To(BeEmpty())
				Expect(block.Type).To(Equal("CERTIFICATE"))
				certs, err := x509.ParseCertificates(block.Bytes)
				Expect(err).NotTo(HaveOccurred())
				Expect(certs).To(HaveLen(1))
				cert = certs[0]
				Expect(cert).To(Equal(CaCert))
			})
		})
	})

	Context("RemoveCreds", func() {
		var container executor.Container

		BeforeEach(func() {
			container = executor.Container{
				Guid:       "container-guid",
				InternalIP: "127.0.0.1",
			}
		})

		It("removes container credentials from the filesystem", func() {
			certMount, _, err := credManager.CreateCredDir(logger, container)
			Expect(err).NotTo(HaveOccurred())
			Expect(certMount[0].SrcPath).To(BeADirectory())

			credManager.RemoveCreds(logger, container)
			Expect(certMount[0].SrcPath).ToNot(BeADirectory())
		})
	})
})

func createIntermediateCert() (*x509.Certificate, *rsa.PrivateKey) {
	privateKey, err := rsa.GenerateKey(rand.Reader, 2048)
	Expect(err).NotTo(HaveOccurred())

	template := &x509.Certificate{
		IsCA: true,
		BasicConstraintsValid: true,
		SerialNumber:          big.NewInt(1),
		NotAfter:              time.Now().Add(36 * time.Hour),
	}
	certBytes, err := x509.CreateCertificate(rand.Reader, template, template, privateKey.Public(), privateKey)
	Expect(err).NotTo(HaveOccurred())

	certs, err := x509.ParseCertificates(certBytes)
	Expect(err).NotTo(HaveOccurred())
	Expect(certs).To(HaveLen(1))
	return certs[0], privateKey
}
<|endoftext|>


<commit_before>package fs

import (
	"bufio"
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"sync"

	"github.com/git-lfs/git-lfs/tools"
	"github.com/rubyist/tracerx"
)

var oidRE = regexp.MustCompile(`\A[[:alnum:]]{64}`)

// Environment is a copy of a subset of the interface
// github.com/git-lfs/git-lfs/config.Environment.
//
// For more information, see config/environment.go.
type Environment interface {
	Get(key string) (val string, ok bool)
}

// Object represents a locally stored LFS object.
type Object struct {
	Oid  string
	Size int64
}

type Filesystem struct {
	GitStorageDir string   // parent of objects/lfs (may be same as GitDir but may not)
	LFSStorageDir string   // parent of lfs objects and tmp dirs. Default: ".git/lfs"
	ReferenceDirs []string // alternative local media dirs (relative to clone reference repo)
	lfsobjdir     string
	tmpdir        string
	logdir        string
	mu            sync.Mutex
}

func (f *Filesystem) EachObject(fn func(Object) error) error {
	var eachErr error
	tools.FastWalkGitRepo(f.LFSObjectDir(), func(parentDir string, info os.FileInfo, err error) {
		if err != nil {
			eachErr = err
			return
		}
		if eachErr != nil || info.IsDir() {
			return
		}
		if oidRE.MatchString(info.Name()) {
			fn(Object{Oid: info.Name(), Size: info.Size()})
		}
	})
	return eachErr
}

func (f *Filesystem) ObjectExists(oid string, size int64) bool {
	return tools.FileExistsOfSize(f.ObjectPathname(oid), size)
}

func (f *Filesystem) ObjectPath(oid string) (string, error) {
	dir := f.localObjectDir(oid)
	if err := os.MkdirAll(dir, 0755); err != nil {
		return "", fmt.Errorf("Error trying to create local storage directory in %q: %s", dir, err)
	}
	return filepath.Join(dir, oid), nil
}

func (f *Filesystem) ObjectPathname(oid string) string {
	return filepath.Join(f.localObjectDir(oid), oid)
}

func (f *Filesystem) DecodePathname(path string) string {
	return string(DecodePathBytes([]byte(path)))
}

/**
 * Revert non ascii chracters escaped by git or windows (as octal sequences \000) back to bytes.
 */
func DecodePathBytes(path []byte) []byte {
	var expression = regexp.MustCompile(`\\[0-9]{3}`)
	var buffer bytes.Buffer

	// strip quotes if any
	if len(path) > 2 && path[0] == '"' && path[len(path)-1] == '"' {
		path = path[1 : len(path)-1]
	}

	base := 0
	for _, submatches := range expression.FindAllSubmatchIndex(path, -1) {
		buffer.Write(path[base:submatches[0]])

		match := string(path[submatches[0]+1 : submatches[0]+4])

		k, err := strconv.ParseUint(match, 8, 64)
		if err != nil {
			return path
		} // abort on error

		buffer.Write([]byte{byte(k)})
		base = submatches[1]
	}

	buffer.Write(path[base:len(path)])

	return buffer.Bytes()
}

func (f *Filesystem) localObjectDir(oid string) string {
	return filepath.Join(f.LFSObjectDir(), oid[0:2], oid[2:4])
}

func (f *Filesystem) ObjectReferencePaths(oid string) []string {
	if len(f.ReferenceDirs) == 0 {
		return nil
	}

	var paths []string
	for _, ref := range f.ReferenceDirs {
		paths = append(paths, filepath.Join(ref, oid[0:2], oid[2:4], oid))
	}
	return paths
}

func (f *Filesystem) LFSObjectDir() string {
	f.mu.Lock()
	defer f.mu.Unlock()

	if len(f.lfsobjdir) == 0 {
		f.lfsobjdir = filepath.Join(f.LFSStorageDir, "objects")
		os.MkdirAll(f.lfsobjdir, 0755)
	}

	return f.lfsobjdir
}

func (f *Filesystem) LogDir() string {
	f.mu.Lock()
	defer f.mu.Unlock()

	if len(f.logdir) == 0 {
		f.logdir = filepath.Join(f.LFSStorageDir, "logs")
		os.MkdirAll(f.logdir, 0755)
	}

	return f.logdir
}

func (f *Filesystem) TempDir() string {
	f.mu.Lock()
	defer f.mu.Unlock()

	if len(f.tmpdir) == 0 {
		f.tmpdir = filepath.Join(f.LFSStorageDir, "tmp")
		os.MkdirAll(f.tmpdir, 0755)
	}

	return f.tmpdir
}

func (f *Filesystem) Cleanup() error {
	if f == nil {
		return nil
	}
	return f.cleanupTmp()
}

// New initializes a new *Filesystem with the given directories. gitdir is the
// path to the bare repo, workdir is the path to the repository working
// directory, and lfsdir is the optional path to the `.git/lfs` directory.
func New(env Environment, gitdir, workdir, lfsdir string) *Filesystem {
	fs := &Filesystem{
		GitStorageDir: resolveGitStorageDir(gitdir),
	}

	fs.ReferenceDirs = resolveReferenceDirs(env, fs.GitStorageDir)

	if len(lfsdir) == 0 {
		lfsdir = "lfs"
	}

	if filepath.IsAbs(lfsdir) {
		fs.LFSStorageDir = lfsdir
	} else {
		fs.LFSStorageDir = filepath.Join(fs.GitStorageDir, lfsdir)
	}

	return fs
}

func resolveReferenceDirs(env Environment, gitStorageDir string) []string {
	var references []string

	envAlternates, ok := env.Get("GIT_ALTERNATE_OBJECT_DIRECTORIES")
	if ok {
		splits := strings.Split(envAlternates, string(os.PathListSeparator))
		for _, split := range splits {
			if dir, ok := existsAlternate(split); ok {
				references = append(references, dir)
			}
		}
	}

	cloneReferencePath := filepath.Join(gitStorageDir, "objects", "info", "alternates")
	if tools.FileExists(cloneReferencePath) {
		f, err := os.Open(cloneReferencePath)
		if err != nil {
			tracerx.Printf("could not open %s: %s",
				cloneReferencePath, err)
			return nil
		}
		defer f.Close()

		scanner := bufio.NewScanner(f)
		for scanner.Scan() {
			text := strings.TrimSpace(scanner.Text())
			if len(text) == 0 || strings.HasPrefix(text, "#") {
				continue
			}

			if dir, ok := existsAlternate(text); ok {
				references = append(references, dir)
			}
		}

		if err := scanner.Err(); err != nil {
			tracerx.Printf("could not scan %s: %s",
				cloneReferencePath, err)
		}
	}
	return references
}

// existsAlternate takes an object directory given in "objs" (read as a single,
// line from .git/objects/info/alternates). If that is a satisfiable alternates
// directory (i.e., it exists), the directory is returned along with "true". If
// not, the empty string and false is returned instead.
func existsAlternate(objs string) (string, bool) {
	objs = strings.TrimSpace(objs)
	if strings.HasPrefix(objs, "#") {
		var err error

		unquote := strings.LastIndex(objs, "\"")
		if unquote == 0 {
			return "", false
		}

		objs, err = strconv.Unquote(objs[:unquote+1])
		if err != nil {
			return "", false
		}
	}

	storage := filepath.Join(filepath.Dir(objs), "lfs", "objects")

	if tools.DirExists(storage) {
		return storage, true
	}
	return "", false
}

// From a git dir, get the location that objects are to be stored (we will store lfs alongside)
// Sometimes there is an additional level of redirect on the .git folder by way of a commondir file
// before you find object storage, e.g. 'git worktree' uses this. It redirects to gitdir either by GIT_DIR
// (during setup) or .git/git-dir: (during use), but this only contains the index etc, the objects
// are found in another git dir via 'commondir'.
func resolveGitStorageDir(gitDir string) string {
	commondirpath := filepath.Join(gitDir, "commondir")
	if tools.FileExists(commondirpath) && !tools.DirExists(filepath.Join(gitDir, "objects")) {
		// no git-dir: prefix in commondir
		storage, err := processGitRedirectFile(commondirpath, "")
		if err == nil {
			return storage
		}
	}
	return gitDir
}

func processGitRedirectFile(file, prefix string) (string, error) {
	data, err := ioutil.ReadFile(file)
	if err != nil {
		return "", err
	}

	contents := string(data)
	var dir string
	if len(prefix) > 0 {
		if !strings.HasPrefix(contents, prefix) {
			// Prefix required & not found
			return "", nil
		}
		dir = strings.TrimSpace(contents[len(prefix):])
	} else {
		dir = strings.TrimSpace(contents)
	}

	if !filepath.IsAbs(dir) {
		// The .git file contains a relative path.
		// Create an absolute path based on the directory the .git file is located in.
		dir = filepath.Join(filepath.Dir(file), dir)
	}

	return dir, nil
}
<commit_msg>fs/fs.go: strings.HasPrefix typo<commit_after>package fs

import (
	"bufio"
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"sync"

	"github.com/git-lfs/git-lfs/tools"
	"github.com/rubyist/tracerx"
)

var oidRE = regexp.MustCompile(`\A[[:alnum:]]{64}`)

// Environment is a copy of a subset of the interface
// github.com/git-lfs/git-lfs/config.Environment.
//
// For more information, see config/environment.go.
type Environment interface {
	Get(key string) (val string, ok bool)
}

// Object represents a locally stored LFS object.
type Object struct {
	Oid  string
	Size int64
}

type Filesystem struct {
	GitStorageDir string   // parent of objects/lfs (may be same as GitDir but may not)
	LFSStorageDir string   // parent of lfs objects and tmp dirs. Default: ".git/lfs"
	ReferenceDirs []string // alternative local media dirs (relative to clone reference repo)
	lfsobjdir     string
	tmpdir        string
	logdir        string
	mu            sync.Mutex
}

func (f *Filesystem) EachObject(fn func(Object) error) error {
	var eachErr error
	tools.FastWalkGitRepo(f.LFSObjectDir(), func(parentDir string, info os.FileInfo, err error) {
		if err != nil {
			eachErr = err
			return
		}
		if eachErr != nil || info.IsDir() {
			return
		}
		if oidRE.MatchString(info.Name()) {
			fn(Object{Oid: info.Name(), Size: info.Size()})
		}
	})
	return eachErr
}

func (f *Filesystem) ObjectExists(oid string, size int64) bool {
	return tools.FileExistsOfSize(f.ObjectPathname(oid), size)
}

func (f *Filesystem) ObjectPath(oid string) (string, error) {
	dir := f.localObjectDir(oid)
	if err := os.MkdirAll(dir, 0755); err != nil {
		return "", fmt.Errorf("Error trying to create local storage directory in %q: %s", dir, err)
	}
	return filepath.Join(dir, oid), nil
}

func (f *Filesystem) ObjectPathname(oid string) string {
	return filepath.Join(f.localObjectDir(oid), oid)
}

func (f *Filesystem) DecodePathname(path string) string {
	return string(DecodePathBytes([]byte(path)))
}

/**
 * Revert non ascii chracters escaped by git or windows (as octal sequences \000) back to bytes.
 */
func DecodePathBytes(path []byte) []byte {
	var expression = regexp.MustCompile(`\\[0-9]{3}`)
	var buffer bytes.Buffer

	// strip quotes if any
	if len(path) > 2 && path[0] == '"' && path[len(path)-1] == '"' {
		path = path[1 : len(path)-1]
	}

	base := 0
	for _, submatches := range expression.FindAllSubmatchIndex(path, -1) {
		buffer.Write(path[base:submatches[0]])

		match := string(path[submatches[0]+1 : submatches[0]+4])

		k, err := strconv.ParseUint(match, 8, 64)
		if err != nil {
			return path
		} // abort on error

		buffer.Write([]byte{byte(k)})
		base = submatches[1]
	}

	buffer.Write(path[base:len(path)])

	return buffer.Bytes()
}

func (f *Filesystem) localObjectDir(oid string) string {
	return filepath.Join(f.LFSObjectDir(), oid[0:2], oid[2:4])
}

func (f *Filesystem) ObjectReferencePaths(oid string) []string {
	if len(f.ReferenceDirs) == 0 {
		return nil
	}

	var paths []string
	for _, ref := range f.ReferenceDirs {
		paths = append(paths, filepath.Join(ref, oid[0:2], oid[2:4], oid))
	}
	return paths
}

func (f *Filesystem) LFSObjectDir() string {
	f.mu.Lock()
	defer f.mu.Unlock()

	if len(f.lfsobjdir) == 0 {
		f.lfsobjdir = filepath.Join(f.LFSStorageDir, "objects")
		os.MkdirAll(f.lfsobjdir, 0755)
	}

	return f.lfsobjdir
}

func (f *Filesystem) LogDir() string {
	f.mu.Lock()
	defer f.mu.Unlock()

	if len(f.logdir) == 0 {
		f.logdir = filepath.Join(f.LFSStorageDir, "logs")
		os.MkdirAll(f.logdir, 0755)
	}

	return f.logdir
}

func (f *Filesystem) TempDir() string {
	f.mu.Lock()
	defer f.mu.Unlock()

	if len(f.tmpdir) == 0 {
		f.tmpdir = filepath.Join(f.LFSStorageDir, "tmp")
		os.MkdirAll(f.tmpdir, 0755)
	}

	return f.tmpdir
}

func (f *Filesystem) Cleanup() error {
	if f == nil {
		return nil
	}
	return f.cleanupTmp()
}

// New initializes a new *Filesystem with the given directories. gitdir is the
// path to the bare repo, workdir is the path to the repository working
// directory, and lfsdir is the optional path to the `.git/lfs` directory.
func New(env Environment, gitdir, workdir, lfsdir string) *Filesystem {
	fs := &Filesystem{
		GitStorageDir: resolveGitStorageDir(gitdir),
	}

	fs.ReferenceDirs = resolveReferenceDirs(env, fs.GitStorageDir)

	if len(lfsdir) == 0 {
		lfsdir = "lfs"
	}

	if filepath.IsAbs(lfsdir) {
		fs.LFSStorageDir = lfsdir
	} else {
		fs.LFSStorageDir = filepath.Join(fs.GitStorageDir, lfsdir)
	}

	return fs
}

func resolveReferenceDirs(env Environment, gitStorageDir string) []string {
	var references []string

	envAlternates, ok := env.Get("GIT_ALTERNATE_OBJECT_DIRECTORIES")
	if ok {
		splits := strings.Split(envAlternates, string(os.PathListSeparator))
		for _, split := range splits {
			if dir, ok := existsAlternate(split); ok {
				references = append(references, dir)
			}
		}
	}

	cloneReferencePath := filepath.Join(gitStorageDir, "objects", "info", "alternates")
	if tools.FileExists(cloneReferencePath) {
		f, err := os.Open(cloneReferencePath)
		if err != nil {
			tracerx.Printf("could not open %s: %s",
				cloneReferencePath, err)
			return nil
		}
		defer f.Close()

		scanner := bufio.NewScanner(f)
		for scanner.Scan() {
			text := strings.TrimSpace(scanner.Text())
			if len(text) == 0 || strings.HasPrefix(text, "#") {
				continue
			}

			if dir, ok := existsAlternate(text); ok {
				references = append(references, dir)
			}
		}

		if err := scanner.Err(); err != nil {
			tracerx.Printf("could not scan %s: %s",
				cloneReferencePath, err)
		}
	}
	return references
}

// existsAlternate takes an object directory given in "objs" (read as a single,
// line from .git/objects/info/alternates). If that is a satisfiable alternates
// directory (i.e., it exists), the directory is returned along with "true". If
// not, the empty string and false is returned instead.
func existsAlternate(objs string) (string, bool) {
	objs = strings.TrimSpace(objs)
	if strings.HasPrefix(objs, "\"") {
		var err error

		unquote := strings.LastIndex(objs, "\"")
		if unquote == 0 {
			return "", false
		}

		objs, err = strconv.Unquote(objs[:unquote+1])
		if err != nil {
			return "", false
		}
	}

	storage := filepath.Join(filepath.Dir(objs), "lfs", "objects")

	if tools.DirExists(storage) {
		return storage, true
	}
	return "", false
}

// From a git dir, get the location that objects are to be stored (we will store lfs alongside)
// Sometimes there is an additional level of redirect on the .git folder by way of a commondir file
// before you find object storage, e.g. 'git worktree' uses this. It redirects to gitdir either by GIT_DIR
// (during setup) or .git/git-dir: (during use), but this only contains the index etc, the objects
// are found in another git dir via 'commondir'.
func resolveGitStorageDir(gitDir string) string {
	commondirpath := filepath.Join(gitDir, "commondir")
	if tools.FileExists(commondirpath) && !tools.DirExists(filepath.Join(gitDir, "objects")) {
		// no git-dir: prefix in commondir
		storage, err := processGitRedirectFile(commondirpath, "")
		if err == nil {
			return storage
		}
	}
	return gitDir
}

func processGitRedirectFile(file, prefix string) (string, error) {
	data, err := ioutil.ReadFile(file)
	if err != nil {
		return "", err
	}

	contents := string(data)
	var dir string
	if len(prefix) > 0 {
		if !strings.HasPrefix(contents, prefix) {
			// Prefix required & not found
			return "", nil
		}
		dir = strings.TrimSpace(contents[len(prefix):])
	} else {
		dir = strings.TrimSpace(contents)
	}

	if !filepath.IsAbs(dir) {
		// The .git file contains a relative path.
		// Create an absolute path based on the directory the .git file is located in.
		dir = filepath.Join(filepath.Dir(file), dir)
	}

	return dir, nil
}
<|endoftext|>


<commit_before>// Package fs is a generic file system interface for rclone object storage systems
package fs

import (
	"fmt"
	"io"
	"log"
	"path/filepath"
	"regexp"
	"time"
)

// Constants
const (
	// UserAgent for Fs which can set it
	UserAgent = "rclone/" + Version
	// ModTimeNotSupported is a very large precision value to show
	// mod time isn't supported on this Fs
	ModTimeNotSupported = 100 * 365 * 24 * time.Hour
)

// Globals
var (
	// Filesystem registry
	fsRegistry []*Info
	// ErrorNotFoundInConfigFile is returned by NewFs if not found in config file
	ErrorNotFoundInConfigFile = fmt.Errorf("Didn't find section in config file")
	ErrorCantPurge            = fmt.Errorf("Can't purge directory")
	ErrorCantCopy             = fmt.Errorf("Can't copy object - incompatible remotes")
	ErrorCantMove             = fmt.Errorf("Can't copy object - incompatible remotes")
	ErrorCantDirMove          = fmt.Errorf("Can't copy directory - incompatible remotes")
	ErrorDirExists            = fmt.Errorf("Can't copy directory - destination already exists")
)

// Info information about a filesystem
type Info struct {
	// Name of this fs
	Name string
	// Create a new file system.  If root refers to an existing
	// object, then it should return a Fs which only returns that
	// object.
	NewFs func(name string, root string) (Fs, error)
	// Function to call to help with config
	Config func(string)
	// Options for the Fs configuration
	Options []Option
}

// Option is describes an option for the config wizard
type Option struct {
	Name     string
	Help     string
	Optional bool
	Examples []OptionExample
}

// OptionExample describes an example for an Option
type OptionExample struct {
	Value string
	Help  string
}

// Register a filesystem
//
// Fs modules  should use this in an init() function
func Register(info *Info) {
	fsRegistry = append(fsRegistry, info)
}

// Fs is the interface a cloud storage system must provide
type Fs interface {
	// Name of the remote (as passed into NewFs)
	Name() string

	// Root of the remote (as passed into NewFs)
	Root() string

	// String returns a description of the FS
	String() string

	// List the Fs into a channel
	List() ObjectsChan

	// ListDir lists the Fs directories/buckets/containers into a channel
	ListDir() DirChan

	// NewFsObject finds the Object at remote.  Returns nil if can't be found
	NewFsObject(remote string) Object

	// Put in to the remote path with the modTime given of the given size
	//
	// May create the object even if it returns an error - if so
	// will return the object and the error, otherwise will return
	// nil and the error
	Put(in io.Reader, remote string, modTime time.Time, size int64) (Object, error)

	// Mkdir makes the directory (container, bucket)
	//
	// Shouldn't return an error if it already exists
	Mkdir() error

	// Rmdir removes the directory (container, bucket) if empty
	//
	// Return an error if it doesn't exist or isn't empty
	Rmdir() error

	// Precision of the ModTimes in this Fs
	Precision() time.Duration

	// Returns the supported hash types of the filesystem
	Hashes() HashSet
}

// Object is a filesystem like object provided by an Fs
type Object interface {
	// String returns a description of the Object
	String() string

	// Fs returns the Fs that this object is part of
	Fs() Fs

	// Remote returns the remote path
	Remote() string

	// Md5sum returns the md5 checksum of the file
	// If no Md5sum is available it returns ""
	Hash(HashType) (string, error)

	// ModTime returns the modification date of the file
	// It should return a best guess if one isn't available
	ModTime() time.Time

	// SetModTime sets the metadata on the object to set the modification date
	SetModTime(time.Time)

	// Size returns the size of the file
	Size() int64

	// Open opens the file for read.  Call Close() on the returned io.ReadCloser
	Open() (io.ReadCloser, error)

	// Update in to the object with the modTime given of the given size
	Update(in io.Reader, modTime time.Time, size int64) error

	// Storable says whether this object can be stored
	Storable() bool

	// Removes this object
	Remove() error
}

// Purger is an optional interfaces for Fs
type Purger interface {
	// Purge all files in the root and the root directory
	//
	// Implement this if you have a way of deleting all the files
	// quicker than just running Remove() on the result of List()
	//
	// Return an error if it doesn't exist
	Purge() error
}

// Copier is an optional interface for Fs
type Copier interface {
	// Copy src to this remote using server side copy operations.
	//
	// This is stored with the remote path given
	//
	// It returns the destination Object and a possible error
	//
	// Will only be called if src.Fs().Name() == f.Name()
	//
	// If it isn't possible then return fs.ErrorCantCopy
	Copy(src Object, remote string) (Object, error)
}

// Mover is an optional interface for Fs
type Mover interface {
	// Move src to this remote using server side move operations.
	//
	// This is stored with the remote path given
	//
	// It returns the destination Object and a possible error
	//
	// Will only be called if src.Fs().Name() == f.Name()
	//
	// If it isn't possible then return fs.ErrorCantMove
	Move(src Object, remote string) (Object, error)
}

// DirMover is an optional interface for Fs
type DirMover interface {
	// DirMove moves src to this remote using server side move
	// operations.
	//
	// Will only be called if src.Fs().Name() == f.Name()
	//
	// If it isn't possible then return fs.ErrorCantDirMove
	//
	// If destination exists then return fs.ErrorDirExists
	DirMove(src Fs) error
}

// UnWrapper is an optional interfaces for Fs
type UnWrapper interface {
	// UnWrap returns the Fs that this Fs is wrapping
	UnWrap() Fs
}

// ObjectsChan is a channel of Objects
type ObjectsChan chan Object

// Objects is a slice of Object~s
type Objects []Object

// ObjectPair is a pair of Objects used to describe a potential copy
// operation.
type ObjectPair struct {
	src, dst Object
}

// ObjectPairChan is a channel of ObjectPair
type ObjectPairChan chan ObjectPair

// Dir describes a directory for directory/container/bucket lists
type Dir struct {
	Name  string    // name of the directory
	When  time.Time // modification or creation time - IsZero for unknown
	Bytes int64     // size of directory and contents -1 for unknown
	Count int64     // number of objects -1 for unknown
}

// DirChan is a channel of Dir objects
type DirChan chan *Dir

// Find looks for an Info object for the name passed in
//
// Services are looked up in the config file
func Find(name string) (*Info, error) {
	for _, item := range fsRegistry {
		if item.Name == name {
			return item, nil
		}
	}
	return nil, fmt.Errorf("Didn't find filing system for %q", name)
}

// Pattern to match an rclone url
var matcher = regexp.MustCompile(`^([\w_ -]+):(.*)$`)

// NewFs makes a new Fs object from the path
//
// The path is of the form remote:path
//
// Remotes are looked up in the config file.  If the remote isn't
// found then NotFoundInConfigFile will be returned.
//
// On Windows avoid single character remote names as they can be mixed
// up with drive letters.
func NewFs(path string) (Fs, error) {
	parts := matcher.FindStringSubmatch(path)
	fsName, configName, fsPath := "local", "local", path
	if parts != nil && !isDriveLetter(parts[1]) {
		configName, fsPath = parts[1], parts[2]
		var err error
		fsName, err = ConfigFile.GetValue(configName, "type")
		if err != nil {
			return nil, ErrorNotFoundInConfigFile
		}
	}
	fs, err := Find(fsName)
	if err != nil {
		return nil, err
	}
	// change native directory separators to / if there are any
	fsPath = filepath.ToSlash(fsPath)
	return fs.NewFs(configName, fsPath)
}

// OutputLog logs for an object
func OutputLog(o interface{}, text string, args ...interface{}) {
	description := ""
	if o != nil {
		description = fmt.Sprintf("%v: ", o)
	}
	out := fmt.Sprintf(text, args...)
	log.Print(description + out)
}

// Debug writes debuging output for this Object or Fs
func Debug(o interface{}, text string, args ...interface{}) {
	if Config.Verbose {
		OutputLog(o, text, args...)
	}
}

// Log writes log output for this Object or Fs
func Log(o interface{}, text string, args ...interface{}) {
	if !Config.Quiet {
		OutputLog(o, text, args...)
	}
}

// ErrorLog writes error log output for this Object or Fs.  It
// unconditionally logs a message regardless of Config.Quiet or
// Config.Verbose.
func ErrorLog(o interface{}, text string, args ...interface{}) {
	OutputLog(o, text, args...)
}

// CheckClose is a utility function used to check the return from
// Close in a defer statement.
func CheckClose(c io.Closer, err *error) {
	cerr := c.Close()
	if *err == nil {
		*err = cerr
	}
}
<commit_msg>Fix typo in error strings<commit_after>// Package fs is a generic file system interface for rclone object storage systems
package fs

import (
	"fmt"
	"io"
	"log"
	"path/filepath"
	"regexp"
	"time"
)

// Constants
const (
	// UserAgent for Fs which can set it
	UserAgent = "rclone/" + Version
	// ModTimeNotSupported is a very large precision value to show
	// mod time isn't supported on this Fs
	ModTimeNotSupported = 100 * 365 * 24 * time.Hour
)

// Globals
var (
	// Filesystem registry
	fsRegistry []*Info
	// ErrorNotFoundInConfigFile is returned by NewFs if not found in config file
	ErrorNotFoundInConfigFile = fmt.Errorf("Didn't find section in config file")
	ErrorCantPurge            = fmt.Errorf("Can't purge directory")
	ErrorCantCopy             = fmt.Errorf("Can't copy object - incompatible remotes")
	ErrorCantMove             = fmt.Errorf("Can't move object - incompatible remotes")
	ErrorCantDirMove          = fmt.Errorf("Can't move directory - incompatible remotes")
	ErrorDirExists            = fmt.Errorf("Can't copy directory - destination already exists")
)

// Info information about a filesystem
type Info struct {
	// Name of this fs
	Name string
	// Create a new file system.  If root refers to an existing
	// object, then it should return a Fs which only returns that
	// object.
	NewFs func(name string, root string) (Fs, error)
	// Function to call to help with config
	Config func(string)
	// Options for the Fs configuration
	Options []Option
}

// Option is describes an option for the config wizard
type Option struct {
	Name     string
	Help     string
	Optional bool
	Examples []OptionExample
}

// OptionExample describes an example for an Option
type OptionExample struct {
	Value string
	Help  string
}

// Register a filesystem
//
// Fs modules  should use this in an init() function
func Register(info *Info) {
	fsRegistry = append(fsRegistry, info)
}

// Fs is the interface a cloud storage system must provide
type Fs interface {
	// Name of the remote (as passed into NewFs)
	Name() string

	// Root of the remote (as passed into NewFs)
	Root() string

	// String returns a description of the FS
	String() string

	// List the Fs into a channel
	List() ObjectsChan

	// ListDir lists the Fs directories/buckets/containers into a channel
	ListDir() DirChan

	// NewFsObject finds the Object at remote.  Returns nil if can't be found
	NewFsObject(remote string) Object

	// Put in to the remote path with the modTime given of the given size
	//
	// May create the object even if it returns an error - if so
	// will return the object and the error, otherwise will return
	// nil and the error
	Put(in io.Reader, remote string, modTime time.Time, size int64) (Object, error)

	// Mkdir makes the directory (container, bucket)
	//
	// Shouldn't return an error if it already exists
	Mkdir() error

	// Rmdir removes the directory (container, bucket) if empty
	//
	// Return an error if it doesn't exist or isn't empty
	Rmdir() error

	// Precision of the ModTimes in this Fs
	Precision() time.Duration

	// Returns the supported hash types of the filesystem
	Hashes() HashSet
}

// Object is a filesystem like object provided by an Fs
type Object interface {
	// String returns a description of the Object
	String() string

	// Fs returns the Fs that this object is part of
	Fs() Fs

	// Remote returns the remote path
	Remote() string

	// Md5sum returns the md5 checksum of the file
	// If no Md5sum is available it returns ""
	Hash(HashType) (string, error)

	// ModTime returns the modification date of the file
	// It should return a best guess if one isn't available
	ModTime() time.Time

	// SetModTime sets the metadata on the object to set the modification date
	SetModTime(time.Time)

	// Size returns the size of the file
	Size() int64

	// Open opens the file for read.  Call Close() on the returned io.ReadCloser
	Open() (io.ReadCloser, error)

	// Update in to the object with the modTime given of the given size
	Update(in io.Reader, modTime time.Time, size int64) error

	// Storable says whether this object can be stored
	Storable() bool

	// Removes this object
	Remove() error
}

// Purger is an optional interfaces for Fs
type Purger interface {
	// Purge all files in the root and the root directory
	//
	// Implement this if you have a way of deleting all the files
	// quicker than just running Remove() on the result of List()
	//
	// Return an error if it doesn't exist
	Purge() error
}

// Copier is an optional interface for Fs
type Copier interface {
	// Copy src to this remote using server side copy operations.
	//
	// This is stored with the remote path given
	//
	// It returns the destination Object and a possible error
	//
	// Will only be called if src.Fs().Name() == f.Name()
	//
	// If it isn't possible then return fs.ErrorCantCopy
	Copy(src Object, remote string) (Object, error)
}

// Mover is an optional interface for Fs
type Mover interface {
	// Move src to this remote using server side move operations.
	//
	// This is stored with the remote path given
	//
	// It returns the destination Object and a possible error
	//
	// Will only be called if src.Fs().Name() == f.Name()
	//
	// If it isn't possible then return fs.ErrorCantMove
	Move(src Object, remote string) (Object, error)
}

// DirMover is an optional interface for Fs
type DirMover interface {
	// DirMove moves src to this remote using server side move
	// operations.
	//
	// Will only be called if src.Fs().Name() == f.Name()
	//
	// If it isn't possible then return fs.ErrorCantDirMove
	//
	// If destination exists then return fs.ErrorDirExists
	DirMove(src Fs) error
}

// UnWrapper is an optional interfaces for Fs
type UnWrapper interface {
	// UnWrap returns the Fs that this Fs is wrapping
	UnWrap() Fs
}

// ObjectsChan is a channel of Objects
type ObjectsChan chan Object

// Objects is a slice of Object~s
type Objects []Object

// ObjectPair is a pair of Objects used to describe a potential copy
// operation.
type ObjectPair struct {
	src, dst Object
}

// ObjectPairChan is a channel of ObjectPair
type ObjectPairChan chan ObjectPair

// Dir describes a directory for directory/container/bucket lists
type Dir struct {
	Name  string    // name of the directory
	When  time.Time // modification or creation time - IsZero for unknown
	Bytes int64     // size of directory and contents -1 for unknown
	Count int64     // number of objects -1 for unknown
}

// DirChan is a channel of Dir objects
type DirChan chan *Dir

// Find looks for an Info object for the name passed in
//
// Services are looked up in the config file
func Find(name string) (*Info, error) {
	for _, item := range fsRegistry {
		if item.Name == name {
			return item, nil
		}
	}
	return nil, fmt.Errorf("Didn't find filing system for %q", name)
}

// Pattern to match an rclone url
var matcher = regexp.MustCompile(`^([\w_ -]+):(.*)$`)

// NewFs makes a new Fs object from the path
//
// The path is of the form remote:path
//
// Remotes are looked up in the config file.  If the remote isn't
// found then NotFoundInConfigFile will be returned.
//
// On Windows avoid single character remote names as they can be mixed
// up with drive letters.
func NewFs(path string) (Fs, error) {
	parts := matcher.FindStringSubmatch(path)
	fsName, configName, fsPath := "local", "local", path
	if parts != nil && !isDriveLetter(parts[1]) {
		configName, fsPath = parts[1], parts[2]
		var err error
		fsName, err = ConfigFile.GetValue(configName, "type")
		if err != nil {
			return nil, ErrorNotFoundInConfigFile
		}
	}
	fs, err := Find(fsName)
	if err != nil {
		return nil, err
	}
	// change native directory separators to / if there are any
	fsPath = filepath.ToSlash(fsPath)
	return fs.NewFs(configName, fsPath)
}

// OutputLog logs for an object
func OutputLog(o interface{}, text string, args ...interface{}) {
	description := ""
	if o != nil {
		description = fmt.Sprintf("%v: ", o)
	}
	out := fmt.Sprintf(text, args...)
	log.Print(description + out)
}

// Debug writes debuging output for this Object or Fs
func Debug(o interface{}, text string, args ...interface{}) {
	if Config.Verbose {
		OutputLog(o, text, args...)
	}
}

// Log writes log output for this Object or Fs
func Log(o interface{}, text string, args ...interface{}) {
	if !Config.Quiet {
		OutputLog(o, text, args...)
	}
}

// ErrorLog writes error log output for this Object or Fs.  It
// unconditionally logs a message regardless of Config.Quiet or
// Config.Verbose.
func ErrorLog(o interface{}, text string, args ...interface{}) {
	OutputLog(o, text, args...)
}

// CheckClose is a utility function used to check the return from
// Close in a defer statement.
func CheckClose(c io.Closer, err *error) {
	cerr := c.Close()
	if *err == nil {
		*err = cerr
	}
}
<|endoftext|>