<commit_before>package repository

import (
	"fmt"
	"io"
	"time"

	"github.com/jmoiron/sqlx"

	"bufio"

	_ "github.com/lib/pq"
	"github.com/syou6162/go-active-learning/lib/feature"
	"github.com/syou6162/go-active-learning/lib/model"
	"github.com/syou6162/go-active-learning/lib/util"
)

type Repository interface {
	InsertOrUpdateExample(e *model.Example) error
	InsertExampleFromScanner(scanner *bufio.Scanner) (*model.Example, error)
	InsertExamplesFromReader(reader io.Reader) error
	ReadExamples() (model.Examples, error)
	ReadRecentExamples(from time.Time) (model.Examples, error)
	ReadExamplesByLabel(label model.LabelType, limit int) (model.Examples, error)
	ReadLabeledExamples(limit int) (model.Examples, error)
	ReadPositiveExamples(limit int) (model.Examples, error)
	ReadNegativeExamples(limit int) (model.Examples, error)
	ReadUnlabeledExamples(limit int) (model.Examples, error)
	FindExampleByUlr(url string) (*model.Example, error)
	SearchExamplesByUlrs(urls []string) (model.Examples, error)
	SearchExamplesByKeywords(keywords []string, aggregator string, limit int) (model.Examples, error)
	DeleteAllExamples() error

	UpdateFeatureVector(e *model.Example) error
	FindFeatureVector(e *model.Example) (feature.FeatureVector, error)
	SearchFeatureVector(examples model.Examples) ([]feature.FeatureVector, error)

	UpdateHatenaBookmark(e *model.Example) error
	SearchHatenaBookmarks(examples model.Examples) ([]*model.HatenaBookmark, error)
	FindHatenaBookmark(e *model.Example) (*model.HatenaBookmark, error)

	Ping() error
	Close() error
}

type repository struct {
	db *sqlx.DB
}

func GetDataSourceName() string {
	host := util.GetEnv("POSTGRES_HOST", "localhost")
	dbUser := util.GetEnv("DB_USER", "nobody")
	dbPassword := util.GetEnv("DB_PASSWORD", "nobody")
	dbName := util.GetEnv("DB_NAME", "go-active-learning")
	return fmt.Sprintf(
		"host=%s user=%s password=%s dbname=%s sslmode=disable",
		host, dbUser, dbPassword, dbName,
	)
}

func New() (*repository, error) {
	db, err := sqlx.Open("postgres", GetDataSourceName())
	if err != nil {
		return nil, err
	}
	db.SetMaxOpenConns(50)
	return &repository{db: db}, nil
}

func (r *repository) Ping() error {
	return r.db.Ping()
}

func (r *repository) Close() error {
	if r.db != nil {
		return r.db.Close()
	} else {
		return nil
	}
}
<commit_msg>インターフェイス公開<commit_after>package repository

import (
	"fmt"
	"io"
	"time"

	"github.com/jmoiron/sqlx"

	"bufio"

	_ "github.com/lib/pq"
	"github.com/syou6162/go-active-learning/lib/feature"
	"github.com/syou6162/go-active-learning/lib/model"
	"github.com/syou6162/go-active-learning/lib/util"
)

type Repository interface {
	InsertOrUpdateExample(e *model.Example) error
	InsertExampleFromScanner(scanner *bufio.Scanner) (*model.Example, error)
	InsertExamplesFromReader(reader io.Reader) error
	ReadExamples() (model.Examples, error)
	ReadRecentExamples(from time.Time) (model.Examples, error)
	ReadExamplesByLabel(label model.LabelType, limit int) (model.Examples, error)
	ReadLabeledExamples(limit int) (model.Examples, error)
	ReadPositiveExamples(limit int) (model.Examples, error)
	ReadNegativeExamples(limit int) (model.Examples, error)
	ReadUnlabeledExamples(limit int) (model.Examples, error)
	FindExampleByUlr(url string) (*model.Example, error)
	SearchExamplesByUlrs(urls []string) (model.Examples, error)
	SearchExamplesByKeywords(keywords []string, aggregator string, limit int) (model.Examples, error)
	DeleteAllExamples() error

	UpdateFeatureVector(e *model.Example) error
	FindFeatureVector(e *model.Example) (feature.FeatureVector, error)
	SearchFeatureVector(examples model.Examples) ([]feature.FeatureVector, error)

	UpdateHatenaBookmark(e *model.Example) error
	SearchHatenaBookmarks(examples model.Examples) ([]*model.HatenaBookmark, error)
	FindHatenaBookmark(e *model.Example) (*model.HatenaBookmark, error)

	UpdateReferringTweets(e *model.Example) error
	SearchReferringTweetsList(examples model.Examples) (map[int]model.ReferringTweets, error)
	FindReferringTweets(e *model.Example) (model.ReferringTweets, error)

	Ping() error
	Close() error
}

type repository struct {
	db *sqlx.DB
}

func GetDataSourceName() string {
	host := util.GetEnv("POSTGRES_HOST", "localhost")
	dbUser := util.GetEnv("DB_USER", "nobody")
	dbPassword := util.GetEnv("DB_PASSWORD", "nobody")
	dbName := util.GetEnv("DB_NAME", "go-active-learning")
	return fmt.Sprintf(
		"host=%s user=%s password=%s dbname=%s sslmode=disable",
		host, dbUser, dbPassword, dbName,
	)
}

func New() (*repository, error) {
	db, err := sqlx.Open("postgres", GetDataSourceName())
	if err != nil {
		return nil, err
	}
	db.SetMaxOpenConns(50)
	return &repository{db: db}, nil
}

func (r *repository) Ping() error {
	return r.db.Ping()
}

func (r *repository) Close() error {
	if r.db != nil {
		return r.db.Close()
	} else {
		return nil
	}
}
<|endoftext|>


<commit_before>// socket_bsd.go -- Socket handling specific to *BSD based systems.

// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package syscall

const SizeofSockaddrInet4 = 16
const SizeofSockaddrInet6 = 28
const SizeofSockaddrUnix = 110

type RawSockaddrInet4 struct {
	Len uint8;
	Family uint8;
	Port uint16;
	Addr [4]byte /* in_addr */;
	Zero [8]uint8;
}

func (sa *RawSockaddrInet4) setLen() Socklen_t {
	sa.Len = SizeofSockaddrInet4
	return SizeofSockaddrInet4
}

type RawSockaddrInet6 struct {
	Len uint8;
	Family uint8;
	Port uint16;
	Flowinfo uint32;
	Addr [16]byte /* in6_addr */;
	Scope_id uint32;
}

func (sa *RawSockaddrInet6) setLen() Socklen_t {
	sa.raw.Len = SizeofSockaddrInet6
	return SizeofSockaddrInet6
}

type RawSockaddrUnix struct {
	Len uint8;
	Family uint8;
	Path [108]int8;
}

func (sa *RawSockaddrUnix) setLen(n int) {
	sa.Len = uint8(3 + n) // 2 for Family, Len; 1 for NUL.
}

func (sa *RawSockaddrUnix) getLen() (int, int) {
	if sa.Len < 3 || sa.Len > SizeofSockaddrUnix {
		return 0, EINVAL
	}
	n := int(sa.Len) - 3 // subtract leading Family, Len, terminating NUL.
	for i := 0; i < n; i++ {
		if sa.Path[i] == 0 {
			// found early NUL; assume Len is overestimating.
			n = i
			break
		}
	}
	return n, 0
}

type RawSockaddr struct {
	Len uint8;
	Family uint8;
	Data [14]int8;
}

// BindToDevice binds the socket associated with fd to device.
func BindToDevice(fd int, device string) (errno int) {
	return ENOSYS
}
<commit_msg>Fix typo in socket_bsd.go.<commit_after>// socket_bsd.go -- Socket handling specific to *BSD based systems.

// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package syscall

const SizeofSockaddrInet4 = 16
const SizeofSockaddrInet6 = 28
const SizeofSockaddrUnix = 110

type RawSockaddrInet4 struct {
	Len uint8;
	Family uint8;
	Port uint16;
	Addr [4]byte /* in_addr */;
	Zero [8]uint8;
}

func (sa *RawSockaddrInet4) setLen() Socklen_t {
	sa.Len = SizeofSockaddrInet4
	return SizeofSockaddrInet4
}

type RawSockaddrInet6 struct {
	Len uint8;
	Family uint8;
	Port uint16;
	Flowinfo uint32;
	Addr [16]byte /* in6_addr */;
	Scope_id uint32;
}

func (sa *RawSockaddrInet6) setLen() Socklen_t {
	sa.Len = SizeofSockaddrInet6
	return SizeofSockaddrInet6
}

type RawSockaddrUnix struct {
	Len uint8;
	Family uint8;
	Path [108]int8;
}

func (sa *RawSockaddrUnix) setLen(n int) {
	sa.Len = uint8(3 + n) // 2 for Family, Len; 1 for NUL.
}

func (sa *RawSockaddrUnix) getLen() (int, int) {
	if sa.Len < 3 || sa.Len > SizeofSockaddrUnix {
		return 0, EINVAL
	}
	n := int(sa.Len) - 3 // subtract leading Family, Len, terminating NUL.
	for i := 0; i < n; i++ {
		if sa.Path[i] == 0 {
			// found early NUL; assume Len is overestimating.
			n = i
			break
		}
	}
	return n, 0
}

type RawSockaddr struct {
	Len uint8;
	Family uint8;
	Data [14]int8;
}

// BindToDevice binds the socket associated with fd to device.
func BindToDevice(fd int, device string) (errno int) {
	return ENOSYS
}
<|endoftext|>


<commit_before>package libkbfs

import (
	"fmt"
	"sync"

	"github.com/keybase/client/go/logger"
	"golang.org/x/net/context"
)

// CtxCRTagKey is the type used for unique context tags related to
// conflict resolution
type CtxCRTagKey int

const (
	// CtxCRIDKey is the type of the tag for unique operation IDs
	// related to conflict resolution
	CtxCRIDKey CtxCRTagKey = iota
)

// CtxCROpID is the display name for the unique operation
// conflict resolution ID tag.
const CtxCROpID = "CRID"

type conflictInput struct {
	unmerged MetadataRevision
	merged   MetadataRevision
}

// ConflictResolver is responsible for resolving conflicts in the
// background.
type ConflictResolver struct {
	config     Config
	fbo        *FolderBranchOps
	inputChan  chan conflictInput
	inputGroup sync.WaitGroup
	log        logger.Logger

	shutdown     bool
	shutdownLock sync.RWMutex

	currInput conflictInput
	inputLock sync.Mutex
}

// NewConflictResolver constructs a new ConflictResolver (and launches
// any necessary background goroutines).
func NewConflictResolver(
	config Config, fbo *FolderBranchOps) *ConflictResolver {
	// make a logger with an appropriate module name
	branchSuffix := ""
	if fbo.branch() != MasterBranch {
		branchSuffix = " " + string(fbo.branch())
	}
	tlfStringFull := fbo.id().String()
	log := config.MakeLogger(fmt.Sprintf("CR %s%s", tlfStringFull[:8],
		branchSuffix))

	cr := &ConflictResolver{
		config:    config,
		fbo:       fbo,
		inputChan: make(chan conflictInput),
		log:       log,
		currInput: conflictInput{
			unmerged: MetadataRevisionUninitialized,
			merged:   MetadataRevisionUninitialized,
		},
	}

	go cr.processInput()
	return cr
}

func (cr *ConflictResolver) processInput() {
	logTags := make(logger.CtxLogTags)
	logTags[CtxCRIDKey] = CtxCROpID
	backgroundCtx := logger.NewContextWithLogTags(context.Background(), logTags)

	var cancel func()
	defer func() {
		if cancel != nil {
			cancel()
		}
	}()
	for ci := range cr.inputChan {
		ctx := backgroundCtx
		id, err := MakeRandomRequestID()
		if err != nil {
			cr.log.Warning("Couldn't generate a random request ID: %v", err)
		} else {
			ctx = context.WithValue(ctx, CtxCRIDKey, id)
		}

		valid := func() bool {
			cr.inputLock.Lock()
			defer cr.inputLock.Unlock()
			// The input is only interesting if one of the revisions
			// is greater than what we've looked at to date.
			if ci.unmerged <= cr.currInput.unmerged &&
				ci.merged <= cr.currInput.merged {
				return false
			}
			cr.log.CDebugf(ctx, "New conflict input %v following old "+
				"input %v", ci, cr.currInput)
			cr.currInput = ci
			// cancel the existing conflict resolution (if any)
			if cancel != nil {
				cancel()
			}
			return true
		}()
		if !valid {
			cr.log.CDebugf(ctx, "Ignoring uninteresting input: %v", ci)
			cr.inputGroup.Done()
			continue
		}

		ctx, cancel = context.WithCancel(ctx)
		go cr.doResolve(ctx, ci)
	}
}

// Resolve takes the latest known unmerged and merged revision
// numbers, and kicks off the resolution process.
func (cr *ConflictResolver) Resolve(unmerged MetadataRevision,
	merged MetadataRevision) {
	cr.shutdownLock.RLock()
	defer cr.shutdownLock.RUnlock()
	if cr.shutdown {
		return
	}

	cr.inputGroup.Add(1)
	cr.inputChan <- conflictInput{unmerged, merged}
}

// Wait blocks until the current set of submitted resolutions are
// complete (though not necessarily successful), or until the given
// context is canceled.
func (cr *ConflictResolver) Wait(ctx context.Context) error {
	c := make(chan struct{}, 1)
	go func() {
		cr.inputGroup.Wait()
		c <- struct{}{}
	}()

	select {
	case <-c:
		return nil
	case <-ctx.Done():
		return ctx.Err()
	}
}

// Shutdown cancels any ongoing resolutions and stops any background
// goroutines.
func (cr *ConflictResolver) Shutdown() {
	cr.shutdownLock.Lock()
	defer cr.shutdownLock.Unlock()
	cr.shutdown = true
	close(cr.inputChan)
}

func (cr *ConflictResolver) checkDone(ctx context.Context) error {
	select {
	case <-ctx.Done():
		return ctx.Err()
	default:
		return nil
	}
}

func (cr *ConflictResolver) getMDs(ctx context.Context) (
	unmerged []*RootMetadata, merged []*RootMetadata, err error) {
	// first get all outstanding unmerged MDs for this device
	branchPoint, unmerged, err := cr.fbo.getUnmergedMDUpdates(ctx)
	if err != nil {
		return nil, nil, err
	}

	// now get all the merged MDs, starting from after the branch point
	merged, err = getMergedMDUpdates(ctx, cr.fbo.config, cr.fbo.id(),
		branchPoint+1)
	if err != nil {
		return nil, nil, err
	}

	// re-embed all the block changes
	err = cr.fbo.reembedBlockChanges(ctx, unmerged)
	if err != nil {
		return nil, nil, err
	}
	err = cr.fbo.reembedBlockChanges(ctx, merged)
	if err != nil {
		return nil, nil, err
	}

	return unmerged, merged, nil
}

func (cr *ConflictResolver) updateCurrInput(ctx context.Context,
	unmerged []*RootMetadata, merged []*RootMetadata) (err error) {
	cr.inputLock.Lock()
	defer cr.inputLock.Unlock()
	// check done while holding the lock, so we know for sure if
	// we've already been canceled and replaced by a new input.
	err = cr.checkDone(ctx)
	if err != nil {
		return err
	}

	prevInput := cr.currInput
	defer func() {
		// reset the currInput if we get an error below
		if err != nil {
			cr.currInput = prevInput
		}
	}()

	if len(unmerged) > 0 {
		rev := unmerged[len(unmerged)-1].Revision
		if rev < cr.currInput.unmerged {
			return fmt.Errorf("Unmerged revision %d is lower than the "+
				"expected unmerged revision %d", rev, cr.currInput.unmerged)
		}
		cr.currInput.unmerged = rev
	}
	if len(merged) > 0 {
		rev := merged[len(merged)-1].Revision
		if rev < cr.currInput.merged {
			return fmt.Errorf("Merged revision %d is lower than the "+
				"expected merged revision %d", rev, cr.currInput.merged)
		}
		cr.currInput.merged = rev
	}
	return nil
}

func (cr *ConflictResolver) makeChains(ctx context.Context,
	unmerged []*RootMetadata, merged []*RootMetadata) (
	unmergedChains *crChains, mergedChains *crChains, err error) {
	unmergedChains, err = newCRChains(unmerged)
	if err != nil {
		return nil, nil, err
	}

	mergedChains, err = newCRChains(merged)
	if err != nil {
		return nil, nil, err
	}

	cr.fbo.status.setCRChains(unmergedChains, mergedChains)
	return unmergedChains, mergedChains, nil
}

func (cr *ConflictResolver) doResolve(ctx context.Context, ci conflictInput) {
	cr.log.CDebugf(ctx, "Starting conflict resolution with input %v", ci)
	var err error
	defer cr.inputGroup.Done()
	defer func() {
		cr.log.CDebugf(ctx, "Finished conflict resolution: %v", err)
	}()

	// Canceled before we even got started?
	err = cr.checkDone(ctx)
	if err != nil {
		return
	}

	// Fetch the merged and unmerged MDs
	unmerged, merged, err := cr.getMDs(ctx)
	if err != nil {
		return
	}

	// Update the current input to reflect the MDs we'll actually be
	// working with.
	err = cr.updateCurrInput(ctx, unmerged, merged)
	if err != nil {
		return
	}

	// Canceled before we start the heavy lifting?
	err = cr.checkDone(ctx)
	if err != nil {
		return
	}

	// Make the chains
	_, _, err = cr.makeChains(ctx, unmerged, merged)
	if err != nil {
		return
	}
}
<commit_msg>cr: notes on the algorithm for how to resolve conflicting ops<commit_after>package libkbfs

import (
	"fmt"
	"sync"

	"github.com/keybase/client/go/logger"
	"golang.org/x/net/context"
)

// CtxCRTagKey is the type used for unique context tags related to
// conflict resolution
type CtxCRTagKey int

const (
	// CtxCRIDKey is the type of the tag for unique operation IDs
	// related to conflict resolution
	CtxCRIDKey CtxCRTagKey = iota
)

// CtxCROpID is the display name for the unique operation
// conflict resolution ID tag.
const CtxCROpID = "CRID"

type conflictInput struct {
	unmerged MetadataRevision
	merged   MetadataRevision
}

// ConflictResolver is responsible for resolving conflicts in the
// background.
type ConflictResolver struct {
	config     Config
	fbo        *FolderBranchOps
	inputChan  chan conflictInput
	inputGroup sync.WaitGroup
	log        logger.Logger

	shutdown     bool
	shutdownLock sync.RWMutex

	currInput conflictInput
	inputLock sync.Mutex
}

// NewConflictResolver constructs a new ConflictResolver (and launches
// any necessary background goroutines).
func NewConflictResolver(
	config Config, fbo *FolderBranchOps) *ConflictResolver {
	// make a logger with an appropriate module name
	branchSuffix := ""
	if fbo.branch() != MasterBranch {
		branchSuffix = " " + string(fbo.branch())
	}
	tlfStringFull := fbo.id().String()
	log := config.MakeLogger(fmt.Sprintf("CR %s%s", tlfStringFull[:8],
		branchSuffix))

	cr := &ConflictResolver{
		config:    config,
		fbo:       fbo,
		inputChan: make(chan conflictInput),
		log:       log,
		currInput: conflictInput{
			unmerged: MetadataRevisionUninitialized,
			merged:   MetadataRevisionUninitialized,
		},
	}

	go cr.processInput()
	return cr
}

func (cr *ConflictResolver) processInput() {
	logTags := make(logger.CtxLogTags)
	logTags[CtxCRIDKey] = CtxCROpID
	backgroundCtx := logger.NewContextWithLogTags(context.Background(), logTags)

	var cancel func()
	defer func() {
		if cancel != nil {
			cancel()
		}
	}()
	for ci := range cr.inputChan {
		ctx := backgroundCtx
		id, err := MakeRandomRequestID()
		if err != nil {
			cr.log.Warning("Couldn't generate a random request ID: %v", err)
		} else {
			ctx = context.WithValue(ctx, CtxCRIDKey, id)
		}

		valid := func() bool {
			cr.inputLock.Lock()
			defer cr.inputLock.Unlock()
			// The input is only interesting if one of the revisions
			// is greater than what we've looked at to date.
			if ci.unmerged <= cr.currInput.unmerged &&
				ci.merged <= cr.currInput.merged {
				return false
			}
			cr.log.CDebugf(ctx, "New conflict input %v following old "+
				"input %v", ci, cr.currInput)
			cr.currInput = ci
			// cancel the existing conflict resolution (if any)
			if cancel != nil {
				cancel()
			}
			return true
		}()
		if !valid {
			cr.log.CDebugf(ctx, "Ignoring uninteresting input: %v", ci)
			cr.inputGroup.Done()
			continue
		}

		ctx, cancel = context.WithCancel(ctx)
		go cr.doResolve(ctx, ci)
	}
}

// Resolve takes the latest known unmerged and merged revision
// numbers, and kicks off the resolution process.
func (cr *ConflictResolver) Resolve(unmerged MetadataRevision,
	merged MetadataRevision) {
	cr.shutdownLock.RLock()
	defer cr.shutdownLock.RUnlock()
	if cr.shutdown {
		return
	}

	cr.inputGroup.Add(1)
	cr.inputChan <- conflictInput{unmerged, merged}
}

// Wait blocks until the current set of submitted resolutions are
// complete (though not necessarily successful), or until the given
// context is canceled.
func (cr *ConflictResolver) Wait(ctx context.Context) error {
	c := make(chan struct{}, 1)
	go func() {
		cr.inputGroup.Wait()
		c <- struct{}{}
	}()

	select {
	case <-c:
		return nil
	case <-ctx.Done():
		return ctx.Err()
	}
}

// Shutdown cancels any ongoing resolutions and stops any background
// goroutines.
func (cr *ConflictResolver) Shutdown() {
	cr.shutdownLock.Lock()
	defer cr.shutdownLock.Unlock()
	cr.shutdown = true
	close(cr.inputChan)
}

func (cr *ConflictResolver) checkDone(ctx context.Context) error {
	select {
	case <-ctx.Done():
		return ctx.Err()
	default:
		return nil
	}
}

func (cr *ConflictResolver) getMDs(ctx context.Context) (
	unmerged []*RootMetadata, merged []*RootMetadata, err error) {
	// first get all outstanding unmerged MDs for this device
	branchPoint, unmerged, err := cr.fbo.getUnmergedMDUpdates(ctx)
	if err != nil {
		return nil, nil, err
	}

	// now get all the merged MDs, starting from after the branch point
	merged, err = getMergedMDUpdates(ctx, cr.fbo.config, cr.fbo.id(),
		branchPoint+1)
	if err != nil {
		return nil, nil, err
	}

	// re-embed all the block changes
	err = cr.fbo.reembedBlockChanges(ctx, unmerged)
	if err != nil {
		return nil, nil, err
	}
	err = cr.fbo.reembedBlockChanges(ctx, merged)
	if err != nil {
		return nil, nil, err
	}

	return unmerged, merged, nil
}

func (cr *ConflictResolver) updateCurrInput(ctx context.Context,
	unmerged []*RootMetadata, merged []*RootMetadata) (err error) {
	cr.inputLock.Lock()
	defer cr.inputLock.Unlock()
	// check done while holding the lock, so we know for sure if
	// we've already been canceled and replaced by a new input.
	err = cr.checkDone(ctx)
	if err != nil {
		return err
	}

	prevInput := cr.currInput
	defer func() {
		// reset the currInput if we get an error below
		if err != nil {
			cr.currInput = prevInput
		}
	}()

	if len(unmerged) > 0 {
		rev := unmerged[len(unmerged)-1].Revision
		if rev < cr.currInput.unmerged {
			return fmt.Errorf("Unmerged revision %d is lower than the "+
				"expected unmerged revision %d", rev, cr.currInput.unmerged)
		}
		cr.currInput.unmerged = rev
	}
	if len(merged) > 0 {
		rev := merged[len(merged)-1].Revision
		if rev < cr.currInput.merged {
			return fmt.Errorf("Merged revision %d is lower than the "+
				"expected merged revision %d", rev, cr.currInput.merged)
		}
		cr.currInput.merged = rev
	}
	return nil
}

func (cr *ConflictResolver) makeChains(ctx context.Context,
	unmerged []*RootMetadata, merged []*RootMetadata) (
	unmergedChains *crChains, mergedChains *crChains, err error) {
	unmergedChains, err = newCRChains(unmerged)
	if err != nil {
		return nil, nil, err
	}

	mergedChains, err = newCRChains(merged)
	if err != nil {
		return nil, nil, err
	}

	cr.fbo.status.setCRChains(unmergedChains, mergedChains)
	return unmergedChains, mergedChains, nil
}

func (cr *ConflictResolver) doResolve(ctx context.Context, ci conflictInput) {
	cr.log.CDebugf(ctx, "Starting conflict resolution with input %v", ci)
	var err error
	defer cr.inputGroup.Done()
	defer func() {
		cr.log.CDebugf(ctx, "Finished conflict resolution: %v", err)
	}()

	// Canceled before we even got started?
	err = cr.checkDone(ctx)
	if err != nil {
		return
	}

	// Fetch the merged and unmerged MDs
	unmerged, merged, err := cr.getMDs(ctx)
	if err != nil {
		return
	}

	// Update the current input to reflect the MDs we'll actually be
	// working with.
	err = cr.updateCurrInput(ctx, unmerged, merged)
	if err != nil {
		return
	}

	// Canceled before we start the heavy lifting?
	err = cr.checkDone(ctx)
	if err != nil {
		return
	}

	// Make the chains
	_, _, err = cr.makeChains(ctx, unmerged, merged)
	if err != nil {
		return
	}

	// * Get the full path for every most recent pointer with a chain of
	//   unmerged operations
	//   * If the path doesn't exist, that means the whole node was deleted,
	//     so skip.
	// * Order by descending path length.
	// * Find the corresponding path in the merged/resolved branch
	//   * Find the original pointer for the most recent pointer.
	//   * Find the merged most recent pointer of that original pointer.
	//   * Search for the full path to that most recent pointer in the final
	//     state of B.
	//   * If it exists, great!
	//   * If it doesn't exist, repeat on the next component of the path
	//     (closer to the root), until you find a path that does exist.
	//   * Now add createOps as necessary to resolvedChains in order to
	//     create the full path again, and add entries to the corresponding
	//     dirty blocks.  Use the most recent pointer unmerged block pointer
	//     for the directory as the starting pointer for the new directory.
	// * For syncOps, add a syncAttr setAttrOp to the parent unmerged dir chain
	//   * If there's a conflict, add a resolving createOp to the parent.
	// * Otherwise, for each operation in the unmerged chain, check for
	//   conflicts in the corresponding merged chain and resolve accordingly.
	//   * During this process, construct a separate set of notifyOps that
	//     will be played locally to the local caches into line with the
	//     new reality.
	// * In addition, if the op is an rmOp that's not part of a rename, check
	//   whether the original pointer of the actual BlockPointer in the
	//   directory entry being removed has a different most recent pointer
	//   in the merged branch.  If so, something has changed in it or in
	//   one of its children in the merged branch, so we can ignore the rmOp.
	// * Apply the operations by looking up the corresponding unmerged dir
	//   entry and copying it to a copy of the corresponding merged block.
	//   Keep these dirty block copies in a local dirty cache, keyed by
	//   corresponding merged most recent pointer.
	// * Once all the new blocks are ready, calculate the resolvedChain paths
	//   and arrange them into a tree.  Do a recursive descent and
	//   syncBlockLocked each branch (filling in the new BlockChanges in a
	//   new MD object)
	// * Finally attempt to put the final MD object.  If successful, send
	//   out all the notifyOps to observers.
	// Release all locks and reset the currInput, we're done!
}
<|endoftext|>


<commit_before>package listener

import (
	"bufio"
	"code.google.com/p/goprotobuf/proto"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/cep21/gohelpers/structdefaults"
	"github.com/cep21/gohelpers/workarounds"
	"github.com/golang/glog"
	"github.com/signalfuse/com_signalfuse_metrics_protobuf"
	"github.com/signalfuse/signalfxproxy/config"
	"github.com/signalfuse/signalfxproxy/core"
	"github.com/signalfuse/signalfxproxy/core/value"
	"github.com/signalfuse/signalfxproxy/protocoltypes"
	"io"
	"net"
	"net/http"
	"time"
	"sync"
)

type listenerServer struct {
	listener net.Listener
	metricCreationsMap map[string]com_signalfuse_metrics_protobuf.MetricType
	metricCreationsMapMutex *sync.Mutex

}

func (streamer *listenerServer) GetStats() []core.Datapoint {
	ret := []core.Datapoint{}
	return ret
}

func (streamer *listenerServer) Close() {
	streamer.listener.Close()
}

func jsonDecoderFunction(DatapointStreamingAPI core.DatapointStreamingAPI, metricCreationsMap map[string]com_signalfuse_metrics_protobuf.MetricType, metricCreationsMapMutex *sync.Mutex) func(*http.Request) error {
	return func(req *http.Request) error {
		dec := json.NewDecoder(req.Body)
		for {
			var d protocoltypes.SignalfxJsonDatapointV1
			if err := dec.Decode(&d); err == io.EOF {
				break
			} else if err != nil {
				return err
			} else {
				glog.V(3).Info("Got a new point: %s", d)
				if d.Metric == "" {
					glog.Warningf("Invalid datapoint %s", d)
					continue
				}
				mt := func() (com_signalfuse_metrics_protobuf.MetricType) {
					metricCreationsMapMutex.Lock()
					defer metricCreationsMapMutex.Unlock()
					mt, ok := metricCreationsMap[d.Metric]
					if !ok {
						return com_signalfuse_metrics_protobuf.MetricType_GAUGE
					} else {
						return mt
					}

				}()
				DatapointStreamingAPI.DatapointsChannel() <- core.NewRelativeTimeDatapoint(d.Metric, map[string]string{"sf_source": d.Source}, value.NewFloatWire(d.Value), mt, 0)
			}
		}
		return nil
	}
}

func fullyReadFromBuffer(buffer *bufio.Reader, numBytes uint64) ([]byte, error) {
	totalBytesRead := uint64(0)
	buf := make([]byte, numBytes)
	for numBytes > totalBytesRead {
		n, err := buffer.Read(buf[totalBytesRead:numBytes])
		if err != nil {
			return buf, err
		}
		if n == 0 {
			return buf, nil
		}
		totalBytesRead += uint64(n)
	}
	return buf, nil
}

func protobufDecoderFunction(DatapointStreamingAPI core.DatapointStreamingAPI, metricCreationsMap map[string]com_signalfuse_metrics_protobuf.MetricType, metricCreationsMapMutex *sync.Mutex) func(*http.Request) error {
	return func(req *http.Request) error {
		bufferedBody := bufio.NewReaderSize(req.Body, 32768)
		for {
			buf, err := bufferedBody.Peek(16) // should be big enough for any varint
			if err == io.EOF {
				return nil
			}

			if err != nil {
				glog.Infof("Got %s", err)
				return err
			}
			num, bytesRead := proto.DecodeVarint(buf)
			if bytesRead == 0 {
				// Invalid varint?
				return errors.New("invalid varint decode from protobuf stream")
			}
			if num > 32768 {
				// Sanity check
				return fmt.Errorf("invalid varint decode from protobuf stream.  Value too large %d", num)
			}
			// Get the varint out
			fullyReadFromBuffer(bufferedBody, uint64(bytesRead))
			//buf = make([]byte, num)
			buf, err = fullyReadFromBuffer(bufferedBody, num)
			if int(num) != len(buf) {
				return fmt.Errorf("unable to fully read protobuf message: %s", err)
			}
			var msg com_signalfuse_metrics_protobuf.DataPoint
			err = proto.Unmarshal(buf, &msg)
			if err != nil {
				return err
			}
			dp := func() (core.Datapoint) {
				metricCreationsMapMutex.Lock()
				defer metricCreationsMapMutex.Unlock()
				mt, ok := metricCreationsMap[msg.GetMetric()]
				if !ok {
					return protocoltypes.NewProtobufDataPoint(msg)
				} else {
					return protocoltypes.NewProtobufDataPointWithType(msg, mt)
				}

			}()
			DatapointStreamingAPI.DatapointsChannel() <- dp
		}
		return nil
	}
}

var defaultConfig = &config.ListenFrom{
	ListenAddr:      workarounds.GolangDoesnotAllowPointerToStringLiteral("0.0.0.0:12345"),
	TimeoutDuration: workarounds.GolangDoesnotAllowPointerToTimeLiteral(time.Second * 30),
}

// SignalFxListenerLoader loads a listener for signalfx protocol from config
func SignalFxListenerLoader(DatapointStreamingAPI core.DatapointStreamingAPI, listenFrom *config.ListenFrom) (DatapointListener, error) {
	structdefaults.FillDefaultFrom(listenFrom, defaultConfig)
	glog.Infof("Creating signalfx listener using final config %s", listenFrom)
	return StartServingHTTPOnPort(*listenFrom.ListenAddr, DatapointStreamingAPI, *listenFrom.TimeoutDuration)
}

// StartServingHTTPOnPort servers http requests for Signalfx datapoints
func StartServingHTTPOnPort(listenAddr string, DatapointStreamingAPI core.DatapointStreamingAPI, clientTimeout time.Duration) (DatapointListener, error) {
	mux := http.NewServeMux()
	metricCreationsMap := make(map[string]com_signalfuse_metrics_protobuf.MetricType)
	metricCreationsMapMutex := &sync.Mutex{}
	datapointHandler := func(writter http.ResponseWriter, req *http.Request) {
		contentType := req.Header.Get("Content-type")
		var decoderFunc func(*http.Request) error
		if contentType == "" || contentType == "application/json" {
			decoderFunc = jsonDecoderFunction(DatapointStreamingAPI, metricCreationsMap, metricCreationsMapMutex)
		} else if contentType == "" || contentType == "application/x-protobuf" {
			decoderFunc = protobufDecoderFunction(DatapointStreamingAPI, metricCreationsMap, metricCreationsMapMutex)
		} else {
			writter.WriteHeader(http.StatusBadRequest)
			writter.Write([]byte(`{msg:"Unknown content type"}`))
			return
		}
		err := decoderFunc(req)
		if err != nil {
			writter.WriteHeader(http.StatusBadRequest)
			writter.Write([]byte(err.Error()))
		} else {
			writter.WriteHeader(http.StatusOK)
			writter.Write([]byte(`"OK"`))
		}
	}

	metricHandler := func(writter http.ResponseWriter, req *http.Request) {
		dec := json.NewDecoder(req.Body)
		var d []protocoltypes.SignalfxMetricCreationStruct
		if err := dec.Decode(&d); err == io.EOF {
			return
		} else if err != nil {
			glog.Infof("Invalid metric creation request: %s", err)
			writter.WriteHeader(http.StatusBadRequest)
			writter.Write([]byte(`{msg:"Invalid creation request"}`))
			return
		} else {
			glog.V(3).Info("Got a metric types: %s", d)
			metricCreationsMapMutex.Lock()
			defer metricCreationsMapMutex.Unlock()
			ret := []protocoltypes.SignalfxMetricCreationResponse{}
			for _, m := range d {
				metricType, ok := com_signalfuse_metrics_protobuf.MetricType_value[m.MetricType]
				if !ok {
					writter.WriteHeader(http.StatusBadRequest)
					writter.Write([]byte(`{msg:"Invalid metric type"}`))
					return
				}
				metricCreationsMap[m.MetricName] = com_signalfuse_metrics_protobuf.MetricType(metricType)
				ret = append(ret, protocoltypes.SignalfxMetricCreationResponse{Code: 200})
			}
			writter.WriteHeader(http.StatusOK)
			writter.Write([]byte(`"OK"`))
		}
	}
	mux.HandleFunc(
		"/datapoint",
		datapointHandler)
	mux.HandleFunc(
		"/v1/datapoint",
		datapointHandler)
	mux.HandleFunc(
		"/v1/metric",
		metricHandler)
	mux.HandleFunc(
		"/metric",
		metricHandler)
	server := http.Server{
		Handler:      mux,
		Addr:         listenAddr,
		ReadTimeout:  clientTimeout,
		WriteTimeout: clientTimeout,
	}
	listener, err := net.Listen("tcp", listenAddr)
	if err != nil {
		return nil, err
	}
	listenServer := listenerServer{
		listener: listener,
		metricCreationsMap: metricCreationsMap,
		metricCreationsMapMutex: metricCreationsMapMutex,
	}
	go server.Serve(listener)
	return &listenServer, err
}
<commit_msg>Write out the metric creation json response<commit_after>package listener

import (
	"bufio"
	"code.google.com/p/goprotobuf/proto"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/cep21/gohelpers/structdefaults"
	"github.com/cep21/gohelpers/workarounds"
	"github.com/golang/glog"
	"github.com/signalfuse/com_signalfuse_metrics_protobuf"
	"github.com/signalfuse/signalfxproxy/config"
	"github.com/signalfuse/signalfxproxy/core"
	"github.com/signalfuse/signalfxproxy/core/value"
	"github.com/signalfuse/signalfxproxy/protocoltypes"
	"io"
	"net"
	"net/http"
	"time"
	"sync"
)

type listenerServer struct {
	listener net.Listener
	metricCreationsMap map[string]com_signalfuse_metrics_protobuf.MetricType
	metricCreationsMapMutex *sync.Mutex

}

func (streamer *listenerServer) GetStats() []core.Datapoint {
	ret := []core.Datapoint{}
	return ret
}

func (streamer *listenerServer) Close() {
	streamer.listener.Close()
}

func jsonDecoderFunction(DatapointStreamingAPI core.DatapointStreamingAPI, metricCreationsMap map[string]com_signalfuse_metrics_protobuf.MetricType, metricCreationsMapMutex *sync.Mutex) func(*http.Request) error {
	return func(req *http.Request) error {
		dec := json.NewDecoder(req.Body)
		for {
			var d protocoltypes.SignalfxJsonDatapointV1
			if err := dec.Decode(&d); err == io.EOF {
				break
			} else if err != nil {
				return err
			} else {
				glog.V(3).Info("Got a new point: %s", d)
				if d.Metric == "" {
					glog.Warningf("Invalid datapoint %s", d)
					continue
				}
				mt := func() (com_signalfuse_metrics_protobuf.MetricType) {
					metricCreationsMapMutex.Lock()
					defer metricCreationsMapMutex.Unlock()
					mt, ok := metricCreationsMap[d.Metric]
					if !ok {
						return com_signalfuse_metrics_protobuf.MetricType_GAUGE
					} else {
						return mt
					}

				}()
				DatapointStreamingAPI.DatapointsChannel() <- core.NewRelativeTimeDatapoint(d.Metric, map[string]string{"sf_source": d.Source}, value.NewFloatWire(d.Value), mt, 0)
			}
		}
		return nil
	}
}

func fullyReadFromBuffer(buffer *bufio.Reader, numBytes uint64) ([]byte, error) {
	totalBytesRead := uint64(0)
	buf := make([]byte, numBytes)
	for numBytes > totalBytesRead {
		n, err := buffer.Read(buf[totalBytesRead:numBytes])
		if err != nil {
			return buf, err
		}
		if n == 0 {
			return buf, nil
		}
		totalBytesRead += uint64(n)
	}
	return buf, nil
}

func protobufDecoderFunction(DatapointStreamingAPI core.DatapointStreamingAPI, metricCreationsMap map[string]com_signalfuse_metrics_protobuf.MetricType, metricCreationsMapMutex *sync.Mutex) func(*http.Request) error {
	return func(req *http.Request) error {
		bufferedBody := bufio.NewReaderSize(req.Body, 32768)
		for {
			buf, err := bufferedBody.Peek(16) // should be big enough for any varint
			if err == io.EOF {
				return nil
			}

			if err != nil {
				glog.Infof("Got %s", err)
				return err
			}
			num, bytesRead := proto.DecodeVarint(buf)
			if bytesRead == 0 {
				// Invalid varint?
				return errors.New("invalid varint decode from protobuf stream")
			}
			if num > 32768 {
				// Sanity check
				return fmt.Errorf("invalid varint decode from protobuf stream.  Value too large %d", num)
			}
			// Get the varint out
			fullyReadFromBuffer(bufferedBody, uint64(bytesRead))
			//buf = make([]byte, num)
			buf, err = fullyReadFromBuffer(bufferedBody, num)
			if int(num) != len(buf) {
				return fmt.Errorf("unable to fully read protobuf message: %s", err)
			}
			var msg com_signalfuse_metrics_protobuf.DataPoint
			err = proto.Unmarshal(buf, &msg)
			if err != nil {
				return err
			}
			dp := func() (core.Datapoint) {
				metricCreationsMapMutex.Lock()
				defer metricCreationsMapMutex.Unlock()
				mt, ok := metricCreationsMap[msg.GetMetric()]
				if !ok {
					return protocoltypes.NewProtobufDataPoint(msg)
				} else {
					return protocoltypes.NewProtobufDataPointWithType(msg, mt)
				}

			}()
			DatapointStreamingAPI.DatapointsChannel() <- dp
		}
		return nil
	}
}

var defaultConfig = &config.ListenFrom{
	ListenAddr:      workarounds.GolangDoesnotAllowPointerToStringLiteral("0.0.0.0:12345"),
	TimeoutDuration: workarounds.GolangDoesnotAllowPointerToTimeLiteral(time.Second * 30),
}

// SignalFxListenerLoader loads a listener for signalfx protocol from config
func SignalFxListenerLoader(DatapointStreamingAPI core.DatapointStreamingAPI, listenFrom *config.ListenFrom) (DatapointListener, error) {
	structdefaults.FillDefaultFrom(listenFrom, defaultConfig)
	glog.Infof("Creating signalfx listener using final config %s", listenFrom)
	return StartServingHTTPOnPort(*listenFrom.ListenAddr, DatapointStreamingAPI, *listenFrom.TimeoutDuration)
}

// StartServingHTTPOnPort servers http requests for Signalfx datapoints
func StartServingHTTPOnPort(listenAddr string, DatapointStreamingAPI core.DatapointStreamingAPI, clientTimeout time.Duration) (DatapointListener, error) {
	mux := http.NewServeMux()
	metricCreationsMap := make(map[string]com_signalfuse_metrics_protobuf.MetricType)
	metricCreationsMapMutex := &sync.Mutex{}
	datapointHandler := func(writter http.ResponseWriter, req *http.Request) {
		contentType := req.Header.Get("Content-type")
		var decoderFunc func(*http.Request) error
		if contentType == "" || contentType == "application/json" {
			decoderFunc = jsonDecoderFunction(DatapointStreamingAPI, metricCreationsMap, metricCreationsMapMutex)
		} else if contentType == "" || contentType == "application/x-protobuf" {
			decoderFunc = protobufDecoderFunction(DatapointStreamingAPI, metricCreationsMap, metricCreationsMapMutex)
		} else {
			writter.WriteHeader(http.StatusBadRequest)
			writter.Write([]byte(`{msg:"Unknown content type"}`))
			return
		}
		err := decoderFunc(req)
		if err != nil {
			writter.WriteHeader(http.StatusBadRequest)
			writter.Write([]byte(err.Error()))
		} else {
			writter.WriteHeader(http.StatusOK)
			writter.Write([]byte(`"OK"`))
		}
	}

	metricHandler := func(writter http.ResponseWriter, req *http.Request) {
		dec := json.NewDecoder(req.Body)
		var d []protocoltypes.SignalfxMetricCreationStruct
		if err := dec.Decode(&d); err == io.EOF {
			return
		} else if err != nil {
			glog.Infof("Invalid metric creation request: %s", err)
			writter.WriteHeader(http.StatusBadRequest)
			writter.Write([]byte(`{msg:"Invalid creation request"}`))
			return
		} else {
			glog.V(3).Info("Got a metric types: %s", d)
			metricCreationsMapMutex.Lock()
			defer metricCreationsMapMutex.Unlock()
			ret := []protocoltypes.SignalfxMetricCreationResponse{}
			for _, m := range d {
				metricType, ok := com_signalfuse_metrics_protobuf.MetricType_value[m.MetricType]
				if !ok {
					writter.WriteHeader(http.StatusBadRequest)
					writter.Write([]byte(`{msg:"Invalid metric type"}`))
					return
				}
				metricCreationsMap[m.MetricName] = com_signalfuse_metrics_protobuf.MetricType(metricType)
				ret = append(ret, protocoltypes.SignalfxMetricCreationResponse{Code: 200})
			}
			toWrite, err := json.Marshal(ret)
			if err != nil {
				glog.Warningf("Unable to marshal json: %s", err)
				writter.WriteHeader(http.StatusBadRequest)
				writter.Write([]byte(`{msg:"Unable to marshal json!"}`))
				return
			}
			writter.WriteHeader(http.StatusOK)
			writter.Write([]byte(toWrite))
		}
	}
	mux.HandleFunc(
		"/datapoint",
		datapointHandler)
	mux.HandleFunc(
		"/v1/datapoint",
		datapointHandler)
	mux.HandleFunc(
		"/v1/metric",
		metricHandler)
	mux.HandleFunc(
		"/metric",
		metricHandler)
	server := http.Server{
		Handler:      mux,
		Addr:         listenAddr,
		ReadTimeout:  clientTimeout,
		WriteTimeout: clientTimeout,
	}
	listener, err := net.Listen("tcp", listenAddr)
	if err != nil {
		return nil, err
	}
	listenServer := listenerServer{
		listener: listener,
		metricCreationsMap: metricCreationsMap,
		metricCreationsMapMutex: metricCreationsMapMutex,
	}
	go server.Serve(listener)
	return &listenServer, err
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcsproxy

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"

	"github.com/jacobsa/gcloud/gcs"
	"golang.org/x/net/context"
)

// A sentinel error returned by ObjectProxy.Sync.
var ErrNotCurrent error = errors.New("Source generation not current.")

// A view on a particular generation of an object in GCS that allows random
// access reads and writes.
//
// Reads may involve reading from a local cache. Writes are buffered locally
// until the Sync method is called, at which time a new generation of the
// object is created.
//
// This type is not safe for concurrent access. The user must provide external
// synchronization.
type ObjectProxy struct {
	/////////////////////////
	// Dependencies
	/////////////////////////

	bucket gcs.Bucket

	/////////////////////////
	// Constant data
	/////////////////////////

	// The name of the GCS object for which we are a proxy. Might not currently
	// exist in the bucket.
	name string

	/////////////////////////
	// Mutable state
	/////////////////////////

	// The specific generation of the object from which our local state is
	// branched. If we have no local state, the contents of this object are
	// exactly our contents. May be zero if our source is a "doesn't exist"
	// generation.
	srcGeneration uint64

	// A local temporary file containing our current contents. When non-nil, this
	// is the authority on our contents. When nil, our contents are defined by
	// the generation identified by srcGeneration.
	localFile *os.File

	// false if localFile is present but its contents may be different from the
	// contents of our source generation. Sync needs to do work iff this is true.
	//
	// INVARIANT: If srcGeneration == 0, then dirty
	// INVARIANT: If dirty, then localFile != nil
	dirty bool
}

////////////////////////////////////////////////////////////////////////
// Public interface
////////////////////////////////////////////////////////////////////////

// Create a view on the given GCS object generation, or zero if branching from
// a non-existent object (in which case the initial contents are empty).
func NewObjectProxy(
	ctx context.Context,
	bucket gcs.Bucket,
	name string,
	srcGeneration uint64) (op *ObjectProxy, err error) {
	// Set up the basic struct.
	op = &ObjectProxy{
		bucket:        bucket,
		name:          name,
		srcGeneration: srcGeneration,
	}

	// For "doesn't exist" source generations, we must establish an empty local
	// file and mark the proxy dirty.
	if srcGeneration == 0 {
		if err = op.ensureLocalFile(ctx); err != nil {
			return
		}

		op.dirty = true
	}

	return
}

// Return the name of the proxied object. This may or may not be an object that
// currently exists in the bucket.
func (op *ObjectProxy) Name() string {
	return op.name
}

// Panic if any internal invariants are violated. Careful users can call this
// at appropriate times to help debug weirdness. Consider using
// syncutil.InvariantMutex to automate the process.
func (op *ObjectProxy) CheckInvariants() {
	// INVARIANT: If srcGeneration == 0, then dirty
	if op.srcGeneration == 0 && !op.dirty {
		panic("Expected dirty.")
	}

	// INVARIANT: If dirty, then localFile != nil
	if op.dirty && op.localFile == nil {
		panic("Expected non-nil localFile.")
	}
}

// Destroy any local file caches, putting the proxy into an indeterminate
// state. Should be used before dropping the final reference to the proxy.
func (op *ObjectProxy) Destroy() (err error) {
	// Make sure that when we exit no invariants are violated.
	defer func() {
		op.srcGeneration = 1
		op.localFile = nil
		op.dirty = false
	}()

	// If we have no local file, there's nothing to do.
	if op.localFile == nil {
		return
	}

	// Close the local file.
	if err = op.localFile.Close(); err != nil {
		err = fmt.Errorf("Close: %v", err)
		return
	}

	return
}

// Return the current size in bytes of the content and an indication of whether
// the proxied object has changed out from under us (in which case Sync will
// fail).
func (op *ObjectProxy) Stat(
	ctx context.Context) (size uint64, clobbered bool, err error) {
	panic("TODO")
}

// Make a random access read into our view of the content. May block for
// network access.
//
// Guarantees that err != nil if n < len(buf)
func (op *ObjectProxy) ReadAt(
	ctx context.Context,
	buf []byte,
	offset int64) (n int, err error) {
	// Make sure we have a local file.
	if err = op.ensureLocalFile(ctx); err != nil {
		err = fmt.Errorf("ensureLocalFile: %v", err)
		return
	}

	// Serve the read from the file.
	n, err = op.localFile.ReadAt(buf, offset)

	return
}

// Make a random access write into our view of the content. May block for
// network access. Not guaranteed to be reflected remotely until after Sync is
// called successfully.
//
// Guarantees that err != nil if n < len(buf)
func (op *ObjectProxy) WriteAt(
	ctx context.Context,
	buf []byte,
	offset int64) (n int, err error) {
	panic("TODO")
}

// Truncate our view of the content to the given number of bytes, extending if
// n is greater than the current size. May block for network access. Not
// guaranteed to be reflected remotely until after Sync is called successfully.
func (op *ObjectProxy) Truncate(ctx context.Context, n uint64) (err error) {
	panic("TODO")
}

// If the proxy is dirty due to having been written to or due to having a nil
// source, save its current contents to GCS and return a generation number for
// a generation with exactly those contents. Do so with a precondition such
// that the creation will fail if the source generation is not current. In that
// case, return ErrNotCurrent.
func (op *ObjectProxy) Sync(ctx context.Context) (gen uint64, err error) {
	panic("TODO")
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

// Set up an unlinked local temporary file for the given generation of the
// given object. Special case: generation == 0 means an empty file.
func makeLocalFile(
	ctx context.Context,
	bucket gcs.Bucket,
	name string,
	generation uint64) (f *os.File, err error) {
	// Create the file.
	f, err = ioutil.TempFile("", "object_proxy")
	if err != nil {
		err = fmt.Errorf("TempFile: %v", err)
		return
	}

	// Unlink the file so that its inode will be garbage collected when the file
	// is closed.
	if err = os.Remove(f.Name()); err != nil {
		f.Close()
		err = fmt.Errorf("Remove: %v", err)
		return
	}

	// Fetch the object's contents if necessary.
	if generation != 0 {
		panic("TODO")
	}

	return
}

// Ensure that op.localFile is non-nil with an authoritative view of op's
// contents.
func (op *ObjectProxy) ensureLocalFile(ctx context.Context) (err error) {
	// Is there anything to do?
	if op.localFile != nil {
		return
	}

	// Set up the file.
	f, err := makeLocalFile(ctx, op.bucket, op.name, op.srcGeneration)
	if err != nil {
		err = fmt.Errorf("makeLocalFile: %v", err)
		return
	}

	op.localFile = f
	return
}
<commit_msg>Implemented ObjectProxy.WriteAt.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcsproxy

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"

	"github.com/jacobsa/gcloud/gcs"
	"golang.org/x/net/context"
)

// A sentinel error returned by ObjectProxy.Sync.
var ErrNotCurrent error = errors.New("Source generation not current.")

// A view on a particular generation of an object in GCS that allows random
// access reads and writes.
//
// Reads may involve reading from a local cache. Writes are buffered locally
// until the Sync method is called, at which time a new generation of the
// object is created.
//
// This type is not safe for concurrent access. The user must provide external
// synchronization.
type ObjectProxy struct {
	/////////////////////////
	// Dependencies
	/////////////////////////

	bucket gcs.Bucket

	/////////////////////////
	// Constant data
	/////////////////////////

	// The name of the GCS object for which we are a proxy. Might not currently
	// exist in the bucket.
	name string

	/////////////////////////
	// Mutable state
	/////////////////////////

	// The specific generation of the object from which our local state is
	// branched. If we have no local state, the contents of this object are
	// exactly our contents. May be zero if our source is a "doesn't exist"
	// generation.
	srcGeneration uint64

	// A local temporary file containing our current contents. When non-nil, this
	// is the authority on our contents. When nil, our contents are defined by
	// the generation identified by srcGeneration.
	localFile *os.File

	// false if localFile is present but its contents may be different from the
	// contents of our source generation. Sync needs to do work iff this is true.
	//
	// INVARIANT: If srcGeneration == 0, then dirty
	// INVARIANT: If dirty, then localFile != nil
	dirty bool
}

////////////////////////////////////////////////////////////////////////
// Public interface
////////////////////////////////////////////////////////////////////////

// Create a view on the given GCS object generation, or zero if branching from
// a non-existent object (in which case the initial contents are empty).
func NewObjectProxy(
	ctx context.Context,
	bucket gcs.Bucket,
	name string,
	srcGeneration uint64) (op *ObjectProxy, err error) {
	// Set up the basic struct.
	op = &ObjectProxy{
		bucket:        bucket,
		name:          name,
		srcGeneration: srcGeneration,
	}

	// For "doesn't exist" source generations, we must establish an empty local
	// file and mark the proxy dirty.
	if srcGeneration == 0 {
		if err = op.ensureLocalFile(ctx); err != nil {
			return
		}

		op.dirty = true
	}

	return
}

// Return the name of the proxied object. This may or may not be an object that
// currently exists in the bucket.
func (op *ObjectProxy) Name() string {
	return op.name
}

// Panic if any internal invariants are violated. Careful users can call this
// at appropriate times to help debug weirdness. Consider using
// syncutil.InvariantMutex to automate the process.
func (op *ObjectProxy) CheckInvariants() {
	// INVARIANT: If srcGeneration == 0, then dirty
	if op.srcGeneration == 0 && !op.dirty {
		panic("Expected dirty.")
	}

	// INVARIANT: If dirty, then localFile != nil
	if op.dirty && op.localFile == nil {
		panic("Expected non-nil localFile.")
	}
}

// Destroy any local file caches, putting the proxy into an indeterminate
// state. Should be used before dropping the final reference to the proxy.
func (op *ObjectProxy) Destroy() (err error) {
	// Make sure that when we exit no invariants are violated.
	defer func() {
		op.srcGeneration = 1
		op.localFile = nil
		op.dirty = false
	}()

	// If we have no local file, there's nothing to do.
	if op.localFile == nil {
		return
	}

	// Close the local file.
	if err = op.localFile.Close(); err != nil {
		err = fmt.Errorf("Close: %v", err)
		return
	}

	return
}

// Return the current size in bytes of the content and an indication of whether
// the proxied object has changed out from under us (in which case Sync will
// fail).
func (op *ObjectProxy) Stat(
	ctx context.Context) (size uint64, clobbered bool, err error) {
	panic("TODO")
}

// Make a random access read into our view of the content. May block for
// network access.
//
// Guarantees that err != nil if n < len(buf)
func (op *ObjectProxy) ReadAt(
	ctx context.Context,
	buf []byte,
	offset int64) (n int, err error) {
	// Make sure we have a local file.
	if err = op.ensureLocalFile(ctx); err != nil {
		err = fmt.Errorf("ensureLocalFile: %v", err)
		return
	}

	// Serve the read from the file.
	n, err = op.localFile.ReadAt(buf, offset)

	return
}

// Make a random access write into our view of the content. May block for
// network access. Not guaranteed to be reflected remotely until after Sync is
// called successfully.
//
// Guarantees that err != nil if n < len(buf)
func (op *ObjectProxy) WriteAt(
	ctx context.Context,
	buf []byte,
	offset int64) (n int, err error) {
	// Make sure we have a local file.
	if err = op.ensureLocalFile(ctx); err != nil {
		err = fmt.Errorf("ensureLocalFile: %v", err)
		return
	}

	// Serve the write using the file.
	n, err = op.localFile.WriteAt(buf, offset)

	return
}

// Truncate our view of the content to the given number of bytes, extending if
// n is greater than the current size. May block for network access. Not
// guaranteed to be reflected remotely until after Sync is called successfully.
func (op *ObjectProxy) Truncate(ctx context.Context, n uint64) (err error) {
	panic("TODO")
}

// If the proxy is dirty due to having been written to or due to having a nil
// source, save its current contents to GCS and return a generation number for
// a generation with exactly those contents. Do so with a precondition such
// that the creation will fail if the source generation is not current. In that
// case, return ErrNotCurrent.
func (op *ObjectProxy) Sync(ctx context.Context) (gen uint64, err error) {
	panic("TODO")
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

// Set up an unlinked local temporary file for the given generation of the
// given object. Special case: generation == 0 means an empty file.
func makeLocalFile(
	ctx context.Context,
	bucket gcs.Bucket,
	name string,
	generation uint64) (f *os.File, err error) {
	// Create the file.
	f, err = ioutil.TempFile("", "object_proxy")
	if err != nil {
		err = fmt.Errorf("TempFile: %v", err)
		return
	}

	// Unlink the file so that its inode will be garbage collected when the file
	// is closed.
	if err = os.Remove(f.Name()); err != nil {
		f.Close()
		err = fmt.Errorf("Remove: %v", err)
		return
	}

	// Fetch the object's contents if necessary.
	if generation != 0 {
		panic("TODO")
	}

	return
}

// Ensure that op.localFile is non-nil with an authoritative view of op's
// contents.
func (op *ObjectProxy) ensureLocalFile(ctx context.Context) (err error) {
	// Is there anything to do?
	if op.localFile != nil {
		return
	}

	// Set up the file.
	f, err := makeLocalFile(ctx, op.bucket, op.name, op.srcGeneration)
	if err != nil {
		err = fmt.Errorf("makeLocalFile: %v", err)
		return
	}

	op.localFile = f
	return
}
<|endoftext|>


<commit_before>package main

import (
	"bosun.org/graphite"
	"encoding/json"
	"fmt"
	"github.com/Dieterbe/go-metrics"
	"github.com/raintank/raintank-metric/schema"
	"io/ioutil"
	"log"
	"math"
	"math/rand"
	"net"
	"net/http"
	"os"
	"strconv"
	"sync"
	"time"
	//"strings"
)

type Hit struct {
	Index  string                  `json:"_index"`
	Type   string                  `json:"_type"`
	Id     string                  `json:"_id"`
	Score  float64                 `json:"_score"`
	Source schema.MetricDefinition `json:"_source"`
}

type EsResult struct {
	Took     int
	TimedOut bool
	_shards  struct {
		total      int
		successful int
		failed     int
	}
	Hits struct {
		Total    int
		MaxScore int
		Hits     []Hit
	}
}

func perror(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

type stat struct {
	def       schema.MetricDefinition
	firstSeen int64
}

var targets map[string]stat
var targetKeys []string
var targetsLock sync.Mutex

func init() {
	targets = make(map[string]stat)
	targetKeys = make([]string, 0)
}

func main() {
	if len(os.Args) != 4 {
		log.Fatal("usage: graphite-watcher <elasticsearch-addr> <metrics-addr> <graphite-addr>")
	}
	addr, _ := net.ResolveTCPAddr("tcp", os.Args[2])
	go metrics.Graphite(metrics.DefaultRegistry, 10e9, "graphite-watcher.", addr)
	lag := metrics.NewHistogram(metrics.NewExpDecaySample(1028, 0.015))
	metrics.Register("lag", lag)
	numMetrics := metrics.NewGauge()
	metrics.Register("num_metrics", numMetrics)

	// for a metric to exist in ES at t=Y, there must at least have been 1 point for that metric
	// at a time X where X < Y.  Hence, we confidently say that if we see a metric at Y, we can
	// demand data to show up for that metric at >=Y
	// for our data check to be useful we need metrics to show up in ES soon after being in the pipeline,
	// which seems to be true (see nsqadmin)
	updateTargets := func() {
		getEsTick := time.NewTicker(time.Second * time.Duration(10))
		for ts := range getEsTick.C {
			tsUnix := ts.Unix()
			res, err := http.Get("http://" + os.Args[1] + "/metric/_search?q=*:*&size=10000000")
			perror(err)
			defer res.Body.Close()
			body, err := ioutil.ReadAll(res.Body)
			perror(err)
			var data EsResult
			err = json.Unmarshal(body, &data)
			perror(err)
			amount := len(data.Hits.Hits)
			numMetrics.Update(int64(amount))
			for _, h := range data.Hits.Hits {
				//fmt.Println(h.Source.Name)
				targetsLock.Lock()
				if _, ok := targets[h.Source.Name]; !ok {
					targets[h.Source.Name] = stat{h.Source, tsUnix}
					targetKeys = append(targetKeys, h.Source.Name)
				}
				targetsLock.Unlock()
			}
		}
	}

	test := func(wg *sync.WaitGroup, curTs int64, met stat) {
		g := graphite.HostHeader{Host: "http://" + os.Args[3] + "/render", Header: http.Header{}}
		g.Header.Add("X-Org-Id", strconv.FormatInt(int64(met.def.OrgId), 10))
		q := graphite.Request{Targets: []string{met.def.Name}}
		series, err := g.Query(&q)
		perror(err)
		for _, serie := range series {
			if met.def.Name != serie.Target {
				fmt.Println("ERROR: name != target name:", met.def.Name, serie.Target)
			}

			lastTs := int64(0)
			oldestNull := int64(math.MaxInt64)
			if len(serie.Datapoints) == 0 {
				fmt.Println("ERROR: series for", met.def.Name, "contains no points!")
			}
			for _, p := range serie.Datapoints {
				ts, err := p[1].Int64()
				if err != nil {
					fmt.Println("ERROR: could not parse timestamp", p)
				}
				if ts <= lastTs {
					fmt.Println("ERROR: timestamp must be bigger than last", lastTs, ts)
				}
				if lastTs == 0 && (ts < curTs-24*3600-60 || ts > curTs-24*3600+60) {
					fmt.Println("ERROR: first point", p, "should have been about 24h ago")
				}
				if lastTs != 0 && ts != lastTs+int64(met.def.Interval) {
					fmt.Println("ERROR: point", p, " is not interval ", met.def.Interval, "apart from previous point")
				}
				_, err = p[0].Float64()
				if err != nil && ts > met.firstSeen {
					if ts < oldestNull {
						oldestNull = ts
					}
					if ts < curTs-30 {
						// will show up in lag metric too
						fmt.Println("ERROR: point has a recent null value", p)
					}
				}
				lastTs = ts
			}
			if lastTs < curTs-int64(met.def.Interval) || lastTs > curTs+int64(met.def.Interval) {
				fmt.Println("ERROR: last point at ", lastTs, "is out of range")
			}
			// if there was no null, we treat the point after the last one we had as null
			if oldestNull == math.MaxInt64 {
				oldestNull = lastTs + int64(met.def.Interval)
			}
			// lag cannot be < 0
			if oldestNull > curTs {
				oldestNull = curTs
			}
			// lag is from first null, even if there were non-nulls after it
			//fmt.Printf("%60s - lag %d\n", name, curTs-oldestNull)
			lag.Update(curTs - oldestNull)
		}
		wg.Done()
	}

	go updateTargets()

	tick := time.NewTicker(time.Millisecond * time.Duration(100))
	wg := &sync.WaitGroup{}
	for ts := range tick.C {
		var t stat
		targetsLock.Lock()
		if len(targetKeys) > 0 {
			key := targetKeys[rand.Intn(len(targets))]
			t = targets[key]
		}
		targetsLock.Unlock()
		if len(targetKeys) > 0 {
			wg.Add(1)
			go test(wg, ts.Unix(), t)
		}
	}
	wg.Wait()
}
<commit_msg>user agent<commit_after>package main

import (
	"bosun.org/graphite"
	"encoding/json"
	"fmt"
	"github.com/Dieterbe/go-metrics"
	"github.com/raintank/raintank-metric/schema"
	"io/ioutil"
	"log"
	"math"
	"math/rand"
	"net"
	"net/http"
	"os"
	"strconv"
	"sync"
	"time"
	//"strings"
)

type Hit struct {
	Index  string                  `json:"_index"`
	Type   string                  `json:"_type"`
	Id     string                  `json:"_id"`
	Score  float64                 `json:"_score"`
	Source schema.MetricDefinition `json:"_source"`
}

type EsResult struct {
	Took     int
	TimedOut bool
	_shards  struct {
		total      int
		successful int
		failed     int
	}
	Hits struct {
		Total    int
		MaxScore int
		Hits     []Hit
	}
}

func perror(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

type stat struct {
	def       schema.MetricDefinition
	firstSeen int64
}

var targets map[string]stat
var targetKeys []string
var targetsLock sync.Mutex

func init() {
	targets = make(map[string]stat)
	targetKeys = make([]string, 0)
}

func main() {
	if len(os.Args) != 4 {
		log.Fatal("usage: graphite-watcher <elasticsearch-addr> <metrics-addr> <graphite-addr>")
	}
	addr, _ := net.ResolveTCPAddr("tcp", os.Args[2])
	go metrics.Graphite(metrics.DefaultRegistry, 10e9, "graphite-watcher.", addr)
	lag := metrics.NewHistogram(metrics.NewExpDecaySample(1028, 0.015))
	metrics.Register("lag", lag)
	numMetrics := metrics.NewGauge()
	metrics.Register("num_metrics", numMetrics)

	// for a metric to exist in ES at t=Y, there must at least have been 1 point for that metric
	// at a time X where X < Y.  Hence, we confidently say that if we see a metric at Y, we can
	// demand data to show up for that metric at >=Y
	// for our data check to be useful we need metrics to show up in ES soon after being in the pipeline,
	// which seems to be true (see nsqadmin)
	updateTargets := func() {
		getEsTick := time.NewTicker(time.Second * time.Duration(10))
		for ts := range getEsTick.C {
			tsUnix := ts.Unix()
			res, err := http.Get("http://" + os.Args[1] + "/metric/_search?q=*:*&size=10000000")
			perror(err)
			defer res.Body.Close()
			body, err := ioutil.ReadAll(res.Body)
			perror(err)
			var data EsResult
			err = json.Unmarshal(body, &data)
			perror(err)
			amount := len(data.Hits.Hits)
			numMetrics.Update(int64(amount))
			for _, h := range data.Hits.Hits {
				//fmt.Println(h.Source.Name)
				targetsLock.Lock()
				if _, ok := targets[h.Source.Name]; !ok {
					targets[h.Source.Name] = stat{h.Source, tsUnix}
					targetKeys = append(targetKeys, h.Source.Name)
				}
				targetsLock.Unlock()
			}
		}
	}

	test := func(wg *sync.WaitGroup, curTs int64, met stat) {
		g := graphite.HostHeader{Host: "http://" + os.Args[3] + "/render", Header: http.Header{}}
		g.Header.Add("X-Org-Id", strconv.FormatInt(int64(met.def.OrgId), 10))
		g.Header.Set("User-Agent", "graphite-watcher")
		q := graphite.Request{Targets: []string{met.def.Name}}
		series, err := g.Query(&q)
		perror(err)
		for _, serie := range series {
			if met.def.Name != serie.Target {
				fmt.Println("ERROR: name != target name:", met.def.Name, serie.Target)
			}

			lastTs := int64(0)
			oldestNull := int64(math.MaxInt64)
			if len(serie.Datapoints) == 0 {
				fmt.Println("ERROR: series for", met.def.Name, "contains no points!")
			}
			for _, p := range serie.Datapoints {
				ts, err := p[1].Int64()
				if err != nil {
					fmt.Println("ERROR: could not parse timestamp", p)
				}
				if ts <= lastTs {
					fmt.Println("ERROR: timestamp must be bigger than last", lastTs, ts)
				}
				if lastTs == 0 && (ts < curTs-24*3600-60 || ts > curTs-24*3600+60) {
					fmt.Println("ERROR: first point", p, "should have been about 24h ago")
				}
				if lastTs != 0 && ts != lastTs+int64(met.def.Interval) {
					fmt.Println("ERROR: point", p, " is not interval ", met.def.Interval, "apart from previous point")
				}
				_, err = p[0].Float64()
				if err != nil && ts > met.firstSeen {
					if ts < oldestNull {
						oldestNull = ts
					}
					if ts < curTs-30 {
						// will show up in lag metric too
						fmt.Println("ERROR: point has a recent null value", p)
					}
				}
				lastTs = ts
			}
			if lastTs < curTs-int64(met.def.Interval) || lastTs > curTs+int64(met.def.Interval) {
				fmt.Println("ERROR: last point at ", lastTs, "is out of range")
			}
			// if there was no null, we treat the point after the last one we had as null
			if oldestNull == math.MaxInt64 {
				oldestNull = lastTs + int64(met.def.Interval)
			}
			// lag cannot be < 0
			if oldestNull > curTs {
				oldestNull = curTs
			}
			// lag is from first null, even if there were non-nulls after it
			//fmt.Printf("%60s - lag %d\n", name, curTs-oldestNull)
			lag.Update(curTs - oldestNull)
		}
		wg.Done()
	}

	go updateTargets()

	tick := time.NewTicker(time.Millisecond * time.Duration(100))
	wg := &sync.WaitGroup{}
	for ts := range tick.C {
		var t stat
		targetsLock.Lock()
		if len(targetKeys) > 0 {
			key := targetKeys[rand.Intn(len(targets))]
			t = targets[key]
		}
		targetsLock.Unlock()
		if len(targetKeys) > 0 {
			wg.Add(1)
			go test(wg, ts.Unix(), t)
		}
	}
	wg.Wait()
}
<|endoftext|>


<commit_before>package libkbfs

import (
	"golang.org/x/net/context"

	"github.com/keybase/client/go/logger"
	keybase1 "github.com/keybase/client/go/protocol"
)

const connectionStatusConnected keybase1.FSStatusCode = keybase1.FSStatusCode_START
const connectionStatusDisconnected keybase1.FSStatusCode = keybase1.FSStatusCode_ERROR

// ReporterKBPKI implements the Notify function of the Reporter
// interface in addition to embedding ReporterSimple for error
// tracking.  Notify will make RPCs to the keybase daemon.
type ReporterKBPKI struct {
	*ReporterSimple
	config       Config
	log          logger.Logger
	notifyBuffer chan *keybase1.FSNotification
	canceler     func()
}

// NewReporterKBPKI creates a new ReporterKBPKI.
func NewReporterKBPKI(config Config, maxErrors, bufSize int) *ReporterKBPKI {
	r := &ReporterKBPKI{
		ReporterSimple: NewReporterSimple(config.Clock(), maxErrors),
		config:         config,
		log:            config.MakeLogger(""),
		notifyBuffer:   make(chan *keybase1.FSNotification, bufSize),
	}
	var ctx context.Context
	ctx, r.canceler = context.WithCancel(context.Background())
	go r.send(ctx)
	return r
}

// ReportErr implements the Reporter interface for ReporterKBPKI.
func (r *ReporterKBPKI) ReportErr(ctx context.Context, err error) {
	r.ReporterSimple.ReportErr(ctx, err)

	// Fire off error popups
	var n *keybase1.FSNotification
	switch e := err.(type) {
	case ReadAccessError:
		n = readTlfErrorNotification(e.Tlf, e.Public, err)
	case WriteAccessError:
		n = writeTlfErrorNotification(e.Tlf, e.Public, err)
	case NoSuchUserError:
		n = genericErrorNotification(err)
	case UnverifiableTlfUpdateError:
		n = genericErrorNotification(err)
	case NoCurrentSessionError:
		n = genericErrorNotification(err)
	case NeedSelfRekeyError:
		n = readTlfErrorNotification(e.Tlf, e.Public, err)
	case NeedOtherRekeyError:
		n = readTlfErrorNotification(e.Tlf, e.Public, err)
	}

	if n == nil && err == context.DeadlineExceeded {
		n = genericErrorNotification(TimeoutError{})
	}

	if n != nil {
		r.Notify(ctx, n)
	}
}

// Notify implements the Reporter interface for ReporterKBPKI.
//
// TODO: might be useful to get the debug tags out of ctx and store
//       them in the notifyBuffer as well so that send() can put
//       them back in its context.
func (r *ReporterKBPKI) Notify(ctx context.Context, notification *keybase1.FSNotification) {
	select {
	case r.notifyBuffer <- notification:
	default:
		r.log.CDebugf(ctx, "ReporterKBPKI: notify buffer full, dropping %+v",
			notification)
	}
}

// Shutdown implements the Reporter interface for ReporterKBPKI.
func (r *ReporterKBPKI) Shutdown() {
	r.canceler()
	close(r.notifyBuffer)
}

// send takes notifications out of notifyBuffer and sends them to
// the keybase daemon.
func (r *ReporterKBPKI) send(ctx context.Context) {
	for notification := range r.notifyBuffer {
		if err := r.config.KeybaseDaemon().Notify(ctx, notification); err != nil {
			r.log.CDebugf(ctx, "ReporterDaemon: error sending notification: %s",
				err)
		}
	}
}

// writeNotification creates FSNotifications from paths for file
// write events.
func writeNotification(file path, finish bool) *keybase1.FSNotification {
	n := baseNotification(file, finish)
	if file.Tlf.IsPublic() {
		n.NotificationType = keybase1.FSNotificationType_SIGNING
	} else {
		n.NotificationType = keybase1.FSNotificationType_ENCRYPTING
	}
	return n
}

// readNotification creates FSNotifications from paths for file
// read events.
func readNotification(file path, finish bool) *keybase1.FSNotification {
	n := baseNotification(file, finish)
	if file.Tlf.IsPublic() {
		n.NotificationType = keybase1.FSNotificationType_VERIFYING
	} else {
		n.NotificationType = keybase1.FSNotificationType_DECRYPTING
	}
	return n
}

// rekeyNotification creates FSNotifications from TlfHandles for rekey
// events.
func rekeyNotification(ctx context.Context, config Config, handle *TlfHandle, finish bool) *keybase1.FSNotification {
	code := keybase1.FSStatusCode_START
	if finish {
		code = keybase1.FSStatusCode_FINISH
	}

	return &keybase1.FSNotification{
		PublicTopLevelFolder: handle.IsPublic(),
		Filename:             handle.GetCanonicalPath(ctx, config),
		StatusCode:           code,
		NotificationType:     keybase1.FSNotificationType_REKEYING,
	}
}

// connectionNotification creates FSNotifications based on whether
// or not KBFS is online.
func connectionNotification(status keybase1.FSStatusCode) *keybase1.FSNotification {
	// TODO finish placeholder
	return &keybase1.FSNotification{
		NotificationType: keybase1.FSNotificationType_CONNECTION,
		StatusCode:       status,
	}
}

// baseNotification creates a basic FSNotification without a
// NotificationType from a path.
func baseNotification(file path, finish bool) *keybase1.FSNotification {
	code := keybase1.FSStatusCode_START
	if finish {
		code = keybase1.FSStatusCode_FINISH
	}

	return &keybase1.FSNotification{
		PublicTopLevelFolder: file.Tlf.IsPublic(),
		Filename:             file.String(),
		StatusCode:           code,
	}
}

// writeTlfErrorNotification creates FSNotifications for general KBFS
// write error events to a TLF.
func writeTlfErrorNotification(tlf CanonicalTlfName, public bool,
	err error) *keybase1.FSNotification {
	n := &keybase1.FSNotification{
		PublicTopLevelFolder: public,
		Filename:             buildCanonicalPath(public, tlf),
		StatusCode:           keybase1.FSStatusCode_ERROR,
		Status:               err.Error(),
	}
	if public {
		n.NotificationType = keybase1.FSNotificationType_SIGNING
	} else {
		n.NotificationType = keybase1.FSNotificationType_ENCRYPTING
	}
	return n
}

// tlfReadErrorNotification creates FSNotifications for general KBFS
// read error events to a TLF.
func readTlfErrorNotification(tlf CanonicalTlfName, public bool,
	err error) *keybase1.FSNotification {
	n := &keybase1.FSNotification{
		PublicTopLevelFolder: public,
		Filename:             buildCanonicalPath(public, tlf),
		StatusCode:           keybase1.FSStatusCode_ERROR,
		Status:               err.Error(),
	}
	if public {
		n.NotificationType = keybase1.FSNotificationType_VERIFYING
	} else {
		n.NotificationType = keybase1.FSNotificationType_DECRYPTING
	}
	return n
}

// genericErrorNotification creates FSNotifications for generic
// errors, and makes it look like a read error.
func genericErrorNotification(err error) *keybase1.FSNotification {
	// The GUI parses this but doesn't use it if the status is filled
	// in.
	//
	// TODO: plumb through at least the TLF name to wherever generates
	// these errors, and require a path or tlf name here.
	fakeFile := "/"
	n := &keybase1.FSNotification{
		Filename:   fakeFile,
		StatusCode: keybase1.FSStatusCode_ERROR,
		Status:     err.Error(),
	}
	n.NotificationType = keybase1.FSNotificationType_VERIFYING
	return n
}
<commit_msg>reporter_kbpki: suppress username popup errors for auto-lookups<commit_after>package libkbfs

import (
	"golang.org/x/net/context"

	"github.com/keybase/client/go/logger"
	keybase1 "github.com/keybase/client/go/protocol"
)

const connectionStatusConnected keybase1.FSStatusCode = keybase1.FSStatusCode_START
const connectionStatusDisconnected keybase1.FSStatusCode = keybase1.FSStatusCode_ERROR

// noErrorUsernames are user names that should not result in an error
// notification.  These should all be reserved Keybase usernames that
// will never be associated with a real account.
var noErrorUsernames = map[string]bool{
	"objects": true, // git shells
	"gemfile": true, // rvm
	"devfs":   true, // lsof?  KBFS-823
}

// ReporterKBPKI implements the Notify function of the Reporter
// interface in addition to embedding ReporterSimple for error
// tracking.  Notify will make RPCs to the keybase daemon.
type ReporterKBPKI struct {
	*ReporterSimple
	config       Config
	log          logger.Logger
	notifyBuffer chan *keybase1.FSNotification
	canceler     func()
}

// NewReporterKBPKI creates a new ReporterKBPKI.
func NewReporterKBPKI(config Config, maxErrors, bufSize int) *ReporterKBPKI {
	r := &ReporterKBPKI{
		ReporterSimple: NewReporterSimple(config.Clock(), maxErrors),
		config:         config,
		log:            config.MakeLogger(""),
		notifyBuffer:   make(chan *keybase1.FSNotification, bufSize),
	}
	var ctx context.Context
	ctx, r.canceler = context.WithCancel(context.Background())
	go r.send(ctx)
	return r
}

// ReportErr implements the Reporter interface for ReporterKBPKI.
func (r *ReporterKBPKI) ReportErr(ctx context.Context, err error) {
	r.ReporterSimple.ReportErr(ctx, err)

	// Fire off error popups
	var n *keybase1.FSNotification
	switch e := err.(type) {
	case ReadAccessError:
		n = readTlfErrorNotification(e.Tlf, e.Public, err)
	case WriteAccessError:
		n = writeTlfErrorNotification(e.Tlf, e.Public, err)
	case NoSuchUserError:
		if !noErrorUsernames[e.Input] {
			n = genericErrorNotification(err)
		}
	case UnverifiableTlfUpdateError:
		n = genericErrorNotification(err)
	case NoCurrentSessionError:
		n = genericErrorNotification(err)
	case NeedSelfRekeyError:
		n = readTlfErrorNotification(e.Tlf, e.Public, err)
	case NeedOtherRekeyError:
		n = readTlfErrorNotification(e.Tlf, e.Public, err)
	}

	if n == nil && err == context.DeadlineExceeded {
		n = genericErrorNotification(TimeoutError{})
	}

	if n != nil {
		r.Notify(ctx, n)
	}
}

// Notify implements the Reporter interface for ReporterKBPKI.
//
// TODO: might be useful to get the debug tags out of ctx and store
//       them in the notifyBuffer as well so that send() can put
//       them back in its context.
func (r *ReporterKBPKI) Notify(ctx context.Context, notification *keybase1.FSNotification) {
	select {
	case r.notifyBuffer <- notification:
	default:
		r.log.CDebugf(ctx, "ReporterKBPKI: notify buffer full, dropping %+v",
			notification)
	}
}

// Shutdown implements the Reporter interface for ReporterKBPKI.
func (r *ReporterKBPKI) Shutdown() {
	r.canceler()
	close(r.notifyBuffer)
}

// send takes notifications out of notifyBuffer and sends them to
// the keybase daemon.
func (r *ReporterKBPKI) send(ctx context.Context) {
	for notification := range r.notifyBuffer {
		if err := r.config.KeybaseDaemon().Notify(ctx, notification); err != nil {
			r.log.CDebugf(ctx, "ReporterDaemon: error sending notification: %s",
				err)
		}
	}
}

// writeNotification creates FSNotifications from paths for file
// write events.
func writeNotification(file path, finish bool) *keybase1.FSNotification {
	n := baseNotification(file, finish)
	if file.Tlf.IsPublic() {
		n.NotificationType = keybase1.FSNotificationType_SIGNING
	} else {
		n.NotificationType = keybase1.FSNotificationType_ENCRYPTING
	}
	return n
}

// readNotification creates FSNotifications from paths for file
// read events.
func readNotification(file path, finish bool) *keybase1.FSNotification {
	n := baseNotification(file, finish)
	if file.Tlf.IsPublic() {
		n.NotificationType = keybase1.FSNotificationType_VERIFYING
	} else {
		n.NotificationType = keybase1.FSNotificationType_DECRYPTING
	}
	return n
}

// rekeyNotification creates FSNotifications from TlfHandles for rekey
// events.
func rekeyNotification(ctx context.Context, config Config, handle *TlfHandle, finish bool) *keybase1.FSNotification {
	code := keybase1.FSStatusCode_START
	if finish {
		code = keybase1.FSStatusCode_FINISH
	}

	return &keybase1.FSNotification{
		PublicTopLevelFolder: handle.IsPublic(),
		Filename:             handle.GetCanonicalPath(ctx, config),
		StatusCode:           code,
		NotificationType:     keybase1.FSNotificationType_REKEYING,
	}
}

// connectionNotification creates FSNotifications based on whether
// or not KBFS is online.
func connectionNotification(status keybase1.FSStatusCode) *keybase1.FSNotification {
	// TODO finish placeholder
	return &keybase1.FSNotification{
		NotificationType: keybase1.FSNotificationType_CONNECTION,
		StatusCode:       status,
	}
}

// baseNotification creates a basic FSNotification without a
// NotificationType from a path.
func baseNotification(file path, finish bool) *keybase1.FSNotification {
	code := keybase1.FSStatusCode_START
	if finish {
		code = keybase1.FSStatusCode_FINISH
	}

	return &keybase1.FSNotification{
		PublicTopLevelFolder: file.Tlf.IsPublic(),
		Filename:             file.String(),
		StatusCode:           code,
	}
}

// writeTlfErrorNotification creates FSNotifications for general KBFS
// write error events to a TLF.
func writeTlfErrorNotification(tlf CanonicalTlfName, public bool,
	err error) *keybase1.FSNotification {
	n := &keybase1.FSNotification{
		PublicTopLevelFolder: public,
		Filename:             buildCanonicalPath(public, tlf),
		StatusCode:           keybase1.FSStatusCode_ERROR,
		Status:               err.Error(),
	}
	if public {
		n.NotificationType = keybase1.FSNotificationType_SIGNING
	} else {
		n.NotificationType = keybase1.FSNotificationType_ENCRYPTING
	}
	return n
}

// tlfReadErrorNotification creates FSNotifications for general KBFS
// read error events to a TLF.
func readTlfErrorNotification(tlf CanonicalTlfName, public bool,
	err error) *keybase1.FSNotification {
	n := &keybase1.FSNotification{
		PublicTopLevelFolder: public,
		Filename:             buildCanonicalPath(public, tlf),
		StatusCode:           keybase1.FSStatusCode_ERROR,
		Status:               err.Error(),
	}
	if public {
		n.NotificationType = keybase1.FSNotificationType_VERIFYING
	} else {
		n.NotificationType = keybase1.FSNotificationType_DECRYPTING
	}
	return n
}

// genericErrorNotification creates FSNotifications for generic
// errors, and makes it look like a read error.
func genericErrorNotification(err error) *keybase1.FSNotification {
	// The GUI parses this but doesn't use it if the status is filled
	// in.
	//
	// TODO: plumb through at least the TLF name to wherever generates
	// these errors, and require a path or tlf name here.
	fakeFile := "/"
	n := &keybase1.FSNotification{
		Filename:   fakeFile,
		StatusCode: keybase1.FSStatusCode_ERROR,
		Status:     err.Error(),
	}
	n.NotificationType = keybase1.FSNotificationType_VERIFYING
	return n
}
<|endoftext|>


<commit_before>// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.

package model

import (
	"bytes"
	"context"
	"io/ioutil"
	"os"
	"testing"
	"time"

	"github.com/syncthing/syncthing/lib/config"
	"github.com/syncthing/syncthing/lib/db"
	"github.com/syncthing/syncthing/lib/fs"
	"github.com/syncthing/syncthing/lib/protocol"
	"github.com/syncthing/syncthing/lib/scanner"
)

func TestRecvOnlyRevertDeletes(t *testing.T) {
	testOs := &fatalOs{t}

	// Make sure that we delete extraneous files and directories when we hit
	// Revert.

	testOs.RemoveAll("_recvonly")
	defer testOs.RemoveAll("_recvonly")

	// Create some test data

	testOs.MkdirAll("_recvonly/.stfolder", 0755)
	testOs.MkdirAll("_recvonly/ignDir", 0755)
	testOs.MkdirAll("_recvonly/unknownDir", 0755)
	must(t, ioutil.WriteFile("_recvonly/ignDir/ignFile", []byte("hello\n"), 0644))
	must(t, ioutil.WriteFile("_recvonly/unknownDir/unknownFile", []byte("hello\n"), 0644))
	must(t, ioutil.WriteFile("_recvonly/.stignore", []byte("ignDir\n"), 0644))

	knownFiles := setupKnownFiles(t, []byte("hello\n"))

	// Get us a model up and running

	m := setupROFolder()
	defer m.Stop()

	// Send and index update for the known stuff

	m.Index(device1, "ro", knownFiles)
	m.updateLocalsFromScanning("ro", knownFiles)

	size := m.GlobalSize("ro")
	if size.Files != 1 || size.Directories != 1 {
		t.Fatalf("Global: expected 1 file and 1 directory: %+v", size)
	}

	// Start the folder. This will cause a scan, should discover the other stuff in the folder

	m.StartFolder("ro")
	m.ScanFolder("ro")

	// We should now have two files and two directories.

	size = m.GlobalSize("ro")
	if size.Files != 2 || size.Directories != 2 {
		t.Fatalf("Global: expected 2 files and 2 directories: %+v", size)
	}
	size = m.LocalSize("ro")
	if size.Files != 2 || size.Directories != 2 {
		t.Fatalf("Local: expected 2 files and 2 directories: %+v", size)
	}
	size = m.ReceiveOnlyChangedSize("ro")
	if size.Files+size.Directories == 0 {
		t.Fatalf("ROChanged: expected something: %+v", size)
	}

	// Revert should delete the unknown stuff

	m.Revert("ro")

	// These should still exist
	for _, p := range []string{"_recvonly/knownDir/knownFile", "_recvonly/ignDir/ignFile"} {
		_, err := os.Stat(p)
		if err != nil {
			t.Error("Unexpected error:", err)
		}
	}

	// These should have been removed
	for _, p := range []string{"_recvonly/unknownDir", "_recvonly/unknownDir/unknownFile"} {
		_, err := os.Stat(p)
		if !os.IsNotExist(err) {
			t.Error("Unexpected existing thing:", p)
		}
	}

	// We should now have one file and directory again.

	size = m.GlobalSize("ro")
	if size.Files != 1 || size.Directories != 1 {
		t.Fatalf("Global: expected 1 files and 1 directories: %+v", size)
	}
	size = m.LocalSize("ro")
	if size.Files != 1 || size.Directories != 1 {
		t.Fatalf("Local: expected 1 files and 1 directories: %+v", size)
	}
}

func TestRecvOnlyRevertNeeds(t *testing.T) {
	testOs := &fatalOs{t}

	// Make sure that a new file gets picked up and considered latest, then
	// gets considered old when we hit Revert.

	testOs.RemoveAll("_recvonly")
	defer testOs.RemoveAll("_recvonly")

	// Create some test data

	testOs.MkdirAll("_recvonly/.stfolder", 0755)
	oldData := []byte("hello\n")
	knownFiles := setupKnownFiles(t, oldData)

	// Get us a model up and running

	m := setupROFolder()
	defer m.Stop()

	// Send and index update for the known stuff

	m.Index(device1, "ro", knownFiles)
	m.updateLocalsFromScanning("ro", knownFiles)

	// Start the folder. This will cause a scan.

	m.StartFolder("ro")
	m.ScanFolder("ro")

	// Everything should be in sync.

	size := m.GlobalSize("ro")
	if size.Files != 1 || size.Directories != 1 {
		t.Fatalf("Global: expected 1 file and 1 directory: %+v", size)
	}
	size = m.LocalSize("ro")
	if size.Files != 1 || size.Directories != 1 {
		t.Fatalf("Local: expected 1 file and 1 directory: %+v", size)
	}
	size = m.NeedSize("ro")
	if size.Files+size.Directories > 0 {
		t.Fatalf("Need: expected nothing: %+v", size)
	}
	size = m.ReceiveOnlyChangedSize("ro")
	if size.Files+size.Directories > 0 {
		t.Fatalf("ROChanged: expected nothing: %+v", size)
	}

	// Update the file.

	newData := []byte("totally different data\n")
	must(t, ioutil.WriteFile("_recvonly/knownDir/knownFile", newData, 0644))

	// Rescan.

	must(t, m.ScanFolder("ro"))

	// We now have a newer file than the rest of the cluster. Global state should reflect this.

	size = m.GlobalSize("ro")
	const sizeOfDir = 128
	if size.Files != 1 || size.Bytes != sizeOfDir+int64(len(oldData)) {
		t.Fatalf("Global: expected no change due to the new file: %+v", size)
	}
	size = m.LocalSize("ro")
	if size.Files != 1 || size.Bytes != sizeOfDir+int64(len(newData)) {
		t.Fatalf("Local: expected the new file to be reflected: %+v", size)
	}
	size = m.NeedSize("ro")
	if size.Files+size.Directories > 0 {
		t.Fatalf("Need: expected nothing: %+v", size)
	}
	size = m.ReceiveOnlyChangedSize("ro")
	if size.Files+size.Directories == 0 {
		t.Fatalf("ROChanged: expected something: %+v", size)
	}

	// We hit the Revert button. The file that was new should become old.

	m.Revert("ro")

	size = m.GlobalSize("ro")
	if size.Files != 1 || size.Bytes != sizeOfDir+int64(len(oldData)) {
		t.Fatalf("Global: expected the global size to revert: %+v", size)
	}
	size = m.LocalSize("ro")
	if size.Files != 1 || size.Bytes != sizeOfDir+int64(len(newData)) {
		t.Fatalf("Local: expected the local size to remain: %+v", size)
	}
	size = m.NeedSize("ro")
	if size.Files != 1 || size.Bytes != int64(len(oldData)) {
		t.Fatalf("Local: expected to need the old file data: %+v", size)
	}
}

func TestRecvOnlyUndoChanges(t *testing.T) {
	testOs := &fatalOs{t}

	testOs.RemoveAll("_recvonly")
	defer testOs.RemoveAll("_recvonly")

	// Create some test data

	testOs.MkdirAll("_recvonly/.stfolder", 0755)
	oldData := []byte("hello\n")
	knownFiles := setupKnownFiles(t, oldData)

	// Get us a model up and running

	m := setupROFolder()
	defer m.Stop()

	m.fmut.Lock()
	fset := m.folderFiles["ro"]
	m.fmut.Unlock()
	folderFs := fset.MtimeFS()

	// Send and index update for the known stuff

	m.Index(device1, "ro", knownFiles)
	m.updateLocalsFromScanning("ro", knownFiles)

	// Start the folder. This will cause a scan.

	m.StartFolder("ro")
	m.ScanFolder("ro")

	// Everything should be in sync.

	size := m.GlobalSize("ro")
	if size.Files != 1 || size.Directories != 1 {
		t.Fatalf("Global: expected 1 file and 1 directory: %+v", size)
	}
	size = m.LocalSize("ro")
	if size.Files != 1 || size.Directories != 1 {
		t.Fatalf("Local: expected 1 file and 1 directory: %+v", size)
	}
	size = m.NeedSize("ro")
	if size.Files+size.Directories > 0 {
		t.Fatalf("Need: expected nothing: %+v", size)
	}
	size = m.ReceiveOnlyChangedSize("ro")
	if size.Files+size.Directories > 0 {
		t.Fatalf("ROChanged: expected nothing: %+v", size)
	}

	// Create a file and modify another

	file := "_recvonly/foo"
	must(t, ioutil.WriteFile(file, []byte("hello\n"), 0644))

	must(t, ioutil.WriteFile("_recvonly/knownDir/knownFile", []byte("bye\n"), 0644))

	m.ScanFolder("ro")

	size = m.ReceiveOnlyChangedSize("ro")
	if size.Files != 2 {
		t.Fatalf("Receive only: expected 2 files: %+v", size)
	}

	// Remove the file again and undo the modification

	testOs.Remove(file)
	must(t, ioutil.WriteFile("_recvonly/knownDir/knownFile", oldData, 0644))
	folderFs.Chtimes("knownDir/knownFile", knownFiles[1].ModTime(), knownFiles[1].ModTime())

	m.ScanFolder("ro")

	size = m.ReceiveOnlyChangedSize("ro")
	if size.Files+size.Directories+size.Deleted != 0 {
		t.Fatalf("Receive only: expected all zero: %+v", size)
	}
}

func setupKnownFiles(t *testing.T, data []byte) []protocol.FileInfo {
	testOs := &fatalOs{t}

	testOs.MkdirAll("_recvonly/knownDir", 0755)
	must(t, ioutil.WriteFile("_recvonly/knownDir/knownFile", data, 0644))

	t0 := time.Now().Add(-1 * time.Minute)
	testOs.Chtimes("_recvonly/knownDir/knownFile", t0, t0)

	fi := testOs.Stat("_recvonly/knownDir/knownFile")
	blocks, _ := scanner.Blocks(context.TODO(), bytes.NewReader(data), protocol.BlockSize(int64(len(data))), int64(len(data)), nil, true)
	knownFiles := []protocol.FileInfo{
		{
			Name:        "knownDir",
			Type:        protocol.FileInfoTypeDirectory,
			Permissions: 0755,
			Version:     protocol.Vector{Counters: []protocol.Counter{{ID: 42, Value: 42}}},
			Sequence:    42,
		},
		{
			Name:        "knownDir/knownFile",
			Type:        protocol.FileInfoTypeFile,
			Permissions: 0644,
			Size:        fi.Size(),
			ModifiedS:   fi.ModTime().Unix(),
			ModifiedNs:  int32(fi.ModTime().UnixNano() % 1e9),
			Version:     protocol.Vector{Counters: []protocol.Counter{{ID: 42, Value: 42}}},
			Sequence:    42,
			Blocks:      blocks,
		},
	}

	return knownFiles
}

func setupROFolder() *model {
	fcfg := config.NewFolderConfiguration(myID, "ro", "receive only test", fs.FilesystemTypeBasic, "_recvonly")
	fcfg.Type = config.FolderTypeReceiveOnly
	fcfg.Devices = []config.FolderDeviceConfiguration{{DeviceID: device1}}
	fcfg.FSWatcherEnabled = false
	fcfg.RescanIntervalS = 86400

	cfg := defaultCfg.Copy()
	cfg.Folders = append(cfg.Folders, fcfg)

	wrp := createTmpWrapper(cfg)

	db := db.OpenMemory()
	m := newModel(wrp, myID, "syncthing", "dev", db, nil)

	m.ServeBackground()
	m.AddFolder(fcfg)

	return m
}
<commit_msg>lib/model: Run recvonly tests in temporary dirs (#5587)<commit_after>// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.

package model

import (
	"bytes"
	"context"
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"
	"time"

	"github.com/syncthing/syncthing/lib/config"
	"github.com/syncthing/syncthing/lib/db"
	"github.com/syncthing/syncthing/lib/fs"
	"github.com/syncthing/syncthing/lib/protocol"
	"github.com/syncthing/syncthing/lib/scanner"
)

func TestRecvOnlyRevertDeletes(t *testing.T) {
	// Make sure that we delete extraneous files and directories when we hit
	// Revert.

	// Get us a model up and running

	m, fcfg := setupROFolder()
	ffs := fcfg.Filesystem()
	defer os.Remove(m.cfg.ConfigPath())
	defer os.Remove(ffs.URI())
	defer m.Stop()

	// Create some test data

	for _, dir := range []string{".stfolder", "ignDir", "unknownDir"} {
		must(t, ffs.MkdirAll(dir, 0755))
	}
	must(t, ioutil.WriteFile(filepath.Join(ffs.URI(), "ignDir/ignFile"), []byte("hello\n"), 0644))
	must(t, ioutil.WriteFile(filepath.Join(ffs.URI(), "unknownDir/unknownFile"), []byte("hello\n"), 0644))
	must(t, ioutil.WriteFile(filepath.Join(ffs.URI(), ".stignore"), []byte("ignDir\n"), 0644))

	knownFiles := setupKnownFiles(t, ffs, []byte("hello\n"))

	// Send and index update for the known stuff

	m.Index(device1, "ro", knownFiles)
	m.updateLocalsFromScanning("ro", knownFiles)

	size := m.GlobalSize("ro")
	if size.Files != 1 || size.Directories != 1 {
		t.Fatalf("Global: expected 1 file and 1 directory: %+v", size)
	}

	// Start the folder. This will cause a scan, should discover the other stuff in the folder

	m.StartFolder("ro")
	m.ScanFolder("ro")

	// We should now have two files and two directories.

	size = m.GlobalSize("ro")
	if size.Files != 2 || size.Directories != 2 {
		t.Fatalf("Global: expected 2 files and 2 directories: %+v", size)
	}
	size = m.LocalSize("ro")
	if size.Files != 2 || size.Directories != 2 {
		t.Fatalf("Local: expected 2 files and 2 directories: %+v", size)
	}
	size = m.ReceiveOnlyChangedSize("ro")
	if size.Files+size.Directories == 0 {
		t.Fatalf("ROChanged: expected something: %+v", size)
	}

	// Revert should delete the unknown stuff

	m.Revert("ro")

	// These should still exist
	for _, p := range []string{"knownDir/knownFile", "ignDir/ignFile"} {
		if _, err := ffs.Stat(p); err != nil {
			t.Error("Unexpected error:", err)
		}
	}

	// These should have been removed
	for _, p := range []string{"unknownDir", "unknownDir/unknownFile"} {
		if _, err := ffs.Stat(p); !fs.IsNotExist(err) {
			t.Error("Unexpected existing thing:", p)
		}
	}

	// We should now have one file and directory again.

	size = m.GlobalSize("ro")
	if size.Files != 1 || size.Directories != 1 {
		t.Fatalf("Global: expected 1 files and 1 directories: %+v", size)
	}
	size = m.LocalSize("ro")
	if size.Files != 1 || size.Directories != 1 {
		t.Fatalf("Local: expected 1 files and 1 directories: %+v", size)
	}
}

func TestRecvOnlyRevertNeeds(t *testing.T) {
	// Make sure that a new file gets picked up and considered latest, then
	// gets considered old when we hit Revert.

	// Get us a model up and running

	m, fcfg := setupROFolder()
	ffs := fcfg.Filesystem()
	defer os.Remove(m.cfg.ConfigPath())
	defer os.Remove(ffs.URI())
	defer m.Stop()

	// Create some test data

	must(t, ffs.MkdirAll(".stfolder", 0755))
	oldData := []byte("hello\n")
	knownFiles := setupKnownFiles(t, ffs, oldData)

	// Send and index update for the known stuff

	m.Index(device1, "ro", knownFiles)
	m.updateLocalsFromScanning("ro", knownFiles)

	// Start the folder. This will cause a scan.

	m.StartFolder("ro")
	m.ScanFolder("ro")

	// Everything should be in sync.

	size := m.GlobalSize("ro")
	if size.Files != 1 || size.Directories != 1 {
		t.Fatalf("Global: expected 1 file and 1 directory: %+v", size)
	}
	size = m.LocalSize("ro")
	if size.Files != 1 || size.Directories != 1 {
		t.Fatalf("Local: expected 1 file and 1 directory: %+v", size)
	}
	size = m.NeedSize("ro")
	if size.Files+size.Directories > 0 {
		t.Fatalf("Need: expected nothing: %+v", size)
	}
	size = m.ReceiveOnlyChangedSize("ro")
	if size.Files+size.Directories > 0 {
		t.Fatalf("ROChanged: expected nothing: %+v", size)
	}

	// Update the file.

	newData := []byte("totally different data\n")
	must(t, ioutil.WriteFile(filepath.Join(ffs.URI(), "knownDir/knownFile"), newData, 0644))

	// Rescan.

	must(t, m.ScanFolder("ro"))

	// We now have a newer file than the rest of the cluster. Global state should reflect this.

	size = m.GlobalSize("ro")
	const sizeOfDir = 128
	if size.Files != 1 || size.Bytes != sizeOfDir+int64(len(oldData)) {
		t.Fatalf("Global: expected no change due to the new file: %+v", size)
	}
	size = m.LocalSize("ro")
	if size.Files != 1 || size.Bytes != sizeOfDir+int64(len(newData)) {
		t.Fatalf("Local: expected the new file to be reflected: %+v", size)
	}
	size = m.NeedSize("ro")
	if size.Files+size.Directories > 0 {
		t.Fatalf("Need: expected nothing: %+v", size)
	}
	size = m.ReceiveOnlyChangedSize("ro")
	if size.Files+size.Directories == 0 {
		t.Fatalf("ROChanged: expected something: %+v", size)
	}

	// We hit the Revert button. The file that was new should become old.

	m.Revert("ro")

	size = m.GlobalSize("ro")
	if size.Files != 1 || size.Bytes != sizeOfDir+int64(len(oldData)) {
		t.Fatalf("Global: expected the global size to revert: %+v", size)
	}
	size = m.LocalSize("ro")
	if size.Files != 1 || size.Bytes != sizeOfDir+int64(len(newData)) {
		t.Fatalf("Local: expected the local size to remain: %+v", size)
	}
	size = m.NeedSize("ro")
	if size.Files != 1 || size.Bytes != int64(len(oldData)) {
		t.Fatalf("Local: expected to need the old file data: %+v", size)
	}
}

func TestRecvOnlyUndoChanges(t *testing.T) {
	testOs := &fatalOs{t}

	// Get us a model up and running

	m, fcfg := setupROFolder()
	ffs := fcfg.Filesystem()
	defer os.Remove(m.cfg.ConfigPath())
	defer os.Remove(ffs.URI())
	defer m.Stop()

	// Create some test data

	must(t, ffs.MkdirAll(".stfolder", 0755))
	oldData := []byte("hello\n")
	knownFiles := setupKnownFiles(t, ffs, oldData)

	m.fmut.Lock()
	fset := m.folderFiles["ro"]
	m.fmut.Unlock()
	folderFs := fset.MtimeFS()

	// Send and index update for the known stuff

	m.Index(device1, "ro", knownFiles)
	m.updateLocalsFromScanning("ro", knownFiles)

	// Start the folder. This will cause a scan.

	m.StartFolder("ro")
	m.ScanFolder("ro")

	// Everything should be in sync.

	size := m.GlobalSize("ro")
	if size.Files != 1 || size.Directories != 1 {
		t.Fatalf("Global: expected 1 file and 1 directory: %+v", size)
	}
	size = m.LocalSize("ro")
	if size.Files != 1 || size.Directories != 1 {
		t.Fatalf("Local: expected 1 file and 1 directory: %+v", size)
	}
	size = m.NeedSize("ro")
	if size.Files+size.Directories > 0 {
		t.Fatalf("Need: expected nothing: %+v", size)
	}
	size = m.ReceiveOnlyChangedSize("ro")
	if size.Files+size.Directories > 0 {
		t.Fatalf("ROChanged: expected nothing: %+v", size)
	}

	// Create a file and modify another

	file := filepath.Join(ffs.URI(), "foo")
	must(t, ioutil.WriteFile(file, []byte("hello\n"), 0644))

	must(t, ioutil.WriteFile(filepath.Join(ffs.URI(), "knownDir/knownFile"), []byte("bye\n"), 0644))

	m.ScanFolder("ro")

	size = m.ReceiveOnlyChangedSize("ro")
	if size.Files != 2 {
		t.Fatalf("Receive only: expected 2 files: %+v", size)
	}

	// Remove the file again and undo the modification

	testOs.Remove(file)
	must(t, ioutil.WriteFile(filepath.Join(ffs.URI(), "knownDir/knownFile"), oldData, 0644))
	folderFs.Chtimes("knownDir/knownFile", knownFiles[1].ModTime(), knownFiles[1].ModTime())

	m.ScanFolder("ro")

	size = m.ReceiveOnlyChangedSize("ro")
	if size.Files+size.Directories+size.Deleted != 0 {
		t.Fatalf("Receive only: expected all zero: %+v", size)
	}
}

func setupKnownFiles(t *testing.T, ffs fs.Filesystem, data []byte) []protocol.FileInfo {
	t.Helper()

	must(t, ffs.MkdirAll("knownDir", 0755))
	must(t, ioutil.WriteFile(filepath.Join(ffs.URI(), "knownDir/knownFile"), data, 0644))

	t0 := time.Now().Add(-1 * time.Minute)
	must(t, ffs.Chtimes("knownDir/knownFile", t0, t0))

	fi, err := ffs.Stat("knownDir/knownFile")
	if err != nil {
		t.Fatal(err)
	}
	blocks, _ := scanner.Blocks(context.TODO(), bytes.NewReader(data), protocol.BlockSize(int64(len(data))), int64(len(data)), nil, true)
	knownFiles := []protocol.FileInfo{
		{
			Name:        "knownDir",
			Type:        protocol.FileInfoTypeDirectory,
			Permissions: 0755,
			Version:     protocol.Vector{Counters: []protocol.Counter{{ID: 42, Value: 42}}},
			Sequence:    42,
		},
		{
			Name:        "knownDir/knownFile",
			Type:        protocol.FileInfoTypeFile,
			Permissions: 0644,
			Size:        fi.Size(),
			ModifiedS:   fi.ModTime().Unix(),
			ModifiedNs:  int32(fi.ModTime().UnixNano() % 1e9),
			Version:     protocol.Vector{Counters: []protocol.Counter{{ID: 42, Value: 42}}},
			Sequence:    42,
			Blocks:      blocks,
		},
	}

	return knownFiles
}

func setupROFolder() (*model, config.FolderConfiguration) {
	w := createTmpWrapper(defaultCfg)
	fcfg := testFolderConfigTmp()
	fcfg.ID = "ro"
	fcfg.Type = config.FolderTypeReceiveOnly
	w.SetFolder(fcfg)

	m := newModel(w, myID, "syncthing", "dev", db.OpenMemory(), nil)
	m.ServeBackground()
	m.AddFolder(fcfg)

	return m, fcfg

}
<|endoftext|>