<commit_before>package logs

import (
	"net/http"
	"strconv"
	"time"

	"github.com/gorilla/websocket"
	"github.com/pivotal-golang/lager"

	"github.com/concourse/atc/auth"
	"github.com/concourse/atc/config"
	"github.com/concourse/atc/db"
	"github.com/concourse/atc/logfanout"
)

var upgrader = websocket.Upgrader{
	CheckOrigin: func(*http.Request) bool {
		return true
	},
}

func NewHandler(
	logger lager.Logger,
	validator auth.Validator,
	jobs config.Jobs,
	tracker *logfanout.Tracker,
	db db.DB,
) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		buildIDStr := r.FormValue(":build_id")

		log := logger.Session("logs-out", lager.Data{
			"build_id": buildIDStr,
		})

		buildID, err := strconv.Atoi(buildIDStr)
		if err != nil {
			log.Error("invalid-build-id", err)
			return
		}

		authenticated := validator.IsAuthenticated(r)

		if !authenticated {
			build, err := db.GetBuild(buildID)
			if err != nil {
				log.Error("invalid-build-id", err)
				w.WriteHeader(http.StatusNotFound)
				return
			}

			job, found := jobs.Lookup(build.JobName)
			if !found || !job.Public {
				w.WriteHeader(http.StatusNotFound)
				return
			}
		}

		conn, err := upgrader.Upgrade(w, r, nil)
		if err != nil {
			log.Error("upgrade-failed", err)
			return
		}

		defer conn.Close()

		logFanout := tracker.Register(buildID, conn)
		defer tracker.Unregister(buildID, conn)

		var sink logfanout.Sink
		if authenticated {
			sink = logfanout.NewRawSink(conn)
		} else {
			sink = logfanout.NewCensoredSink(conn)
		}

		sink = logfanout.NewAsyncSink(sink, 1000)

		err = logFanout.Attach(sink)
		if err != nil {
			log.Error("attach-failed", err)
			conn.Close()
			return
		}

		for {
			time.Sleep(5 * time.Second)

			err := conn.WriteControl(websocket.PingMessage, []byte("ping"), time.Time{})
			if err != nil {
				log.Error("ping-failed", err)
				break
			}
		}
	})
}
<commit_msg>more reliable dead connection reaping<commit_after>package logs

import (
	"net/http"
	"strconv"
	"time"

	"github.com/gorilla/websocket"
	"github.com/pivotal-golang/lager"

	"github.com/concourse/atc/auth"
	"github.com/concourse/atc/config"
	"github.com/concourse/atc/db"
	"github.com/concourse/atc/logfanout"
)

const pingInterval = 5 * time.Second

var upgrader = websocket.Upgrader{
	CheckOrigin: func(*http.Request) bool {
		return true
	},
}

func NewHandler(
	logger lager.Logger,
	validator auth.Validator,
	jobs config.Jobs,
	tracker *logfanout.Tracker,
	db db.DB,
) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		buildIDStr := r.FormValue(":build_id")

		log := logger.Session("logs-out", lager.Data{
			"build_id": buildIDStr,
		})

		buildID, err := strconv.Atoi(buildIDStr)
		if err != nil {
			log.Error("invalid-build-id", err)
			return
		}

		authenticated := validator.IsAuthenticated(r)

		if !authenticated {
			build, err := db.GetBuild(buildID)
			if err != nil {
				log.Error("invalid-build-id", err)
				w.WriteHeader(http.StatusNotFound)
				return
			}

			job, found := jobs.Lookup(build.JobName)
			if !found || !job.Public {
				w.WriteHeader(http.StatusNotFound)
				return
			}
		}

		conn, err := upgrader.Upgrade(w, r, nil)
		if err != nil {
			log.Error("upgrade-failed", err)
			return
		}

		defer conn.Close()

		pongTimer := time.NewTimer(pingInterval * 2)

		conn.SetPongHandler(func(string) error {
			pongTimer.Reset(pingInterval * 2)
			return nil
		})

		logFanout := tracker.Register(buildID, conn)
		defer tracker.Unregister(buildID, conn)

		var sink logfanout.Sink
		if authenticated {
			sink = logfanout.NewRawSink(conn)
		} else {
			sink = logfanout.NewCensoredSink(conn)
		}

		sink = logfanout.NewAsyncSink(sink, 1000)

		err = logFanout.Attach(sink)
		if err != nil {
			log.Error("attach-failed", err)
			conn.Close()
			return
		}

		go func() {
			for {
				_, _, err := conn.ReadMessage()
				if err != nil {
					return
				}
			}
		}()

		for {
			select {
			case <-pongTimer.C:
				log.Debug("connection-expired")
				return

			case <-time.After(pingInterval):
				err := conn.WriteControl(websocket.PingMessage, []byte("ping"), time.Now().Add(pingInterval))
				if err != nil {
					return
				}
			}
		}
	})
}
<|endoftext|>


<commit_before>package trace

import (
	"context"
	"errors"
	"fmt"
	"net"
	"time"

	"github.com/stripe/veneur/protocol"
	"github.com/stripe/veneur/ssf"
)

func init() {
	cl, err := NewClient(DefaultVeneurAddress)
	if err != nil {
		return
	}
	DefaultClient = cl
}

// op is a function invoked on a backend, such as sending a span
// synchronously, flushing the backend buffer, or closing the backend.
type op func(context.Context, ClientBackend)

// Client is a Client that sends traces to Veneur over the network. It
// represents a pump for span packets from user code to the network
// (whether it be UDP or streaming sockets, with or without buffers).
//
// Structure
//
// A Client is composed of two parts (each with its own purpose): A
// serialization part providing backpressure (the front end) and a
// backend (which is called on a single goroutine).
type Client struct {
	backend ClientBackend
	cap     uint
	cancel  context.CancelFunc
	flush   func(context.Context)
	ops     chan op
}

// Close tears down the entire client. It waits until the backend has
// closed the network connection (if one was established) and returns
// any error from closing the connection.
func (c *Client) Close() error {
	ch := make(chan error)
	c.cancel()
	c.ops <- func(ctx context.Context, s ClientBackend) {
		ch <- s.Close()
	}
	close(c.ops)
	return <-ch
}

func (c *Client) run(ctx context.Context) {
	if c.flush != nil {
		go c.flush(ctx)
	}
	for {
		do, ok := <-c.ops
		if !ok {
			return
		}
		do(ctx, c.backend)
	}
}

// ClientParam is an option for NewClient. Its implementation borrows
// from Dave Cheney's functional options API
// (https://dave.cheney.net/2014/10/17/functional-options-for-friendly-apis).
//
// Unless otherwise noted, ClientParams only apply to networked
// backends (i.e., those used by NewClient). Using them on
// non-network-backed clients will return ErrClientNotNetworked on
// client creation.
type ClientParam func(*Client) error

// ErrClientNotNetworked indicates that the client being constructed
// does not support options relevant only to networked clients.
var ErrClientNotNetworked = fmt.Errorf("client is not using a network backend")

// Capacity indicates how many spans a client's channel should
// accommodate. This parameter can be used on both generic and
// networked backends.
func Capacity(n uint) ClientParam {
	return func(cl *Client) error {
		cl.cap = n
		return nil
	}
}

// Buffered sets the client to be buffered with the default buffer
// size (enough to accomodate a single, maximum-sized SSF frame,
// currently about 16MB).
//
// When using buffered clients, since buffers tend to be large and SSF
// packets are fairly small, it might appear as if buffered clients
// are not sending any spans at all.
//
// Code using a buffered client should ensure that the client gets
// flushed in a reasonable interval, either by calling Flush manually
// in an appropriate goroutine, or by also using the FlushInterval
// functional option.
func Buffered(cl *Client) error {
	return BufferedSize(uint(BufferSize))(cl)
}

// BufferedSize indicates that a client should have a buffer size
// bytes large. See the note on the Buffered option about flushing the
// buffer.
func BufferedSize(size uint) ClientParam {
	return func(cl *Client) error {
		if nb, ok := cl.backend.(networkBackend); ok {
			nb.params().bufferSize = size
			return nil
		}
		return ErrClientNotNetworked
	}
}

// FlushInterval sets up a buffered client to perform one synchronous
// flush per time interval in a new goroutine. The goroutine closes
// down when the Client's Close method is called.
//
// This uses a time.Ticker to trigger the flush, so will not trigger
// multiple times if flushing should be slower than the trigger
// interval.
func FlushInterval(interval time.Duration) ClientParam {
	t := time.NewTicker(interval)
	return FlushChannel(t.C, t.Stop)
}

// FlushChannel sets up a buffered client to perform one synchronous
// flush any time the given channel has a Time element ready. When the
// Client is closed, FlushWith invokes the passed stop function.
//
// This functional option is mostly useful for tests; code intended to
// be used in production should rely on FlushInterval instead, as
// time.Ticker is set up to deal with slow flushes.
func FlushChannel(ch <-chan time.Time, stop func()) ClientParam {
	return func(cl *Client) error {
		if _, ok := cl.backend.(networkBackend); !ok {
			return ErrClientNotNetworked
		}
		cl.flush = func(ctx context.Context) {
			defer stop()
			for {
				select {
				case <-ch:
					_ = Flush(cl)
				case <-ctx.Done():
					return
				}
			}
		}
		return nil
	}
}

// BackoffTime sets the time increment that backoff time is increased
// (linearly) between every reconnection attempt the backend makes. If
// this option is not used, the backend uses DefaultBackoff.
func BackoffTime(t time.Duration) ClientParam {
	return func(cl *Client) error {
		if nb, ok := cl.backend.(networkBackend); ok {
			nb.params().backoff = t
			return nil
		}
		return ErrClientNotNetworked
	}
}

// MaxBackoffTime sets the maximum time duration waited between
// reconnection attempts. If this option is not used, the backend uses
// DefaultMaxBackoff.
func MaxBackoffTime(t time.Duration) ClientParam {
	return func(cl *Client) error {
		if nb, ok := cl.backend.(networkBackend); ok {
			nb.params().maxBackoff = t
			return nil
		}
		return ErrClientNotNetworked
	}
}

// ConnectTimeout sets the maximum total amount of time a client
// backend spends trying to establish a connection to a veneur. If a
// connection can not be established after this timeout has expired
// (counting from the time the connection is first attempted), the
// span is discarded. If this option is not used, the backend uses
// DefaultConnectTimeout.
func ConnectTimeout(t time.Duration) ClientParam {
	return func(cl *Client) error {
		if nb, ok := cl.backend.(networkBackend); ok {
			nb.params().connectTimeout = t
			return nil
		}
		return ErrClientNotNetworked
	}
}

// NewClient constructs a new client that will attempt to connect
// to addrStr (an address in veneur URL format) using the parameters
// in opts. It returns the constructed client or an error.
func NewClient(addrStr string, opts ...ClientParam) (*Client, error) {
	addr, err := protocol.ResolveAddr(addrStr)
	if err != nil {
		return nil, err
	}
	cl := &Client{}
	var nb networkBackend
	switch addr := addr.(type) {
	case *net.UDPAddr:
		nb = &packetBackend{}
	case *net.UnixAddr:
		nb = &streamBackend{}
	default:
		return nil, fmt.Errorf("can not connect to %v addresses", addr.Network())
	}
	cl.backend = nb
	params := nb.params()
	params.addr = addr
	cl.cap = DefaultCapacity
	for _, opt := range opts {
		if err = opt(cl); err != nil {
			return nil, err
		}
	}
	ch := make(chan op, cl.cap)
	cl.ops = ch
	ctx := context.Background()
	ctx, cl.cancel = context.WithCancel(ctx)
	go cl.run(ctx)
	return cl, nil
}

// NewBackendClient constructs and returns a Client sending to the
// ClientBackend passed. Most user code should use NewClient, as
// NewBackendClient is primarily useful for processing spans
// internally (e.g. in veneur itself or in test code), without making
// trips over the network.
func NewBackendClient(b ClientBackend, opts ...ClientParam) (*Client, error) {
	cl := &Client{}
	cl.backend = b
	cl.cap = 1

	for _, opt := range opts {
		if err := opt(cl); err != nil {
			return nil, err
		}
	}
	cl.ops = make(chan op, cl.cap)
	ctx := context.Background()
	ctx, cl.cancel = context.WithCancel(ctx)
	go cl.run(ctx)
	return cl, nil
}

// DefaultClient is the client that trace recording happens on by
// default. If it is nil, no recording happens and ErrNoClient is
// returned from recording functions.
//
// Note that it is not safe to set this variable concurrently with
// other goroutines that use the DefaultClient.
var DefaultClient *Client

// DefaultCapacity is the capacity of the span submission queue in a
// veneur client.
const DefaultCapacity = 64

// DefaultVeneurAddress is the address that a reasonable veneur should
// listen on. Currently it defaults to UDP port 8128.
const DefaultVeneurAddress string = "udp://127.0.0.1:8128"

// ErrNoClient indicates that no client is yet initialized.
var ErrNoClient = errors.New("client is not initialized")

// ErrWouldBlock indicates that a client is not able to send a span at
// the current time.
var ErrWouldBlock = errors.New("sending span would block")

// Record instructs the client to serialize and send a span. It does
// not wait for a delivery attempt, instead the Client will send the
// result from serializing and submitting the span to the channel
// done, if it is non-nil.
//
// Record returns ErrNoClient if client is nil and ErrWouldBlock if
// the client is not able to accomodate another span.
func Record(cl *Client, span *ssf.SSFSpan, done chan<- error) error {
	if cl == nil {
		return ErrNoClient
	}

	op := func(ctx context.Context, s ClientBackend) {
		err := s.SendSync(ctx, span)
		if done != nil {
			done <- err
		}
	}
	select {
	case cl.ops <- op:
		return nil
	default:
	}
	return ErrWouldBlock
}

// Flush instructs a client to flush to the upstream veneur all the
// spans that were serialized up until the moment that the flush was
// received. It will wait until the flush is completed (including all
// reconnection attempts), and return any error caused by flushing the
// buffer.
//
// Flush returns ErrNoClient if client is nil and ErrWouldBlock if the
// client is not able to take more requests.
func Flush(cl *Client) error {
	ch := make(chan error)
	err := FlushAsync(cl, ch)
	if err != nil {
		return err
	}
	return <-ch
}

// FlushAsync instructs a buffered client to flush to the upstream
// veneur all the spans that were serialized up until the moment that
// the flush was received. Once the client has completed the flush,
// any error (or nil) is sent down the error channel.
//
// FlushAsync returns ErrNoClient if client is nil and ErrWouldBlock
// if the client is not able to take more requests.
func FlushAsync(cl *Client, ch chan<- error) error {
	if cl == nil {
		return ErrNoClient
	}
	op := func(ctx context.Context, s ClientBackend) {
		err := s.FlushSync(ctx)
		if ch != nil {
			ch <- err
		}
	}
	select {
	case cl.ops <- op:
		return nil
	default:
	}
	return ErrWouldBlock
}
<commit_msg>Update the number of flushed/records that failed on a client<commit_after>package trace

import (
	"context"
	"errors"
	"fmt"
	"net"
	"time"

	"sync/atomic"

	"github.com/stripe/veneur/protocol"
	"github.com/stripe/veneur/ssf"
)

func init() {
	cl, err := NewClient(DefaultVeneurAddress)
	if err != nil {
		return
	}
	DefaultClient = cl
}

// op is a function invoked on a backend, such as sending a span
// synchronously, flushing the backend buffer, or closing the backend.
type op func(context.Context, ClientBackend)

// Client is a Client that sends traces to Veneur over the network. It
// represents a pump for span packets from user code to the network
// (whether it be UDP or streaming sockets, with or without buffers).
//
// Structure
//
// A Client is composed of two parts (each with its own purpose): A
// serialization part providing backpressure (the front end) and a
// backend (which is called on a single goroutine).
type Client struct {
	backend ClientBackend
	cap     uint
	cancel  context.CancelFunc
	flush   func(context.Context)
	ops     chan op

	failedFlushes int64
	failedRecords int64
}

// Close tears down the entire client. It waits until the backend has
// closed the network connection (if one was established) and returns
// any error from closing the connection.
func (c *Client) Close() error {
	ch := make(chan error)
	c.cancel()
	c.ops <- func(ctx context.Context, s ClientBackend) {
		ch <- s.Close()
	}
	close(c.ops)
	return <-ch
}

func (c *Client) run(ctx context.Context) {
	if c.flush != nil {
		go c.flush(ctx)
	}
	for {
		do, ok := <-c.ops
		if !ok {
			return
		}
		do(ctx, c.backend)
	}
}

// ClientParam is an option for NewClient. Its implementation borrows
// from Dave Cheney's functional options API
// (https://dave.cheney.net/2014/10/17/functional-options-for-friendly-apis).
//
// Unless otherwise noted, ClientParams only apply to networked
// backends (i.e., those used by NewClient). Using them on
// non-network-backed clients will return ErrClientNotNetworked on
// client creation.
type ClientParam func(*Client) error

// ErrClientNotNetworked indicates that the client being constructed
// does not support options relevant only to networked clients.
var ErrClientNotNetworked = fmt.Errorf("client is not using a network backend")

// Capacity indicates how many spans a client's channel should
// accommodate. This parameter can be used on both generic and
// networked backends.
func Capacity(n uint) ClientParam {
	return func(cl *Client) error {
		cl.cap = n
		return nil
	}
}

// Buffered sets the client to be buffered with the default buffer
// size (enough to accomodate a single, maximum-sized SSF frame,
// currently about 16MB).
//
// When using buffered clients, since buffers tend to be large and SSF
// packets are fairly small, it might appear as if buffered clients
// are not sending any spans at all.
//
// Code using a buffered client should ensure that the client gets
// flushed in a reasonable interval, either by calling Flush manually
// in an appropriate goroutine, or by also using the FlushInterval
// functional option.
func Buffered(cl *Client) error {
	return BufferedSize(uint(BufferSize))(cl)
}

// BufferedSize indicates that a client should have a buffer size
// bytes large. See the note on the Buffered option about flushing the
// buffer.
func BufferedSize(size uint) ClientParam {
	return func(cl *Client) error {
		if nb, ok := cl.backend.(networkBackend); ok {
			nb.params().bufferSize = size
			return nil
		}
		return ErrClientNotNetworked
	}
}

// FlushInterval sets up a buffered client to perform one synchronous
// flush per time interval in a new goroutine. The goroutine closes
// down when the Client's Close method is called.
//
// This uses a time.Ticker to trigger the flush, so will not trigger
// multiple times if flushing should be slower than the trigger
// interval.
func FlushInterval(interval time.Duration) ClientParam {
	t := time.NewTicker(interval)
	return FlushChannel(t.C, t.Stop)
}

// FlushChannel sets up a buffered client to perform one synchronous
// flush any time the given channel has a Time element ready. When the
// Client is closed, FlushWith invokes the passed stop function.
//
// This functional option is mostly useful for tests; code intended to
// be used in production should rely on FlushInterval instead, as
// time.Ticker is set up to deal with slow flushes.
func FlushChannel(ch <-chan time.Time, stop func()) ClientParam {
	return func(cl *Client) error {
		if _, ok := cl.backend.(networkBackend); !ok {
			return ErrClientNotNetworked
		}
		cl.flush = func(ctx context.Context) {
			defer stop()
			for {
				select {
				case <-ch:
					_ = Flush(cl)
				case <-ctx.Done():
					return
				}
			}
		}
		return nil
	}
}

// BackoffTime sets the time increment that backoff time is increased
// (linearly) between every reconnection attempt the backend makes. If
// this option is not used, the backend uses DefaultBackoff.
func BackoffTime(t time.Duration) ClientParam {
	return func(cl *Client) error {
		if nb, ok := cl.backend.(networkBackend); ok {
			nb.params().backoff = t
			return nil
		}
		return ErrClientNotNetworked
	}
}

// MaxBackoffTime sets the maximum time duration waited between
// reconnection attempts. If this option is not used, the backend uses
// DefaultMaxBackoff.
func MaxBackoffTime(t time.Duration) ClientParam {
	return func(cl *Client) error {
		if nb, ok := cl.backend.(networkBackend); ok {
			nb.params().maxBackoff = t
			return nil
		}
		return ErrClientNotNetworked
	}
}

// ConnectTimeout sets the maximum total amount of time a client
// backend spends trying to establish a connection to a veneur. If a
// connection can not be established after this timeout has expired
// (counting from the time the connection is first attempted), the
// span is discarded. If this option is not used, the backend uses
// DefaultConnectTimeout.
func ConnectTimeout(t time.Duration) ClientParam {
	return func(cl *Client) error {
		if nb, ok := cl.backend.(networkBackend); ok {
			nb.params().connectTimeout = t
			return nil
		}
		return ErrClientNotNetworked
	}
}

// NewClient constructs a new client that will attempt to connect
// to addrStr (an address in veneur URL format) using the parameters
// in opts. It returns the constructed client or an error.
func NewClient(addrStr string, opts ...ClientParam) (*Client, error) {
	addr, err := protocol.ResolveAddr(addrStr)
	if err != nil {
		return nil, err
	}
	cl := &Client{}
	var nb networkBackend
	switch addr := addr.(type) {
	case *net.UDPAddr:
		nb = &packetBackend{}
	case *net.UnixAddr:
		nb = &streamBackend{}
	default:
		return nil, fmt.Errorf("can not connect to %v addresses", addr.Network())
	}
	cl.backend = nb
	params := nb.params()
	params.addr = addr
	cl.cap = DefaultCapacity
	for _, opt := range opts {
		if err = opt(cl); err != nil {
			return nil, err
		}
	}
	ch := make(chan op, cl.cap)
	cl.ops = ch
	ctx := context.Background()
	ctx, cl.cancel = context.WithCancel(ctx)
	go cl.run(ctx)
	return cl, nil
}

// NewBackendClient constructs and returns a Client sending to the
// ClientBackend passed. Most user code should use NewClient, as
// NewBackendClient is primarily useful for processing spans
// internally (e.g. in veneur itself or in test code), without making
// trips over the network.
func NewBackendClient(b ClientBackend, opts ...ClientParam) (*Client, error) {
	cl := &Client{}
	cl.backend = b
	cl.cap = 1

	for _, opt := range opts {
		if err := opt(cl); err != nil {
			return nil, err
		}
	}
	cl.ops = make(chan op, cl.cap)
	ctx := context.Background()
	ctx, cl.cancel = context.WithCancel(ctx)
	go cl.run(ctx)
	return cl, nil
}

// DefaultClient is the client that trace recording happens on by
// default. If it is nil, no recording happens and ErrNoClient is
// returned from recording functions.
//
// Note that it is not safe to set this variable concurrently with
// other goroutines that use the DefaultClient.
var DefaultClient *Client

// DefaultCapacity is the capacity of the span submission queue in a
// veneur client.
const DefaultCapacity = 64

// DefaultVeneurAddress is the address that a reasonable veneur should
// listen on. Currently it defaults to UDP port 8128.
const DefaultVeneurAddress string = "udp://127.0.0.1:8128"

// ErrNoClient indicates that no client is yet initialized.
var ErrNoClient = errors.New("client is not initialized")

// ErrWouldBlock indicates that a client is not able to send a span at
// the current time.
var ErrWouldBlock = errors.New("sending span would block")

// Record instructs the client to serialize and send a span. It does
// not wait for a delivery attempt, instead the Client will send the
// result from serializing and submitting the span to the channel
// done, if it is non-nil.
//
// Record returns ErrNoClient if client is nil and ErrWouldBlock if
// the client is not able to accomodate another span.
func Record(cl *Client, span *ssf.SSFSpan, done chan<- error) error {
	if cl == nil {
		return ErrNoClient
	}

	op := func(ctx context.Context, s ClientBackend) {
		err := s.SendSync(ctx, span)
		if done != nil {
			done <- err
		}
	}
	select {
	case cl.ops <- op:
		return nil
	default:
	}
	atomic.AddInt64(&cl.failedRecords, 1)
	return ErrWouldBlock
}

// Flush instructs a client to flush to the upstream veneur all the
// spans that were serialized up until the moment that the flush was
// received. It will wait until the flush is completed (including all
// reconnection attempts), and return any error caused by flushing the
// buffer.
//
// Flush returns ErrNoClient if client is nil and ErrWouldBlock if the
// client is not able to take more requests.
func Flush(cl *Client) error {
	ch := make(chan error)
	err := FlushAsync(cl, ch)
	if err != nil {
		return err
	}
	return <-ch
}

// FlushAsync instructs a buffered client to flush to the upstream
// veneur all the spans that were serialized up until the moment that
// the flush was received. Once the client has completed the flush,
// any error (or nil) is sent down the error channel.
//
// FlushAsync returns ErrNoClient if client is nil and ErrWouldBlock
// if the client is not able to take more requests.
func FlushAsync(cl *Client, ch chan<- error) error {
	if cl == nil {
		return ErrNoClient
	}
	op := func(ctx context.Context, s ClientBackend) {
		err := s.FlushSync(ctx)
		if ch != nil {
			ch <- err
		}
	}
	select {
	case cl.ops <- op:
		return nil
	default:
	}
	atomic.AddInt64(&cl.failedFlushes, 1)
	return ErrWouldBlock
}
<|endoftext|>


<commit_before>package internal

import (
	"context"
	"fmt"
	"reflect"
	"runtime"
	"sync"
	"time"

	"github.com/onsi/gomega/format"
	"github.com/onsi/gomega/types"
)

var errInterface = reflect.TypeOf((*error)(nil)).Elem()
var gomegaType = reflect.TypeOf((*types.Gomega)(nil)).Elem()
var contextType = reflect.TypeOf(new(context.Context)).Elem()

type contextWithAttachProgressReporter interface {
	AttachProgressReporter(func() string) func()
}

type AsyncAssertionType uint

const (
	AsyncAssertionTypeEventually AsyncAssertionType = iota
	AsyncAssertionTypeConsistently
)

func (at AsyncAssertionType) String() string {
	switch at {
	case AsyncAssertionTypeEventually:
		return "Eventually"
	case AsyncAssertionTypeConsistently:
		return "Consistently"
	}
	return "INVALID ASYNC ASSERTION TYPE"
}

type AsyncAssertion struct {
	asyncType AsyncAssertionType

	actualIsFunc  bool
	actual        interface{}
	argsToForward []interface{}

	timeoutInterval time.Duration
	pollingInterval time.Duration
	ctx             context.Context
	offset          int
	g               *Gomega
}

func NewAsyncAssertion(asyncType AsyncAssertionType, actualInput interface{}, g *Gomega, timeoutInterval time.Duration, pollingInterval time.Duration, ctx context.Context, offset int) *AsyncAssertion {
	out := &AsyncAssertion{
		asyncType:       asyncType,
		timeoutInterval: timeoutInterval,
		pollingInterval: pollingInterval,
		offset:          offset,
		ctx:             ctx,
		g:               g,
	}

	out.actual = actualInput
	if actualInput != nil && reflect.TypeOf(actualInput).Kind() == reflect.Func {
		out.actualIsFunc = true
	}

	return out
}

func (assertion *AsyncAssertion) WithOffset(offset int) types.AsyncAssertion {
	assertion.offset = offset
	return assertion
}

func (assertion *AsyncAssertion) WithTimeout(interval time.Duration) types.AsyncAssertion {
	assertion.timeoutInterval = interval
	return assertion
}

func (assertion *AsyncAssertion) WithPolling(interval time.Duration) types.AsyncAssertion {
	assertion.pollingInterval = interval
	return assertion
}

func (assertion *AsyncAssertion) Within(timeout time.Duration) types.AsyncAssertion {
	assertion.timeoutInterval = timeout
	return assertion
}

func (assertion *AsyncAssertion) ProbeEvery(interval time.Duration) types.AsyncAssertion {
	assertion.pollingInterval = interval
	return assertion
}

func (assertion *AsyncAssertion) WithContext(ctx context.Context) types.AsyncAssertion {
	assertion.ctx = ctx
	return assertion
}

func (assertion *AsyncAssertion) WithArguments(argsToForward ...interface{}) types.AsyncAssertion {
	assertion.argsToForward = argsToForward
	return assertion
}

func (assertion *AsyncAssertion) Should(matcher types.GomegaMatcher, optionalDescription ...interface{}) bool {
	assertion.g.THelper()
	vetOptionalDescription("Asynchronous assertion", optionalDescription...)
	return assertion.match(matcher, true, optionalDescription...)
}

func (assertion *AsyncAssertion) ShouldNot(matcher types.GomegaMatcher, optionalDescription ...interface{}) bool {
	assertion.g.THelper()
	vetOptionalDescription("Asynchronous assertion", optionalDescription...)
	return assertion.match(matcher, false, optionalDescription...)
}

func (assertion *AsyncAssertion) buildDescription(optionalDescription ...interface{}) string {
	switch len(optionalDescription) {
	case 0:
		return ""
	case 1:
		if describe, ok := optionalDescription[0].(func() string); ok {
			return describe() + "\n"
		}
	}
	return fmt.Sprintf(optionalDescription[0].(string), optionalDescription[1:]...) + "\n"
}

func (assertion *AsyncAssertion) processReturnValues(values []reflect.Value) (interface{}, error) {
	if len(values) == 0 {
		return nil, fmt.Errorf("No values were returned by the function passed to Gomega")
	}

	actual := values[0].Interface()
	if _, ok := AsAsyncSignalError(actual); ok {
		return actual, actual.(error)
	}

	var err error
	for i, extraValue := range values[1:] {
		extra := extraValue.Interface()
		if extra == nil {
			continue
		}
		if _, ok := AsAsyncSignalError(extra); ok {
			return actual, extra.(error)
		}
		extraType := reflect.TypeOf(extra)
		zero := reflect.Zero(extraType).Interface()
		if reflect.DeepEqual(extra, zero) {
			continue
		}
		if i == len(values)-2 && extraType.Implements(errInterface) {
			err = fmt.Errorf("function returned error: %w", extra.(error))
		}
		if err == nil {
			err = fmt.Errorf("Unexpected non-nil/non-zero return value at index %d:\n\t<%T>: %#v", i+1, extra, extra)
		}
	}

	return actual, err
}

func (assertion *AsyncAssertion) invalidFunctionError(t reflect.Type) error {
	return fmt.Errorf(`The function passed to %s had an invalid signature of %s.  Functions passed to %s must either:

	(a) have return values or
	(b) take a Gomega interface as their first argument and use that Gomega instance to make assertions.

You can learn more at https://onsi.github.io/gomega/#eventually
`, assertion.asyncType, t, assertion.asyncType)
}

func (assertion *AsyncAssertion) noConfiguredContextForFunctionError() error {
	return fmt.Errorf(`The function passed to %s requested a context.Context, but no context has been provided.  Please pass one in using %s().WithContext().

You can learn more at https://onsi.github.io/gomega/#eventually
`, assertion.asyncType, assertion.asyncType)
}

func (assertion *AsyncAssertion) argumentMismatchError(t reflect.Type, numProvided int) error {
	have := "have"
	if numProvided == 1 {
		have = "has"
	}
	return fmt.Errorf(`The function passed to %s has signature %s takes %d arguments but %d %s been provided.  Please use %s().WithArguments() to pass the corect set of arguments.

You can learn more at https://onsi.github.io/gomega/#eventually
`, assertion.asyncType, t, t.NumIn(), numProvided, have, assertion.asyncType)
}

func (assertion *AsyncAssertion) buildActualPoller() (func() (interface{}, error), error) {
	if !assertion.actualIsFunc {
		return func() (interface{}, error) { return assertion.actual, nil }, nil
	}
	actualValue := reflect.ValueOf(assertion.actual)
	actualType := reflect.TypeOf(assertion.actual)
	numIn, numOut, isVariadic := actualType.NumIn(), actualType.NumOut(), actualType.IsVariadic()

	if numIn == 0 && numOut == 0 {
		return nil, assertion.invalidFunctionError(actualType)
	}
	takesGomega, takesContext := false, false
	if numIn > 0 {
		takesGomega, takesContext = actualType.In(0).Implements(gomegaType), actualType.In(0).Implements(contextType)
	}
	if takesGomega && numIn > 1 && actualType.In(1).Implements(contextType) {
		takesContext = true
	}
	if takesContext && len(assertion.argsToForward) > 0 && reflect.TypeOf(assertion.argsToForward[0]).Implements(contextType) {
		takesContext = false
	}
	if !takesGomega && numOut == 0 {
		return nil, assertion.invalidFunctionError(actualType)
	}
	if takesContext && assertion.ctx == nil {
		return nil, assertion.noConfiguredContextForFunctionError()
	}

	var assertionFailure error
	inValues := []reflect.Value{}
	if takesGomega {
		inValues = append(inValues, reflect.ValueOf(NewGomega(assertion.g.DurationBundle).ConfigureWithFailHandler(func(message string, callerSkip ...int) {
			skip := 0
			if len(callerSkip) > 0 {
				skip = callerSkip[0]
			}
			_, file, line, _ := runtime.Caller(skip + 1)
			assertionFailure = fmt.Errorf("Assertion in callback at %s:%d failed:\n%s", file, line, message)
			panic("stop execution")
		})))
	}
	if takesContext {
		inValues = append(inValues, reflect.ValueOf(assertion.ctx))
	}
	for _, arg := range assertion.argsToForward {
		inValues = append(inValues, reflect.ValueOf(arg))
	}

	if !isVariadic && numIn != len(inValues) {
		return nil, assertion.argumentMismatchError(actualType, len(inValues))
	} else if isVariadic && len(inValues) < numIn-1 {
		return nil, assertion.argumentMismatchError(actualType, len(inValues))
	}

	return func() (actual interface{}, err error) {
		var values []reflect.Value
		assertionFailure = nil
		defer func() {
			if numOut == 0 && takesGomega {
				actual = assertionFailure
			} else {
				actual, err = assertion.processReturnValues(values)
				_, isAsyncError := AsAsyncSignalError(err)
				if assertionFailure != nil && !isAsyncError {
					err = assertionFailure
				}
			}
			if e := recover(); e != nil {
				if _, isAsyncError := AsAsyncSignalError(e); isAsyncError {
					err = e.(error)
				} else if assertionFailure == nil {
					panic(e)
				}
			}
		}()
		values = actualValue.Call(inValues)
		return
	}, nil
}

func (assertion *AsyncAssertion) afterTimeout() <-chan time.Time {
	if assertion.timeoutInterval >= 0 {
		return time.After(assertion.timeoutInterval)
	}

	if assertion.asyncType == AsyncAssertionTypeConsistently {
		return time.After(assertion.g.DurationBundle.ConsistentlyDuration)
	} else {
		if assertion.ctx == nil {
			return time.After(assertion.g.DurationBundle.EventuallyTimeout)
		} else {
			return nil
		}
	}
}

func (assertion *AsyncAssertion) afterPolling() <-chan time.Time {
	if assertion.pollingInterval >= 0 {
		return time.After(assertion.pollingInterval)
	}
	if assertion.asyncType == AsyncAssertionTypeConsistently {
		return time.After(assertion.g.DurationBundle.ConsistentlyPollingInterval)
	} else {
		return time.After(assertion.g.DurationBundle.EventuallyPollingInterval)
	}
}

func (assertion *AsyncAssertion) matcherSaysStopTrying(matcher types.GomegaMatcher, value interface{}) bool {
	if assertion.actualIsFunc || types.MatchMayChangeInTheFuture(matcher, value) {
		return false
	}
	return true
}

func (assertion *AsyncAssertion) pollMatcher(matcher types.GomegaMatcher, value interface{}) (matches bool, err error) {
	defer func() {
		if e := recover(); e != nil {
			if _, isAsyncError := AsAsyncSignalError(e); isAsyncError {
				err = e.(error)
			} else {
				panic(e)
			}
		}
	}()

	matches, err = matcher.Match(value)

	return
}

func (assertion *AsyncAssertion) match(matcher types.GomegaMatcher, desiredMatch bool, optionalDescription ...interface{}) bool {
	timer := time.Now()
	timeout := assertion.afterTimeout()
	lock := sync.Mutex{}

	var matches bool
	var err error
	var oracleMatcherSaysStop bool

	assertion.g.THelper()

	pollActual, err := assertion.buildActualPoller()
	if err != nil {
		assertion.g.Fail(err.Error(), 2+assertion.offset)
		return false
	}

	value, err := pollActual()
	if err == nil {
		oracleMatcherSaysStop = assertion.matcherSaysStopTrying(matcher, value)
		matches, err = assertion.pollMatcher(matcher, value)
	}

	messageGenerator := func() string {
		// can be called out of band by Ginkgo if the user requests a progress report
		lock.Lock()
		defer lock.Unlock()
		message := ""
		if err != nil {
			//TODO - formatting for TryAgainAfter?
			if asyncSignal, ok := AsAsyncSignalError(err); ok && asyncSignal.IsStopTrying() {
				message = err.Error()
				for _, attachment := range asyncSignal.Attachments {
					message += fmt.Sprintf("\n%s:\n", attachment.Description)
					message += format.Object(attachment.Object, 1)
				}
			} else {
				message = "Error: " + err.Error()
			}
		} else {
			if desiredMatch {
				message = matcher.FailureMessage(value)
			} else {
				message = matcher.NegatedFailureMessage(value)
			}
		}
		description := assertion.buildDescription(optionalDescription...)
		return fmt.Sprintf("%s%s", description, message)
	}

	fail := func(preamble string) {
		assertion.g.THelper()
		assertion.g.Fail(fmt.Sprintf("%s after %.3fs.\n%s", preamble, time.Since(timer).Seconds(), messageGenerator()), 3+assertion.offset)
	}

	var contextDone <-chan struct{}
	if assertion.ctx != nil {
		contextDone = assertion.ctx.Done()
		if v, ok := assertion.ctx.Value("GINKGO_SPEC_CONTEXT").(contextWithAttachProgressReporter); ok {
			detach := v.AttachProgressReporter(messageGenerator)
			defer detach()
		}
	}

	for {
		var nextPoll <-chan time.Time = nil
		var isTryAgainAfterError = false

		if asyncSignal, ok := AsAsyncSignalError(err); ok {
			if asyncSignal.IsStopTrying() {
				fail("Told to stop trying")
				return false
			}
			if asyncSignal.IsTryAgainAfter() {
				nextPoll = time.After(asyncSignal.TryAgainDuration())
				isTryAgainAfterError = true
			}
		}

		if err == nil && matches == desiredMatch {
			if assertion.asyncType == AsyncAssertionTypeEventually {
				return true
			}
		} else if !isTryAgainAfterError {
			if assertion.asyncType == AsyncAssertionTypeConsistently {
				fail("Failed")
				return false
			}
		}

		if oracleMatcherSaysStop {
			if assertion.asyncType == AsyncAssertionTypeEventually {
				fail("No future change is possible.  Bailing out early")
				return false
			} else {
				return true
			}
		}

		if nextPoll == nil {
			nextPoll = assertion.afterPolling()
		}

		select {
		case <-nextPoll:
			v, e := pollActual()
			lock.Lock()
			value, err = v, e
			lock.Unlock()
			if err == nil {
				oracleMatcherSaysStop = assertion.matcherSaysStopTrying(matcher, value)
				m, e := assertion.pollMatcher(matcher, value)
				lock.Lock()
				matches, err = m, e
				lock.Unlock()
			}
		case <-contextDone:
			fail("Context was cancelled")
			return false
		case <-timeout:
			if assertion.asyncType == AsyncAssertionTypeEventually {
				fail("Timed out")
				return false
			} else {
				if isTryAgainAfterError {
					fail("Timed out while waiting on TryAgainAfter")
					return false
				}
				return true
			}
		}
	}

	return false
}
<commit_msg>fix go vet<commit_after>package internal

import (
	"context"
	"fmt"
	"reflect"
	"runtime"
	"sync"
	"time"

	"github.com/onsi/gomega/format"
	"github.com/onsi/gomega/types"
)

var errInterface = reflect.TypeOf((*error)(nil)).Elem()
var gomegaType = reflect.TypeOf((*types.Gomega)(nil)).Elem()
var contextType = reflect.TypeOf(new(context.Context)).Elem()

type contextWithAttachProgressReporter interface {
	AttachProgressReporter(func() string) func()
}

type AsyncAssertionType uint

const (
	AsyncAssertionTypeEventually AsyncAssertionType = iota
	AsyncAssertionTypeConsistently
)

func (at AsyncAssertionType) String() string {
	switch at {
	case AsyncAssertionTypeEventually:
		return "Eventually"
	case AsyncAssertionTypeConsistently:
		return "Consistently"
	}
	return "INVALID ASYNC ASSERTION TYPE"
}

type AsyncAssertion struct {
	asyncType AsyncAssertionType

	actualIsFunc  bool
	actual        interface{}
	argsToForward []interface{}

	timeoutInterval time.Duration
	pollingInterval time.Duration
	ctx             context.Context
	offset          int
	g               *Gomega
}

func NewAsyncAssertion(asyncType AsyncAssertionType, actualInput interface{}, g *Gomega, timeoutInterval time.Duration, pollingInterval time.Duration, ctx context.Context, offset int) *AsyncAssertion {
	out := &AsyncAssertion{
		asyncType:       asyncType,
		timeoutInterval: timeoutInterval,
		pollingInterval: pollingInterval,
		offset:          offset,
		ctx:             ctx,
		g:               g,
	}

	out.actual = actualInput
	if actualInput != nil && reflect.TypeOf(actualInput).Kind() == reflect.Func {
		out.actualIsFunc = true
	}

	return out
}

func (assertion *AsyncAssertion) WithOffset(offset int) types.AsyncAssertion {
	assertion.offset = offset
	return assertion
}

func (assertion *AsyncAssertion) WithTimeout(interval time.Duration) types.AsyncAssertion {
	assertion.timeoutInterval = interval
	return assertion
}

func (assertion *AsyncAssertion) WithPolling(interval time.Duration) types.AsyncAssertion {
	assertion.pollingInterval = interval
	return assertion
}

func (assertion *AsyncAssertion) Within(timeout time.Duration) types.AsyncAssertion {
	assertion.timeoutInterval = timeout
	return assertion
}

func (assertion *AsyncAssertion) ProbeEvery(interval time.Duration) types.AsyncAssertion {
	assertion.pollingInterval = interval
	return assertion
}

func (assertion *AsyncAssertion) WithContext(ctx context.Context) types.AsyncAssertion {
	assertion.ctx = ctx
	return assertion
}

func (assertion *AsyncAssertion) WithArguments(argsToForward ...interface{}) types.AsyncAssertion {
	assertion.argsToForward = argsToForward
	return assertion
}

func (assertion *AsyncAssertion) Should(matcher types.GomegaMatcher, optionalDescription ...interface{}) bool {
	assertion.g.THelper()
	vetOptionalDescription("Asynchronous assertion", optionalDescription...)
	return assertion.match(matcher, true, optionalDescription...)
}

func (assertion *AsyncAssertion) ShouldNot(matcher types.GomegaMatcher, optionalDescription ...interface{}) bool {
	assertion.g.THelper()
	vetOptionalDescription("Asynchronous assertion", optionalDescription...)
	return assertion.match(matcher, false, optionalDescription...)
}

func (assertion *AsyncAssertion) buildDescription(optionalDescription ...interface{}) string {
	switch len(optionalDescription) {
	case 0:
		return ""
	case 1:
		if describe, ok := optionalDescription[0].(func() string); ok {
			return describe() + "\n"
		}
	}
	return fmt.Sprintf(optionalDescription[0].(string), optionalDescription[1:]...) + "\n"
}

func (assertion *AsyncAssertion) processReturnValues(values []reflect.Value) (interface{}, error) {
	if len(values) == 0 {
		return nil, fmt.Errorf("No values were returned by the function passed to Gomega")
	}

	actual := values[0].Interface()
	if _, ok := AsAsyncSignalError(actual); ok {
		return actual, actual.(error)
	}

	var err error
	for i, extraValue := range values[1:] {
		extra := extraValue.Interface()
		if extra == nil {
			continue
		}
		if _, ok := AsAsyncSignalError(extra); ok {
			return actual, extra.(error)
		}
		extraType := reflect.TypeOf(extra)
		zero := reflect.Zero(extraType).Interface()
		if reflect.DeepEqual(extra, zero) {
			continue
		}
		if i == len(values)-2 && extraType.Implements(errInterface) {
			err = fmt.Errorf("function returned error: %w", extra.(error))
		}
		if err == nil {
			err = fmt.Errorf("Unexpected non-nil/non-zero return value at index %d:\n\t<%T>: %#v", i+1, extra, extra)
		}
	}

	return actual, err
}

func (assertion *AsyncAssertion) invalidFunctionError(t reflect.Type) error {
	return fmt.Errorf(`The function passed to %s had an invalid signature of %s.  Functions passed to %s must either:

	(a) have return values or
	(b) take a Gomega interface as their first argument and use that Gomega instance to make assertions.

You can learn more at https://onsi.github.io/gomega/#eventually
`, assertion.asyncType, t, assertion.asyncType)
}

func (assertion *AsyncAssertion) noConfiguredContextForFunctionError() error {
	return fmt.Errorf(`The function passed to %s requested a context.Context, but no context has been provided.  Please pass one in using %s().WithContext().

You can learn more at https://onsi.github.io/gomega/#eventually
`, assertion.asyncType, assertion.asyncType)
}

func (assertion *AsyncAssertion) argumentMismatchError(t reflect.Type, numProvided int) error {
	have := "have"
	if numProvided == 1 {
		have = "has"
	}
	return fmt.Errorf(`The function passed to %s has signature %s takes %d arguments but %d %s been provided.  Please use %s().WithArguments() to pass the corect set of arguments.

You can learn more at https://onsi.github.io/gomega/#eventually
`, assertion.asyncType, t, t.NumIn(), numProvided, have, assertion.asyncType)
}

func (assertion *AsyncAssertion) buildActualPoller() (func() (interface{}, error), error) {
	if !assertion.actualIsFunc {
		return func() (interface{}, error) { return assertion.actual, nil }, nil
	}
	actualValue := reflect.ValueOf(assertion.actual)
	actualType := reflect.TypeOf(assertion.actual)
	numIn, numOut, isVariadic := actualType.NumIn(), actualType.NumOut(), actualType.IsVariadic()

	if numIn == 0 && numOut == 0 {
		return nil, assertion.invalidFunctionError(actualType)
	}
	takesGomega, takesContext := false, false
	if numIn > 0 {
		takesGomega, takesContext = actualType.In(0).Implements(gomegaType), actualType.In(0).Implements(contextType)
	}
	if takesGomega && numIn > 1 && actualType.In(1).Implements(contextType) {
		takesContext = true
	}
	if takesContext && len(assertion.argsToForward) > 0 && reflect.TypeOf(assertion.argsToForward[0]).Implements(contextType) {
		takesContext = false
	}
	if !takesGomega && numOut == 0 {
		return nil, assertion.invalidFunctionError(actualType)
	}
	if takesContext && assertion.ctx == nil {
		return nil, assertion.noConfiguredContextForFunctionError()
	}

	var assertionFailure error
	inValues := []reflect.Value{}
	if takesGomega {
		inValues = append(inValues, reflect.ValueOf(NewGomega(assertion.g.DurationBundle).ConfigureWithFailHandler(func(message string, callerSkip ...int) {
			skip := 0
			if len(callerSkip) > 0 {
				skip = callerSkip[0]
			}
			_, file, line, _ := runtime.Caller(skip + 1)
			assertionFailure = fmt.Errorf("Assertion in callback at %s:%d failed:\n%s", file, line, message)
			panic("stop execution")
		})))
	}
	if takesContext {
		inValues = append(inValues, reflect.ValueOf(assertion.ctx))
	}
	for _, arg := range assertion.argsToForward {
		inValues = append(inValues, reflect.ValueOf(arg))
	}

	if !isVariadic && numIn != len(inValues) {
		return nil, assertion.argumentMismatchError(actualType, len(inValues))
	} else if isVariadic && len(inValues) < numIn-1 {
		return nil, assertion.argumentMismatchError(actualType, len(inValues))
	}

	return func() (actual interface{}, err error) {
		var values []reflect.Value
		assertionFailure = nil
		defer func() {
			if numOut == 0 && takesGomega {
				actual = assertionFailure
			} else {
				actual, err = assertion.processReturnValues(values)
				_, isAsyncError := AsAsyncSignalError(err)
				if assertionFailure != nil && !isAsyncError {
					err = assertionFailure
				}
			}
			if e := recover(); e != nil {
				if _, isAsyncError := AsAsyncSignalError(e); isAsyncError {
					err = e.(error)
				} else if assertionFailure == nil {
					panic(e)
				}
			}
		}()
		values = actualValue.Call(inValues)
		return
	}, nil
}

func (assertion *AsyncAssertion) afterTimeout() <-chan time.Time {
	if assertion.timeoutInterval >= 0 {
		return time.After(assertion.timeoutInterval)
	}

	if assertion.asyncType == AsyncAssertionTypeConsistently {
		return time.After(assertion.g.DurationBundle.ConsistentlyDuration)
	} else {
		if assertion.ctx == nil {
			return time.After(assertion.g.DurationBundle.EventuallyTimeout)
		} else {
			return nil
		}
	}
}

func (assertion *AsyncAssertion) afterPolling() <-chan time.Time {
	if assertion.pollingInterval >= 0 {
		return time.After(assertion.pollingInterval)
	}
	if assertion.asyncType == AsyncAssertionTypeConsistently {
		return time.After(assertion.g.DurationBundle.ConsistentlyPollingInterval)
	} else {
		return time.After(assertion.g.DurationBundle.EventuallyPollingInterval)
	}
}

func (assertion *AsyncAssertion) matcherSaysStopTrying(matcher types.GomegaMatcher, value interface{}) bool {
	if assertion.actualIsFunc || types.MatchMayChangeInTheFuture(matcher, value) {
		return false
	}
	return true
}

func (assertion *AsyncAssertion) pollMatcher(matcher types.GomegaMatcher, value interface{}) (matches bool, err error) {
	defer func() {
		if e := recover(); e != nil {
			if _, isAsyncError := AsAsyncSignalError(e); isAsyncError {
				err = e.(error)
			} else {
				panic(e)
			}
		}
	}()

	matches, err = matcher.Match(value)

	return
}

func (assertion *AsyncAssertion) match(matcher types.GomegaMatcher, desiredMatch bool, optionalDescription ...interface{}) bool {
	timer := time.Now()
	timeout := assertion.afterTimeout()
	lock := sync.Mutex{}

	var matches bool
	var err error
	var oracleMatcherSaysStop bool

	assertion.g.THelper()

	pollActual, err := assertion.buildActualPoller()
	if err != nil {
		assertion.g.Fail(err.Error(), 2+assertion.offset)
		return false
	}

	value, err := pollActual()
	if err == nil {
		oracleMatcherSaysStop = assertion.matcherSaysStopTrying(matcher, value)
		matches, err = assertion.pollMatcher(matcher, value)
	}

	messageGenerator := func() string {
		// can be called out of band by Ginkgo if the user requests a progress report
		lock.Lock()
		defer lock.Unlock()
		message := ""
		if err != nil {
			//TODO - formatting for TryAgainAfter?
			if asyncSignal, ok := AsAsyncSignalError(err); ok && asyncSignal.IsStopTrying() {
				message = err.Error()
				for _, attachment := range asyncSignal.Attachments {
					message += fmt.Sprintf("\n%s:\n", attachment.Description)
					message += format.Object(attachment.Object, 1)
				}
			} else {
				message = "Error: " + err.Error()
			}
		} else {
			if desiredMatch {
				message = matcher.FailureMessage(value)
			} else {
				message = matcher.NegatedFailureMessage(value)
			}
		}
		description := assertion.buildDescription(optionalDescription...)
		return fmt.Sprintf("%s%s", description, message)
	}

	fail := func(preamble string) {
		assertion.g.THelper()
		assertion.g.Fail(fmt.Sprintf("%s after %.3fs.\n%s", preamble, time.Since(timer).Seconds(), messageGenerator()), 3+assertion.offset)
	}

	var contextDone <-chan struct{}
	if assertion.ctx != nil {
		contextDone = assertion.ctx.Done()
		if v, ok := assertion.ctx.Value("GINKGO_SPEC_CONTEXT").(contextWithAttachProgressReporter); ok {
			detach := v.AttachProgressReporter(messageGenerator)
			defer detach()
		}
	}

	for {
		var nextPoll <-chan time.Time = nil
		var isTryAgainAfterError = false

		if asyncSignal, ok := AsAsyncSignalError(err); ok {
			if asyncSignal.IsStopTrying() {
				fail("Told to stop trying")
				return false
			}
			if asyncSignal.IsTryAgainAfter() {
				nextPoll = time.After(asyncSignal.TryAgainDuration())
				isTryAgainAfterError = true
			}
		}

		if err == nil && matches == desiredMatch {
			if assertion.asyncType == AsyncAssertionTypeEventually {
				return true
			}
		} else if !isTryAgainAfterError {
			if assertion.asyncType == AsyncAssertionTypeConsistently {
				fail("Failed")
				return false
			}
		}

		if oracleMatcherSaysStop {
			if assertion.asyncType == AsyncAssertionTypeEventually {
				fail("No future change is possible.  Bailing out early")
				return false
			} else {
				return true
			}
		}

		if nextPoll == nil {
			nextPoll = assertion.afterPolling()
		}

		select {
		case <-nextPoll:
			v, e := pollActual()
			lock.Lock()
			value, err = v, e
			lock.Unlock()
			if err == nil {
				oracleMatcherSaysStop = assertion.matcherSaysStopTrying(matcher, value)
				m, e := assertion.pollMatcher(matcher, value)
				lock.Lock()
				matches, err = m, e
				lock.Unlock()
			}
		case <-contextDone:
			fail("Context was cancelled")
			return false
		case <-timeout:
			if assertion.asyncType == AsyncAssertionTypeEventually {
				fail("Timed out")
				return false
			} else {
				if isTryAgainAfterError {
					fail("Timed out while waiting on TryAgainAfter")
					return false
				}
				return true
			}
		}
	}
}
<|endoftext|>


<commit_before>package fs

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
)

// fixpath returns an absolute path on windows, so restic can open long file
// names.
func fixpath(name string) string {
	abspath, err := filepath.Abs(name)
	if err == nil {
		// Check if \\?\UNC\ already exist
		if strings.HasPrefix(abspath, `\\?\UNC\`) {
			return abspath
		}
		// Check if \\?\ already exist
		if strings.HasPrefix(abspath, `\\?\`) {
			return abspath
		}
		// Check if path starts with \\
		if strings.HasPrefix(abspath, `\\`) {
			return strings.Replace(abspath, `\\`, `\\?\UNC\`, 1)
		}
		// Normal path
		return `\\?\` + abspath
	}
	return name
}

// TempFile creates a temporary file.
func TempFile(dir, prefix string) (f *os.File, err error) {
	return ioutil.TempFile(dir, prefix)
}

// Chmod changes the mode of the named file to mode.
func Chmod(name string, mode os.FileMode) error {
	return os.Chmod(fixpath(name), mode)
}
<commit_msg>Better temp file cleanup on Windows.<commit_after>package fs

import (
	"os"
	"path/filepath"
	"strings"
	"syscall"
	"strconv"
	"sync"
	"time"
)

// Random number state.
// We generate random temporary file names so that there's a good
// chance the file doesn't exist yet - keeps the number of tries in
// TempFile to a minimum.
var rand uint32
var randmu sync.Mutex

func reseed() uint32 {
	return uint32(time.Now().UnixNano() + int64(os.Getpid()))
}

func nextRandom() string {
	randmu.Lock()
	r := rand
	if r == 0 {
		r = reseed()
	}
	r = r*1664525 + 1013904223 // constants from Numerical Recipes
	rand = r
	randmu.Unlock()
	return strconv.Itoa(int(1e9 + r%1e9))[1:]
}

const (
	FILE_ATTRIBUTE_TEMPORARY  = 0x00000100
	FILE_FLAG_DELETE_ON_CLOSE = 0x04000000
)

// fixpath returns an absolute path on windows, so restic can open long file
// names.
func fixpath(name string) string {
	abspath, err := filepath.Abs(name)
	if err == nil {
		// Check if \\?\UNC\ already exist
		if strings.HasPrefix(abspath, `\\?\UNC\`) {
			return abspath
		}
		// Check if \\?\ already exist
		if strings.HasPrefix(abspath, `\\?\`) {
			return abspath
		}
		// Check if path starts with \\
		if strings.HasPrefix(abspath, `\\`) {
			return strings.Replace(abspath, `\\`, `\\?\UNC\`, 1)
		}
		// Normal path
		return `\\?\` + abspath
	}
	return name
}

// TempFile creates a temporary file.
func TempFile(dir, prefix string) (f *os.File, err error) {
	if dir == "" {
		dir = os.TempDir()
	}

	access := uint32(syscall.GENERIC_READ | syscall.GENERIC_WRITE)
	creation := uint32(syscall.CREATE_NEW)
	flags := uint32(FILE_ATTRIBUTE_TEMPORARY | FILE_FLAG_DELETE_ON_CLOSE)

	for i := 0; i < 10000; i++ {
		path := filepath.Join(dir, prefix+nextRandom())

		h, err := syscall.CreateFile(syscall.StringToUTF16Ptr(path), access, 0, nil, creation, flags, 0)
		if err == nil {
			return os.NewFile(uintptr(h), path), nil
		}
	}

	// Proper error handling is still to do
	return nil, os.ErrExist
}

// Chmod changes the mode of the named file to mode.
func Chmod(name string, mode os.FileMode) error {
	return os.Chmod(fixpath(name), mode)
}
<|endoftext|>


<commit_before>package restic

import (
	"context"
	"fmt"
	"os/user"
	"path/filepath"
	"sync"
	"time"

	"golang.org/x/sync/errgroup"

	"github.com/restic/restic/internal/debug"
)

// Snapshot is the state of a resource at one point in time.
type Snapshot struct {
	Time     time.Time `json:"time"`
	Parent   *ID       `json:"parent,omitempty"`
	Tree     *ID       `json:"tree"`
	Paths    []string  `json:"paths"`
	Hostname string    `json:"hostname,omitempty"`
	Username string    `json:"username,omitempty"`
	UID      uint32    `json:"uid,omitempty"`
	GID      uint32    `json:"gid,omitempty"`
	Excludes []string  `json:"excludes,omitempty"`
	Tags     []string  `json:"tags,omitempty"`
	Original *ID       `json:"original,omitempty"`

	id *ID // plaintext ID, used during restore
}

// NewSnapshot returns an initialized snapshot struct for the current user and
// time.
func NewSnapshot(paths []string, tags []string, hostname string, time time.Time) (*Snapshot, error) {
	absPaths := make([]string, 0, len(paths))
	for _, path := range paths {
		p, err := filepath.Abs(path)
		if err == nil {
			absPaths = append(absPaths, p)
		} else {
			absPaths = append(absPaths, path)
		}
	}

	sn := &Snapshot{
		Paths:    absPaths,
		Time:     time,
		Tags:     tags,
		Hostname: hostname,
	}

	err := sn.fillUserInfo()
	if err != nil {
		return nil, err
	}

	return sn, nil
}

// LoadSnapshot loads the snapshot with the id and returns it.
func LoadSnapshot(ctx context.Context, loader LoaderUnpacked, id ID) (*Snapshot, error) {
	sn := &Snapshot{id: &id}
	err := LoadJSONUnpacked(ctx, loader, SnapshotFile, id, sn)
	if err != nil {
		return nil, err
	}

	return sn, nil
}

// SaveSnapshot saves the snapshot sn and returns its ID.
func SaveSnapshot(ctx context.Context, repo SaverUnpacked, sn *Snapshot) (ID, error) {
	return SaveJSONUnpacked(ctx, repo, SnapshotFile, sn)
}

// ForAllSnapshots reads all snapshots in parallel and calls the
// given function. It is guaranteed that the function is not run concurrently.
// If the called function returns an error, this function is cancelled and
// also returns this error.
// If a snapshot ID is in excludeIDs, it will be ignored.
func ForAllSnapshots(ctx context.Context, be Lister, loader LoaderUnpacked, excludeIDs IDSet, fn func(ID, *Snapshot, error) error) error {
	var m sync.Mutex

	// track spawned goroutines using wg, create a new context which is
	// cancelled as soon as an error occurs.
	wg, ctx := errgroup.WithContext(ctx)

	ch := make(chan ID)

	// send list of snapshot files through ch, which is closed afterwards
	wg.Go(func() error {
		defer close(ch)
		return be.List(ctx, SnapshotFile, func(fi FileInfo) error {
			id, err := ParseID(fi.Name)
			if err != nil {
				debug.Log("unable to parse %v as an ID", fi.Name)
				return nil
			}

			if excludeIDs.Has(id) {
				return nil
			}

			select {
			case <-ctx.Done():
				return nil
			case ch <- id:
			}
			return nil
		})
	})

	// a worker receives an snapshot ID from ch, loads the snapshot
	// and runs fn with id, the snapshot and the error
	worker := func() error {
		for id := range ch {
			debug.Log("load snapshot %v", id)
			sn, err := LoadSnapshot(ctx, loader, id)

			m.Lock()
			err = fn(id, sn, err)
			m.Unlock()
			if err != nil {
				return err
			}
		}
		return nil
	}

	// For most snapshots decoding is nearly for free, thus just assume were only limited by IO
	for i := 0; i < int(loader.Connections()); i++ {
		wg.Go(worker)
	}

	return wg.Wait()
}

func (sn Snapshot) String() string {
	return fmt.Sprintf("<Snapshot %s of %v at %s by %s@%s>",
		sn.id.Str(), sn.Paths, sn.Time, sn.Username, sn.Hostname)
}

// ID returns the snapshot's ID.
func (sn Snapshot) ID() *ID {
	return sn.id
}

func (sn *Snapshot) fillUserInfo() error {
	usr, err := user.Current()
	if err != nil {
		return nil
	}
	sn.Username = usr.Username

	// set userid and groupid
	sn.UID, sn.GID, err = uidGidInt(*usr)
	return err
}

// AddTags adds the given tags to the snapshots tags, preventing duplicates.
// It returns true if any changes were made.
func (sn *Snapshot) AddTags(addTags []string) (changed bool) {
nextTag:
	for _, add := range addTags {
		for _, tag := range sn.Tags {
			if tag == add {
				continue nextTag
			}
		}
		sn.Tags = append(sn.Tags, add)
		changed = true
	}
	return
}

// RemoveTags removes the given tags from the snapshots tags and
// returns true if any changes were made.
func (sn *Snapshot) RemoveTags(removeTags []string) (changed bool) {
	for _, remove := range removeTags {
		for i, tag := range sn.Tags {
			if tag == remove {
				// https://github.com/golang/go/wiki/SliceTricks
				sn.Tags[i] = sn.Tags[len(sn.Tags)-1]
				sn.Tags[len(sn.Tags)-1] = ""
				sn.Tags = sn.Tags[:len(sn.Tags)-1]

				changed = true
				break
			}
		}
	}
	return
}

func (sn *Snapshot) hasTag(tag string) bool {
	for _, snTag := range sn.Tags {
		if tag == snTag {
			return true
		}
	}
	return false
}

// HasTags returns true if the snapshot has all the tags in l.
func (sn *Snapshot) HasTags(l []string) bool {
	for _, tag := range l {
		if tag == "" && len(sn.Tags) == 0 {
			return true
		}
		if !sn.hasTag(tag) {
			return false
		}
	}

	return true
}

// HasTagList returns true if either
//   - the snapshot satisfies at least one TagList, so there is a TagList in l
//     for which all tags are included in sn, or
//   - l is empty
func (sn *Snapshot) HasTagList(l []TagList) bool {
	debug.Log("testing snapshot with tags %v against list: %v", sn.Tags, l)

	if len(l) == 0 {
		return true
	}

	for _, tags := range l {
		if sn.HasTags(tags) {
			debug.Log("  snapshot satisfies %v %v", tags, l)
			return true
		}
	}

	return false
}

func (sn *Snapshot) hasPath(path string) bool {
	for _, snPath := range sn.Paths {
		if path == snPath {
			return true
		}
	}
	return false
}

// HasPaths returns true if the snapshot has all of the paths.
func (sn *Snapshot) HasPaths(paths []string) bool {
	for _, path := range paths {
		if !sn.hasPath(path) {
			return false
		}
	}

	return true
}

// HasHostname returns true if either
// - the snapshot hostname is in the list of the given hostnames, or
// - the list of given hostnames is empty
func (sn *Snapshot) HasHostname(hostnames []string) bool {
	if len(hostnames) == 0 {
		return true
	}

	for _, hostname := range hostnames {
		if sn.Hostname == hostname {
			return true
		}
	}

	return false
}

// Snapshots is a list of snapshots.
type Snapshots []*Snapshot

// Len returns the number of snapshots in sn.
func (sn Snapshots) Len() int {
	return len(sn)
}

// Less returns true iff the ith snapshot has been made after the jth.
func (sn Snapshots) Less(i, j int) bool {
	return sn[i].Time.After(sn[j].Time)
}

// Swap exchanges the two snapshots.
func (sn Snapshots) Swap(i, j int) {
	sn[i], sn[j] = sn[j], sn[i]
}
<commit_msg>Fix quadratic time complexity of Snapshot.HasPaths<commit_after>package restic

import (
	"context"
	"fmt"
	"os/user"
	"path/filepath"
	"sync"
	"time"

	"golang.org/x/sync/errgroup"

	"github.com/restic/restic/internal/debug"
)

// Snapshot is the state of a resource at one point in time.
type Snapshot struct {
	Time     time.Time `json:"time"`
	Parent   *ID       `json:"parent,omitempty"`
	Tree     *ID       `json:"tree"`
	Paths    []string  `json:"paths"`
	Hostname string    `json:"hostname,omitempty"`
	Username string    `json:"username,omitempty"`
	UID      uint32    `json:"uid,omitempty"`
	GID      uint32    `json:"gid,omitempty"`
	Excludes []string  `json:"excludes,omitempty"`
	Tags     []string  `json:"tags,omitempty"`
	Original *ID       `json:"original,omitempty"`

	id *ID // plaintext ID, used during restore
}

// NewSnapshot returns an initialized snapshot struct for the current user and
// time.
func NewSnapshot(paths []string, tags []string, hostname string, time time.Time) (*Snapshot, error) {
	absPaths := make([]string, 0, len(paths))
	for _, path := range paths {
		p, err := filepath.Abs(path)
		if err == nil {
			absPaths = append(absPaths, p)
		} else {
			absPaths = append(absPaths, path)
		}
	}

	sn := &Snapshot{
		Paths:    absPaths,
		Time:     time,
		Tags:     tags,
		Hostname: hostname,
	}

	err := sn.fillUserInfo()
	if err != nil {
		return nil, err
	}

	return sn, nil
}

// LoadSnapshot loads the snapshot with the id and returns it.
func LoadSnapshot(ctx context.Context, loader LoaderUnpacked, id ID) (*Snapshot, error) {
	sn := &Snapshot{id: &id}
	err := LoadJSONUnpacked(ctx, loader, SnapshotFile, id, sn)
	if err != nil {
		return nil, err
	}

	return sn, nil
}

// SaveSnapshot saves the snapshot sn and returns its ID.
func SaveSnapshot(ctx context.Context, repo SaverUnpacked, sn *Snapshot) (ID, error) {
	return SaveJSONUnpacked(ctx, repo, SnapshotFile, sn)
}

// ForAllSnapshots reads all snapshots in parallel and calls the
// given function. It is guaranteed that the function is not run concurrently.
// If the called function returns an error, this function is cancelled and
// also returns this error.
// If a snapshot ID is in excludeIDs, it will be ignored.
func ForAllSnapshots(ctx context.Context, be Lister, loader LoaderUnpacked, excludeIDs IDSet, fn func(ID, *Snapshot, error) error) error {
	var m sync.Mutex

	// track spawned goroutines using wg, create a new context which is
	// cancelled as soon as an error occurs.
	wg, ctx := errgroup.WithContext(ctx)

	ch := make(chan ID)

	// send list of snapshot files through ch, which is closed afterwards
	wg.Go(func() error {
		defer close(ch)
		return be.List(ctx, SnapshotFile, func(fi FileInfo) error {
			id, err := ParseID(fi.Name)
			if err != nil {
				debug.Log("unable to parse %v as an ID", fi.Name)
				return nil
			}

			if excludeIDs.Has(id) {
				return nil
			}

			select {
			case <-ctx.Done():
				return nil
			case ch <- id:
			}
			return nil
		})
	})

	// a worker receives an snapshot ID from ch, loads the snapshot
	// and runs fn with id, the snapshot and the error
	worker := func() error {
		for id := range ch {
			debug.Log("load snapshot %v", id)
			sn, err := LoadSnapshot(ctx, loader, id)

			m.Lock()
			err = fn(id, sn, err)
			m.Unlock()
			if err != nil {
				return err
			}
		}
		return nil
	}

	// For most snapshots decoding is nearly for free, thus just assume were only limited by IO
	for i := 0; i < int(loader.Connections()); i++ {
		wg.Go(worker)
	}

	return wg.Wait()
}

func (sn Snapshot) String() string {
	return fmt.Sprintf("<Snapshot %s of %v at %s by %s@%s>",
		sn.id.Str(), sn.Paths, sn.Time, sn.Username, sn.Hostname)
}

// ID returns the snapshot's ID.
func (sn Snapshot) ID() *ID {
	return sn.id
}

func (sn *Snapshot) fillUserInfo() error {
	usr, err := user.Current()
	if err != nil {
		return nil
	}
	sn.Username = usr.Username

	// set userid and groupid
	sn.UID, sn.GID, err = uidGidInt(*usr)
	return err
}

// AddTags adds the given tags to the snapshots tags, preventing duplicates.
// It returns true if any changes were made.
func (sn *Snapshot) AddTags(addTags []string) (changed bool) {
nextTag:
	for _, add := range addTags {
		for _, tag := range sn.Tags {
			if tag == add {
				continue nextTag
			}
		}
		sn.Tags = append(sn.Tags, add)
		changed = true
	}
	return
}

// RemoveTags removes the given tags from the snapshots tags and
// returns true if any changes were made.
func (sn *Snapshot) RemoveTags(removeTags []string) (changed bool) {
	for _, remove := range removeTags {
		for i, tag := range sn.Tags {
			if tag == remove {
				// https://github.com/golang/go/wiki/SliceTricks
				sn.Tags[i] = sn.Tags[len(sn.Tags)-1]
				sn.Tags[len(sn.Tags)-1] = ""
				sn.Tags = sn.Tags[:len(sn.Tags)-1]

				changed = true
				break
			}
		}
	}
	return
}

func (sn *Snapshot) hasTag(tag string) bool {
	for _, snTag := range sn.Tags {
		if tag == snTag {
			return true
		}
	}
	return false
}

// HasTags returns true if the snapshot has all the tags in l.
func (sn *Snapshot) HasTags(l []string) bool {
	for _, tag := range l {
		if tag == "" && len(sn.Tags) == 0 {
			return true
		}
		if !sn.hasTag(tag) {
			return false
		}
	}

	return true
}

// HasTagList returns true if either
//   - the snapshot satisfies at least one TagList, so there is a TagList in l
//     for which all tags are included in sn, or
//   - l is empty
func (sn *Snapshot) HasTagList(l []TagList) bool {
	debug.Log("testing snapshot with tags %v against list: %v", sn.Tags, l)

	if len(l) == 0 {
		return true
	}

	for _, tags := range l {
		if sn.HasTags(tags) {
			debug.Log("  snapshot satisfies %v %v", tags, l)
			return true
		}
	}

	return false
}

// HasPaths returns true if the snapshot has all of the paths.
func (sn *Snapshot) HasPaths(paths []string) bool {
	m := make(map[string]struct{}, len(sn.Paths))
	for _, snPath := range sn.Paths {
		m[snPath] = struct{}{}
	}
	for _, path := range paths {
		if _, ok := m[path]; !ok {
			return false
		}
	}

	return true
}

// HasHostname returns true if either
// - the snapshot hostname is in the list of the given hostnames, or
// - the list of given hostnames is empty
func (sn *Snapshot) HasHostname(hostnames []string) bool {
	if len(hostnames) == 0 {
		return true
	}

	for _, hostname := range hostnames {
		if sn.Hostname == hostname {
			return true
		}
	}

	return false
}

// Snapshots is a list of snapshots.
type Snapshots []*Snapshot

// Len returns the number of snapshots in sn.
func (sn Snapshots) Len() int {
	return len(sn)
}

// Less returns true iff the ith snapshot has been made after the jth.
func (sn Snapshots) Less(i, j int) bool {
	return sn[i].Time.After(sn[j].Time)
}

// Swap exchanges the two snapshots.
func (sn Snapshots) Swap(i, j int) {
	sn[i], sn[j] = sn[j], sn[i]
}
<|endoftext|>


<commit_before>////////////////////////////////////////////////////////////////////////////
// Program: fsimilar
// Purpose: find/file similar
// Authors: Tong Sun (c) 2017, All rights reserved
////////////////////////////////////////////////////////////////////////////

package main

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"math/rand"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"time"

	"github.com/go-dedup/simhash"
	"github.com/go-dedup/simhash/sho"

	"github.com/mkideal/cli"
)

////////////////////////////////////////////////////////////////////////////
// Constant and data type/structure definitions

//==========================================================================
// Main dispatcher

func fsimilar(ctx *cli.Context) error {
	ctx.JSON(ctx.RootArgv())
	ctx.JSON(ctx.Argv())
	fmt.Println()
	rootArgv = ctx.RootArgv().(*rootT)

	Opts.Distance, Opts.SizeGiven, Opts.Template, Opts.Verbose =
		rootArgv.Distance, rootArgv.SizeGiven, rootArgv.Template,
		rootArgv.Verbose.Value()

	return fSimilar(rootArgv.Filei)
}

func fSimilar(cin io.Reader) error {
	rand.Seed(time.Now().UTC().UnixNano())
	//tmpfile := fmt.Sprintf("%s.%d", file, 99999999-rand.Int31n(90000000))

	oracle := sho.NewOracle()
	sh := simhash.NewSimhash()
	r := Opts.Distance
	fAll := make(FAll)     // the all file to FCItem map
	fc := NewFCollection() // the FCollection that holds everything

	// read input line by line
	scanner := bufio.NewScanner(cin)
	for scanner.Scan() {
		file := FileT{}
		fn := scanner.Text()

		// == Gather file info
		if Opts.SizeGiven {
			_, err := fmt.Sscan(fn, &file.Size)
			abortOn("Parsing file size", err)
			il := regexp.MustCompile(`^ *\d+\s+(.*)$`).FindStringSubmatchIndex(fn)
			//fmt.Println(il)
			fn = fn[il[2]:]
		} else {
			s, err := os.Stat(fn)
			warnOn("Get file size", err)
			file.Size = int(s.Size())
		}
		p, n := filepath.Split(fn)
		file.Dir, file.Name, file.Ext = p, Basename(n), filepath.Ext(n)
		verbose(2, " n='%s', e='%s', s='%d', d='%s'",
			file.Name, file.Ext, file.Size, file.Dir)

		hash := sh.GetSimhash(sh.NewWordFeatureSet([]byte(file.Name)))
		file.Hash = hash
		fi := FCItem{Hash: hash, Index: fc.LenOf(hash)}
		fAll[fn] = fi
		fc.Add(hash, file)

		// == Build similarity knowledge
		if h, d, seen := oracle.Find(hash, r); seen == true {
			verbose(1, "=: Simhash of %x ignored for %x (%d).", hash, h, d)
			if d > 0 {
				oracle.See(hash)
			}
		} else {
			oracle.See(hash)
			verbose(2, "+: Simhash of %x added.", hash)
		}
	}

	if err := scanner.Err(); err != nil {
		return err
	}

	// process all, the sorted fAll map
	visited := make(HVisited)
	var keys []string
	for k := range fAll {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	// for each sorted item according to file path & name
	for _, k := range keys {
		// get next file item from FAll file to FCItem map
		fi := fAll[k]
		// and skip if the hash has already been visited
		if visited[fi.Hash] {
			continue
		}
		visited[fi.Hash] = true
		// also skip if no similar items at this hash
		if fc.LenOf(fi.Hash) <= 1 {
			continue
		}

		// similarity exist, start digging
		files, ok := fc.Get(fi.Hash)
		if !ok {
			abortOn("Internal error", errors.New("fc integrity checking"))
		}
		for ii, _ := range files {
			files[ii].Dist = 0
		}
		for _, nigh := range oracle.Search(fi.Hash, r) {
			visited[nigh.H] = true
			verbose(1, "## nigh found\n %v.", nigh)
			// files more
			fm, ok := fc.Get(nigh.H)
			if ok {
				for ii, _ := range fm {
					fm[ii].Dist = nigh.D
				}
				files = append(files, fm...)
			}
		}

		// One group of similar items found, output
		sort.Sort(files)
		verbose(2, "## Similar items\n %v.", files)
	}

	return nil
}
<commit_msg>- [*] working correctly. oracle.Search works for test/test1.lst when -d is 5 or 6<commit_after>////////////////////////////////////////////////////////////////////////////
// Program: fsimilar
// Purpose: find/file similar
// Authors: Tong Sun (c) 2017, All rights reserved
////////////////////////////////////////////////////////////////////////////

package main

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"math/rand"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"time"

	"github.com/go-dedup/simhash"
	"github.com/go-dedup/simhash/sho"

	"github.com/mkideal/cli"
)

////////////////////////////////////////////////////////////////////////////
// Constant and data type/structure definitions

//==========================================================================
// Main dispatcher

func fsimilar(ctx *cli.Context) error {
	ctx.JSON(ctx.RootArgv())
	ctx.JSON(ctx.Argv())
	fmt.Println()
	rootArgv = ctx.RootArgv().(*rootT)

	Opts.Distance, Opts.SizeGiven, Opts.Template, Opts.Verbose =
		rootArgv.Distance, rootArgv.SizeGiven, rootArgv.Template,
		rootArgv.Verbose.Value()

	return fSimilar(rootArgv.Filei)
}

func fSimilar(cin io.Reader) error {
	rand.Seed(time.Now().UTC().UnixNano())
	//tmpfile := fmt.Sprintf("%s.%d", file, 99999999-rand.Int31n(90000000))

	oracle := sho.NewOracle()
	sh := simhash.NewSimhash()
	r := Opts.Distance
	fAll := make(FAll)     // the all file to FCItem map
	fc := NewFCollection() // the FCollection that holds everything

	// read input line by line
	scanner := bufio.NewScanner(cin)
	for scanner.Scan() {
		file := FileT{}
		fn := scanner.Text()

		// == Gather file info
		if Opts.SizeGiven {
			_, err := fmt.Sscan(fn, &file.Size)
			abortOn("Parsing file size", err)
			il := regexp.MustCompile(`^ *\d+\s+(.*)$`).FindStringSubmatchIndex(fn)
			//fmt.Println(il)
			fn = fn[il[2]:]
		} else {
			s, err := os.Stat(fn)
			warnOn("Get file size", err)
			file.Size = int(s.Size())
		}
		p, n := filepath.Split(fn)
		file.Dir, file.Name, file.Ext = p, Basename(n), filepath.Ext(n)
		verbose(2, " n='%s', e='%s', s='%d', d='%s'",
			file.Name, file.Ext, file.Size, file.Dir)

		hash := sh.GetSimhash(sh.NewWordFeatureSet([]byte(file.Name)))
		file.Hash = hash
		fi := FCItem{Hash: hash, Index: fc.LenOf(hash)}
		fAll[fn] = fi
		fc.Add(hash, file)

		// == Build similarity knowledge
		if h, d, seen := oracle.Find(hash, r); seen == true {
			verbose(2, "=: Simhash of %x ignored for %x (%d).", hash, h, d)
		} else {
			oracle.See(hash)
			verbose(2, "+: Simhash of %x added.", hash)
		}
	}

	if err := scanner.Err(); err != nil {
		return err
	}

	// process all, the sorted fAll map
	visited := make(HVisited)
	var keys []string
	for k := range fAll {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	// for each sorted item according to file path & name
	for _, k := range keys {
		// get next file item from FAll file to FCItem map
		fi := fAll[k]
		// and skip if the hash has already been visited
		if visited[fi.Hash] {
			continue
		}
		visited[fi.Hash] = true
		// also skip if no similar items at this hash
		if fc.LenOf(fi.Hash) <= 1 {
			continue
		}

		// similarity exist, start digging
		files, ok := fc.Get(fi.Hash)
		if !ok {
			abortOn("Internal error", errors.New("fc integrity checking"))
		}
		for ii, _ := range files {
			files[ii].Dist = 0
		}
		for _, nigh := range oracle.Search(fi.Hash, r+1) {
			visited[nigh.H] = true
			verbose(2, "### Nigh found <----- \n %v.", nigh)
			// files more
			fm, ok := fc.Get(nigh.H)
			if ok {
				for ii, _ := range fm {
					fm[ii].Dist = nigh.D
				}
				files = append(files, fm...)
			}
		}

		// One group of similar items found, output
		sort.Sort(files)
		verbose(2, "## Similar items\n %v.", files)
	}

	return nil
}
<|endoftext|>