<commit_before>package engineio

import (
	"io"
	"net/http"
	"time"

	ws "github.com/gorilla/websocket"
)

func init() {
	t := &websocket{}
	RegisterTransport(t.Name(), t.HandlesUpgrades(), t.SupportsFraming(), NewWebsocketTransport)
}

type websocket struct {
	socket       Socket
	conn         *ws.Conn
	quitChan     chan struct{}
	pingInterval time.Duration
	pingTimeout  time.Duration
	lastPing     time.Time
	isClosed     bool
}

func NewWebsocketTransport(req *http.Request, pingInterval, pingTimeout time.Duration) (Transport, error) {
	ret := &websocket{
		quitChan:     make(chan struct{}, 1),
		pingInterval: pingInterval,
		pingTimeout:  pingTimeout,
		lastPing:     time.Now(),
		isClosed:     false,
	}
	return ret, nil
}

func (*websocket) Name() string {
	return "websocket"
}

func (p *websocket) SetSocket(s Socket) {
	p.socket = s
}

func (*websocket) HandlesUpgrades() bool {
	return true
}

func (*websocket) SupportsFraming() bool {
	return true
}

func (p *websocket) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	name := r.URL.Query().Get("transport")
	if name != p.Name() {
		encoder := NewBinaryPayloadEncoder()
		writer, err := encoder.NextString(NOOP)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		writer.Close()
		encoder.EncodeTo(w)
		return
	}

	conn, err := ws.Upgrade(w, r, nil, 10240, 10240)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	p.conn = conn
	defer func() {
		p.conn.Close()
		p.socket.onClose()
	}()

	go func() {
		defer func() {
			if !p.isClosed {
				p.isClosed = true
				close(p.quitChan)
			}
		}()
		for {
			t, r, err := conn.NextReader()
			if err != nil {
				return
			}

			if t == ws.TextMessage || t == ws.BinaryMessage {
				decoder, _ := NewDecoder(r)
				switch decoder.Type() {
				case PING:
					w, _ := p.NextWriter(MessageText, PONG)
					io.Copy(w, decoder)
					w.Close()
					fallthrough
				case PONG:
					p.lastPing = time.Now()
				case CLOSE:
					p.Close()
					return
				case UPGRADE:
				case NOOP:
				default:
					p.socket.onMessage(decoder)
				}
				decoder.Close()
			}
		}
	}()

	for {
		diff := time.Now().Sub(p.lastPing)
		select {
		case <-time.After(p.pingInterval - diff):
			w, _ := p.NextWriter(MessageText, PING)
			w.Close()
		case <-time.After(p.pingTimeout - diff):
			return
		case <-p.quitChan:
			return
		}
	}
}

func (p *websocket) NextWriter(msgType MessageType, packetType PacketType) (io.WriteCloser, error) {
	if p.isClosed {
		return nil, io.EOF
	}
	if packetType == CLOSE {
		ret, err := p.conn.NextWriter(ws.CloseMessage)
		if err != nil {
			return nil, err
		}
		return ret, nil
	}
	var ret io.WriteCloser
	var err error
	switch msgType {
	case MessageText:
		ret, err = p.conn.NextWriter(ws.TextMessage)
		if err != nil {
			return nil, err
		}
		ret, err = NewStringEncoder(ret, packetType)
	case MessageBinary:
		ret, err = p.conn.NextWriter(ws.BinaryMessage)
		if err != nil {
			return nil, err
		}
		ret, err = NewBinaryEncoder(ret, packetType)
	}
	if err != nil {
		return nil, err
	}
	return ret, nil
}

func (p *websocket) Close() error {
	w, err := p.NextWriter(MessageText, CLOSE)
	if err != nil {
		return err
	}
	w.Close()
	p.isClosed = true
	close(p.quitChan)
	return nil
}

func (p websocket) Upgraded() error {
	return nil
}
<commit_msg>clear race condition of websocket writer<commit_after>package engineio

import (
	"io"
	"net/http"
	"sync"
	"time"

	ws "github.com/gorilla/websocket"
)

func init() {
	t := &websocket{}
	RegisterTransport(t.Name(), t.HandlesUpgrades(), t.SupportsFraming(), NewWebsocketTransport)
}

type websocket struct {
	socket       Socket
	conn         *ws.Conn
	quitChan     chan struct{}
	pingChan     chan time.Time
	pingInterval time.Duration
	pingTimeout  time.Duration
	lastPing     time.Time
	isClosed     bool
	writeLocker  sync.Mutex
}

func NewWebsocketTransport(req *http.Request, pingInterval, pingTimeout time.Duration) (Transport, error) {
	ret := &websocket{
		quitChan:     make(chan struct{}, 1),
		pingChan:     make(chan time.Time),
		pingInterval: pingInterval,
		pingTimeout:  pingTimeout,
		lastPing:     time.Now(),
		isClosed:     false,
	}
	return ret, nil
}

func (*websocket) Name() string {
	return "websocket"
}

func (p *websocket) SetSocket(s Socket) {
	p.socket = s
}

func (*websocket) HandlesUpgrades() bool {
	return true
}

func (*websocket) SupportsFraming() bool {
	return true
}

func (p *websocket) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	name := r.URL.Query().Get("transport")
	if name != p.Name() {
		encoder := NewBinaryPayloadEncoder()
		writer, err := encoder.NextString(NOOP)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		writer.Close()
		encoder.EncodeTo(w)
		return
	}

	conn, err := ws.Upgrade(w, r, nil, 10240, 10240)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	p.conn = conn
	defer func() {
		close(p.quitChan)
		p.conn.Close()
		p.socket.onClose()
	}()

	go func(lastPing time.Time) {
		for {
			diff := time.Now().Sub(lastPing)
			select {
			case lastPing = <-p.pingChan:
			case <-time.After(p.pingInterval - diff):
				w, _ := p.NextWriter(MessageText, PING)
				w.Close()
			case <-time.After(p.pingTimeout - diff):
				return
			case <-p.quitChan:
				return
			}
		}
	}(p.lastPing)

	for {
		t, r, err := conn.NextReader()
		if err != nil {
			return
		}

		if t == ws.TextMessage || t == ws.BinaryMessage {
			decoder, _ := NewDecoder(r)
			switch decoder.Type() {
			case PING:
				w, _ := p.NextWriter(MessageText, PONG)
				io.Copy(w, decoder)
				w.Close()
				fallthrough
			case PONG:
				p.lastPing = time.Now()
				p.pingChan <- p.lastPing
			case CLOSE:
				p.Close()
				return
			case UPGRADE:
			case NOOP:
			default:
				p.socket.onMessage(decoder)
			}
			decoder.Close()
		}
	}
}

type websocketWriter struct {
	io.WriteCloser
	locker *sync.Mutex
}

func newWebsocketWriter(w io.WriteCloser, locker *sync.Mutex) websocketWriter {
	return websocketWriter{
		WriteCloser: w,
		locker:      locker,
	}
}

func (p *websocket) NextWriter(msgType MessageType, packetType PacketType) (io.WriteCloser, error) {
	if p.isClosed {
		return nil, io.EOF
	}
	if packetType == CLOSE {
		ret, err := p.conn.NextWriter(ws.CloseMessage)
		if err != nil {
			return nil, err
		}
		return newWebsocketWriter(ret, &p.writeLocker), nil
	}
	var ret io.WriteCloser
	var err error
	switch msgType {
	case MessageText:
		ret, err = p.conn.NextWriter(ws.TextMessage)
		if err != nil {
			return nil, err
		}
		ret, err = NewStringEncoder(ret, packetType)
	case MessageBinary:
		ret, err = p.conn.NextWriter(ws.BinaryMessage)
		if err != nil {
			return nil, err
		}
		ret, err = NewBinaryEncoder(ret, packetType)
	}
	if err != nil {
		return nil, err
	}
	return newWebsocketWriter(ret, &p.writeLocker), nil
}

func (p *websocket) Close() error {
	w, err := p.NextWriter(MessageText, CLOSE)
	if err != nil {
		return err
	}
	w.Close()
	p.isClosed = true
	p.conn.Close()
	return nil
}

func (p websocket) Upgraded() error {
	return nil
}
<|endoftext|>


<commit_before>// Copyright (C) 2015 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.

package model

import (
	"context"
	"fmt"
	"strings"
	"time"

	"github.com/thejerf/suture"

	"github.com/syncthing/syncthing/lib/config"
	"github.com/syncthing/syncthing/lib/events"
	"github.com/syncthing/syncthing/lib/protocol"
	"github.com/syncthing/syncthing/lib/sync"
	"github.com/syncthing/syncthing/lib/util"
)

const minSummaryInterval = time.Minute

type FolderSummaryService interface {
	suture.Service
	Summary(folder string) (map[string]interface{}, error)
	OnEventRequest()
}

// The folderSummaryService adds summary information events (FolderSummary and
// FolderCompletion) into the event stream at certain intervals.
type folderSummaryService struct {
	*suture.Supervisor

	cfg       config.Wrapper
	model     Model
	id        protocol.DeviceID
	evLogger  events.Logger
	immediate chan string

	// For keeping track of folders to recalculate for
	foldersMut sync.Mutex
	folders    map[string]struct{}

	// For keeping track of when the last event request on the API was
	lastEventReq    time.Time
	lastEventReqMut sync.Mutex
}

func NewFolderSummaryService(cfg config.Wrapper, m Model, id protocol.DeviceID, evLogger events.Logger) FolderSummaryService {
	service := &folderSummaryService{
		Supervisor: suture.New("folderSummaryService", suture.Spec{
			PassThroughPanics: true,
		}),
		cfg:             cfg,
		model:           m,
		id:              id,
		evLogger:        evLogger,
		immediate:       make(chan string),
		folders:         make(map[string]struct{}),
		foldersMut:      sync.NewMutex(),
		lastEventReqMut: sync.NewMutex(),
	}

	service.Add(util.AsService(service.listenForUpdates, fmt.Sprintf("%s/listenForUpdates", service)))
	service.Add(util.AsService(service.calculateSummaries, fmt.Sprintf("%s/calculateSummaries", service)))

	return service
}

func (c *folderSummaryService) String() string {
	return fmt.Sprintf("FolderSummaryService@%p", c)
}

func (c *folderSummaryService) Summary(folder string) (map[string]interface{}, error) {
	var res = make(map[string]interface{})

	errors, err := c.model.FolderErrors(folder)
	if err != nil && err != ErrFolderPaused {
		// Stats from the db can still be obtained if the folder is just paused
		return nil, err
	}
	res["errors"] = len(errors)
	res["pullErrors"] = len(errors) // deprecated

	res["invalid"] = "" // Deprecated, retains external API for now

	global := c.model.GlobalSize(folder)
	res["globalFiles"], res["globalDirectories"], res["globalSymlinks"], res["globalDeleted"], res["globalBytes"], res["globalTotalItems"] = global.Files, global.Directories, global.Symlinks, global.Deleted, global.Bytes, global.TotalItems()

	local := c.model.LocalSize(folder)
	res["localFiles"], res["localDirectories"], res["localSymlinks"], res["localDeleted"], res["localBytes"], res["localTotalItems"] = local.Files, local.Directories, local.Symlinks, local.Deleted, local.Bytes, local.TotalItems()

	need := c.model.NeedSize(folder)
	res["needFiles"], res["needDirectories"], res["needSymlinks"], res["needDeletes"], res["needBytes"], res["needTotalItems"] = need.Files, need.Directories, need.Symlinks, need.Deleted, need.Bytes, need.TotalItems()

	if c.cfg.Folders()[folder].Type == config.FolderTypeReceiveOnly {
		// Add statistics for things that have changed locally in a receive
		// only folder.
		ro := c.model.ReceiveOnlyChangedSize(folder)
		res["receiveOnlyChangedFiles"] = ro.Files
		res["receiveOnlyChangedDirectories"] = ro.Directories
		res["receiveOnlyChangedSymlinks"] = ro.Symlinks
		res["receiveOnlyChangedDeletes"] = ro.Deleted
		res["receiveOnlyChangedBytes"] = ro.Bytes
		res["receiveOnlyTotalItems"] = ro.TotalItems()
	}

	res["inSyncFiles"], res["inSyncBytes"] = global.Files-need.Files, global.Bytes-need.Bytes

	res["state"], res["stateChanged"], err = c.model.State(folder)
	if err != nil {
		res["error"] = err.Error()
	}

	ourSeq, _ := c.model.CurrentSequence(folder)
	remoteSeq, _ := c.model.RemoteSequence(folder)

	res["version"] = ourSeq + remoteSeq  // legacy
	res["sequence"] = ourSeq + remoteSeq // new name

	ignorePatterns, _, _ := c.model.GetIgnores(folder)
	res["ignorePatterns"] = false
	for _, line := range ignorePatterns {
		if len(line) > 0 && !strings.HasPrefix(line, "//") {
			res["ignorePatterns"] = true
			break
		}
	}

	err = c.model.WatchError(folder)
	if err != nil {
		res["watchError"] = err.Error()
	}

	return res, nil
}

func (c *folderSummaryService) OnEventRequest() {
	c.lastEventReqMut.Lock()
	c.lastEventReq = time.Now()
	c.lastEventReqMut.Unlock()
}

// listenForUpdates subscribes to the event bus and makes note of folders that
// need their data recalculated.
func (c *folderSummaryService) listenForUpdates(ctx context.Context) {
	sub := c.evLogger.Subscribe(events.LocalIndexUpdated | events.RemoteIndexUpdated | events.StateChanged | events.RemoteDownloadProgress | events.DeviceConnected | events.FolderWatchStateChanged | events.DownloadProgress)
	defer sub.Unsubscribe()

	for {
		// This loop needs to be fast so we don't miss too many events.

		select {
		case ev := <-sub.C():
			c.processUpdate(ev)
		case <-ctx.Done():
			return
		}
	}
}

func (c *folderSummaryService) processUpdate(ev events.Event) {
	var folder string

	switch ev.Type {
	case events.DeviceConnected:
		// When a device connects we schedule a refresh of all
		// folders shared with that device.

		data := ev.Data.(map[string]string)
		deviceID, _ := protocol.DeviceIDFromString(data["id"])

		c.foldersMut.Lock()
	nextFolder:
		for _, folder := range c.cfg.Folders() {
			for _, dev := range folder.Devices {
				if dev.DeviceID == deviceID {
					c.folders[folder.ID] = struct{}{}
					continue nextFolder
				}
			}
		}
		c.foldersMut.Unlock()

		return

	case events.DownloadProgress:
		data := ev.Data.(map[string]map[string]*pullerProgress)
		c.foldersMut.Lock()
		for folder := range data {
			c.folders[folder] = struct{}{}
		}
		c.foldersMut.Unlock()
		return

	case events.StateChanged:
		data := ev.Data.(map[string]interface{})
		if !(data["to"].(string) == "idle" && data["from"].(string) == "syncing") {
			return
		}

		// The folder changed to idle from syncing. We should do an
		// immediate refresh to update the GUI. The send to
		// c.immediate must be nonblocking so that we can continue
		// handling events.

		folder = data["folder"].(string)
		select {
		case c.immediate <- folder:
			c.foldersMut.Lock()
			delete(c.folders, folder)
			c.foldersMut.Unlock()
			return
		default:
			// Refresh whenever we do the next summary.
		}

	default:
		// The other events all have a "folder" attribute that they
		// affect. Whenever the local or remote index is updated for a
		// given folder we make a note of it.
		// This folder needs to be refreshed whenever we do the next
		// refresh.

		folder = ev.Data.(map[string]interface{})["folder"].(string)
	}

	c.foldersMut.Lock()
	c.folders[folder] = struct{}{}
	c.foldersMut.Unlock()
}

// calculateSummaries periodically recalculates folder summaries and
// completion percentage, and sends the results on the event bus.
func (c *folderSummaryService) calculateSummaries(ctx context.Context) {
	const pumpInterval = 2 * time.Second
	pump := time.NewTimer(pumpInterval)

	for {
		select {
		case <-pump.C:
			t0 := time.Now()
			for _, folder := range c.foldersToHandle() {
				c.sendSummary(folder)
			}

			// We don't want to spend all our time calculating summaries. Lets
			// set an arbitrary limit at not spending more than about 30% of
			// our time here...
			wait := 2*time.Since(t0) + pumpInterval
			pump.Reset(wait)

		case folder := <-c.immediate:
			c.sendSummary(folder)

		case <-ctx.Done():
			return
		}
	}
}

// foldersToHandle returns the list of folders needing a summary update, and
// clears the list.
func (c *folderSummaryService) foldersToHandle() []string {
	// We only recalculate summaries if someone is listening to events
	// (a request to /rest/events has been made within the last
	// pingEventInterval).

	c.lastEventReqMut.Lock()
	last := c.lastEventReq
	c.lastEventReqMut.Unlock()
	if time.Since(last) > minSummaryInterval {
		return nil
	}

	c.foldersMut.Lock()
	res := make([]string, 0, len(c.folders))
	for folder := range c.folders {
		res = append(res, folder)
		delete(c.folders, folder)
	}
	c.foldersMut.Unlock()
	return res
}

// sendSummary send the summary events for a single folder
func (c *folderSummaryService) sendSummary(folder string) {
	// The folder summary contains how many bytes, files etc
	// are in the folder and how in sync we are.
	data, err := c.Summary(folder)
	if err != nil {
		return
	}
	c.evLogger.Log(events.FolderSummary, map[string]interface{}{
		"folder":  folder,
		"summary": data,
	})

	for _, devCfg := range c.cfg.Folders()[folder].Devices {
		if devCfg.DeviceID.Equals(c.id) {
			// We already know about ourselves.
			continue
		}
		if _, ok := c.model.Connection(devCfg.DeviceID); !ok {
			// We're not interested in disconnected devices.
			continue
		}

		// Get completion percentage of this folder for the
		// remote device.
		comp := c.model.Completion(devCfg.DeviceID, folder).Map()
		comp["folder"] = folder
		comp["device"] = devCfg.DeviceID.String()
		c.evLogger.Log(events.FolderCompletion, comp)
	}
}
<commit_msg>lib/model: Also send folder summary from sync-preparing (ref #6028) (#6202)<commit_after>// Copyright (C) 2015 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.

package model

import (
	"context"
	"fmt"
	"strings"
	"time"

	"github.com/thejerf/suture"

	"github.com/syncthing/syncthing/lib/config"
	"github.com/syncthing/syncthing/lib/events"
	"github.com/syncthing/syncthing/lib/protocol"
	"github.com/syncthing/syncthing/lib/sync"
	"github.com/syncthing/syncthing/lib/util"
)

const minSummaryInterval = time.Minute

type FolderSummaryService interface {
	suture.Service
	Summary(folder string) (map[string]interface{}, error)
	OnEventRequest()
}

// The folderSummaryService adds summary information events (FolderSummary and
// FolderCompletion) into the event stream at certain intervals.
type folderSummaryService struct {
	*suture.Supervisor

	cfg       config.Wrapper
	model     Model
	id        protocol.DeviceID
	evLogger  events.Logger
	immediate chan string

	// For keeping track of folders to recalculate for
	foldersMut sync.Mutex
	folders    map[string]struct{}

	// For keeping track of when the last event request on the API was
	lastEventReq    time.Time
	lastEventReqMut sync.Mutex
}

func NewFolderSummaryService(cfg config.Wrapper, m Model, id protocol.DeviceID, evLogger events.Logger) FolderSummaryService {
	service := &folderSummaryService{
		Supervisor: suture.New("folderSummaryService", suture.Spec{
			PassThroughPanics: true,
		}),
		cfg:             cfg,
		model:           m,
		id:              id,
		evLogger:        evLogger,
		immediate:       make(chan string),
		folders:         make(map[string]struct{}),
		foldersMut:      sync.NewMutex(),
		lastEventReqMut: sync.NewMutex(),
	}

	service.Add(util.AsService(service.listenForUpdates, fmt.Sprintf("%s/listenForUpdates", service)))
	service.Add(util.AsService(service.calculateSummaries, fmt.Sprintf("%s/calculateSummaries", service)))

	return service
}

func (c *folderSummaryService) String() string {
	return fmt.Sprintf("FolderSummaryService@%p", c)
}

func (c *folderSummaryService) Summary(folder string) (map[string]interface{}, error) {
	var res = make(map[string]interface{})

	errors, err := c.model.FolderErrors(folder)
	if err != nil && err != ErrFolderPaused {
		// Stats from the db can still be obtained if the folder is just paused
		return nil, err
	}
	res["errors"] = len(errors)
	res["pullErrors"] = len(errors) // deprecated

	res["invalid"] = "" // Deprecated, retains external API for now

	global := c.model.GlobalSize(folder)
	res["globalFiles"], res["globalDirectories"], res["globalSymlinks"], res["globalDeleted"], res["globalBytes"], res["globalTotalItems"] = global.Files, global.Directories, global.Symlinks, global.Deleted, global.Bytes, global.TotalItems()

	local := c.model.LocalSize(folder)
	res["localFiles"], res["localDirectories"], res["localSymlinks"], res["localDeleted"], res["localBytes"], res["localTotalItems"] = local.Files, local.Directories, local.Symlinks, local.Deleted, local.Bytes, local.TotalItems()

	need := c.model.NeedSize(folder)
	res["needFiles"], res["needDirectories"], res["needSymlinks"], res["needDeletes"], res["needBytes"], res["needTotalItems"] = need.Files, need.Directories, need.Symlinks, need.Deleted, need.Bytes, need.TotalItems()

	if c.cfg.Folders()[folder].Type == config.FolderTypeReceiveOnly {
		// Add statistics for things that have changed locally in a receive
		// only folder.
		ro := c.model.ReceiveOnlyChangedSize(folder)
		res["receiveOnlyChangedFiles"] = ro.Files
		res["receiveOnlyChangedDirectories"] = ro.Directories
		res["receiveOnlyChangedSymlinks"] = ro.Symlinks
		res["receiveOnlyChangedDeletes"] = ro.Deleted
		res["receiveOnlyChangedBytes"] = ro.Bytes
		res["receiveOnlyTotalItems"] = ro.TotalItems()
	}

	res["inSyncFiles"], res["inSyncBytes"] = global.Files-need.Files, global.Bytes-need.Bytes

	res["state"], res["stateChanged"], err = c.model.State(folder)
	if err != nil {
		res["error"] = err.Error()
	}

	ourSeq, _ := c.model.CurrentSequence(folder)
	remoteSeq, _ := c.model.RemoteSequence(folder)

	res["version"] = ourSeq + remoteSeq  // legacy
	res["sequence"] = ourSeq + remoteSeq // new name

	ignorePatterns, _, _ := c.model.GetIgnores(folder)
	res["ignorePatterns"] = false
	for _, line := range ignorePatterns {
		if len(line) > 0 && !strings.HasPrefix(line, "//") {
			res["ignorePatterns"] = true
			break
		}
	}

	err = c.model.WatchError(folder)
	if err != nil {
		res["watchError"] = err.Error()
	}

	return res, nil
}

func (c *folderSummaryService) OnEventRequest() {
	c.lastEventReqMut.Lock()
	c.lastEventReq = time.Now()
	c.lastEventReqMut.Unlock()
}

// listenForUpdates subscribes to the event bus and makes note of folders that
// need their data recalculated.
func (c *folderSummaryService) listenForUpdates(ctx context.Context) {
	sub := c.evLogger.Subscribe(events.LocalIndexUpdated | events.RemoteIndexUpdated | events.StateChanged | events.RemoteDownloadProgress | events.DeviceConnected | events.FolderWatchStateChanged | events.DownloadProgress)
	defer sub.Unsubscribe()

	for {
		// This loop needs to be fast so we don't miss too many events.

		select {
		case ev := <-sub.C():
			c.processUpdate(ev)
		case <-ctx.Done():
			return
		}
	}
}

func (c *folderSummaryService) processUpdate(ev events.Event) {
	var folder string

	switch ev.Type {
	case events.DeviceConnected:
		// When a device connects we schedule a refresh of all
		// folders shared with that device.

		data := ev.Data.(map[string]string)
		deviceID, _ := protocol.DeviceIDFromString(data["id"])

		c.foldersMut.Lock()
	nextFolder:
		for _, folder := range c.cfg.Folders() {
			for _, dev := range folder.Devices {
				if dev.DeviceID == deviceID {
					c.folders[folder.ID] = struct{}{}
					continue nextFolder
				}
			}
		}
		c.foldersMut.Unlock()

		return

	case events.DownloadProgress:
		data := ev.Data.(map[string]map[string]*pullerProgress)
		c.foldersMut.Lock()
		for folder := range data {
			c.folders[folder] = struct{}{}
		}
		c.foldersMut.Unlock()
		return

	case events.StateChanged:
		data := ev.Data.(map[string]interface{})
		if data["to"].(string) != "idle" {
			return
		}
		if from := data["from"].(string); from != "syncing" && from != "sync-preparing" {
			return
		}

		// The folder changed to idle from syncing. We should do an
		// immediate refresh to update the GUI. The send to
		// c.immediate must be nonblocking so that we can continue
		// handling events.

		folder = data["folder"].(string)
		select {
		case c.immediate <- folder:
			c.foldersMut.Lock()
			delete(c.folders, folder)
			c.foldersMut.Unlock()
			return
		default:
			// Refresh whenever we do the next summary.
		}

	default:
		// The other events all have a "folder" attribute that they
		// affect. Whenever the local or remote index is updated for a
		// given folder we make a note of it.
		// This folder needs to be refreshed whenever we do the next
		// refresh.

		folder = ev.Data.(map[string]interface{})["folder"].(string)
	}

	c.foldersMut.Lock()
	c.folders[folder] = struct{}{}
	c.foldersMut.Unlock()
}

// calculateSummaries periodically recalculates folder summaries and
// completion percentage, and sends the results on the event bus.
func (c *folderSummaryService) calculateSummaries(ctx context.Context) {
	const pumpInterval = 2 * time.Second
	pump := time.NewTimer(pumpInterval)

	for {
		select {
		case <-pump.C:
			t0 := time.Now()
			for _, folder := range c.foldersToHandle() {
				c.sendSummary(folder)
			}

			// We don't want to spend all our time calculating summaries. Lets
			// set an arbitrary limit at not spending more than about 30% of
			// our time here...
			wait := 2*time.Since(t0) + pumpInterval
			pump.Reset(wait)

		case folder := <-c.immediate:
			c.sendSummary(folder)

		case <-ctx.Done():
			return
		}
	}
}

// foldersToHandle returns the list of folders needing a summary update, and
// clears the list.
func (c *folderSummaryService) foldersToHandle() []string {
	// We only recalculate summaries if someone is listening to events
	// (a request to /rest/events has been made within the last
	// pingEventInterval).

	c.lastEventReqMut.Lock()
	last := c.lastEventReq
	c.lastEventReqMut.Unlock()
	if time.Since(last) > minSummaryInterval {
		return nil
	}

	c.foldersMut.Lock()
	res := make([]string, 0, len(c.folders))
	for folder := range c.folders {
		res = append(res, folder)
		delete(c.folders, folder)
	}
	c.foldersMut.Unlock()
	return res
}

// sendSummary send the summary events for a single folder
func (c *folderSummaryService) sendSummary(folder string) {
	// The folder summary contains how many bytes, files etc
	// are in the folder and how in sync we are.
	data, err := c.Summary(folder)
	if err != nil {
		return
	}
	c.evLogger.Log(events.FolderSummary, map[string]interface{}{
		"folder":  folder,
		"summary": data,
	})

	for _, devCfg := range c.cfg.Folders()[folder].Devices {
		if devCfg.DeviceID.Equals(c.id) {
			// We already know about ourselves.
			continue
		}
		if _, ok := c.model.Connection(devCfg.DeviceID); !ok {
			// We're not interested in disconnected devices.
			continue
		}

		// Get completion percentage of this folder for the
		// remote device.
		comp := c.model.Completion(devCfg.DeviceID, folder).Map()
		comp["folder"] = folder
		comp["device"] = devCfg.DeviceID.String()
		c.evLogger.Log(events.FolderCompletion, comp)
	}
}
<|endoftext|>


<commit_before>/*
 * Copyright 2014 Google Inc. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package service

import (
	"fmt"
	"reflect"
	"testing"

	"third_party/kythe/go/rpc/server"
	testutil "shipshape/test"
	strset "shipshape/util/strings"

	"code.google.com/p/goprotobuf/proto"

	notepb "shipshape/proto/note_proto"
	ctxpb "shipshape/proto/shipshape_context_proto"
	rpcpb "shipshape/proto/shipshape_rpc_proto"
)

type fakeDispatcher struct {
	categories []string
	files      []string
}

func (f fakeDispatcher) GetCategory(ctx server.Context, in *rpcpb.GetCategoryRequest) (*rpcpb.GetCategoryResponse, error) {
	return &rpcpb.GetCategoryResponse{
		Category: f.categories,
	}, nil
}

func isSubset(a, b strset.Set) bool {
	return len(a.Intersect(b)) == len(a)
}

func (f fakeDispatcher) Analyze(ctx server.Context, in *rpcpb.AnalyzeRequest) (*rpcpb.AnalyzeResponse, error) {
	var nts []*notepb.Note

	// Assert that the analyzer was called with the right categories.
	if !isSubset(strset.New(in.Category...), strset.New(f.categories...)) {
		return nil, fmt.Errorf("Category mismatch: got %v, supports %v", in.Category, f.categories)
	}

	for _, file := range f.files {
		nts = append(nts, &notepb.Note{
			Category:    proto.String(f.categories[0]),
			Description: proto.String("Hello world"),
			Location:    testutil.CreateLocation(file),
		})
	}

	return &rpcpb.AnalyzeResponse{
		Note: nts,
	}, nil
}

type errDispatcher struct{}

func (errDispatcher) GetCategory(ctx server.Context, in *rpcpb.GetCategoryRequest) (*rpcpb.GetCategoryResponse, error) {
	return nil, fmt.Errorf("An error")
}

type panicDispatcher struct{}

func (panicDispatcher) GetCategory(ctx server.Context, in *rpcpb.GetCategoryRequest) (*rpcpb.GetCategoryResponse, error) {
	panic("panic")
}

type fullFakeDispatcher struct {
	response *rpcpb.AnalyzeResponse
}

func (f fullFakeDispatcher) Analyze(ctx server.Context, in *rpcpb.AnalyzeRequest) (*rpcpb.AnalyzeResponse, error) {
	return f.response, nil
}

func TestGetCategories(t *testing.T) {
	addr2, cleanup, err := testutil.CreatekRPCTestServer(&fakeDispatcher{[]string{"Foo", "Bar"}, nil}, "AnalyzerService")
	if err != nil {
		t.Fatalf("Registering analyzer service failed: %v", err)
	}
	defer cleanup()

	addr0, cleanup, err := testutil.CreatekRPCTestServer(&fakeDispatcher{nil, nil}, "AnalyzerService")
	if err != nil {
		t.Fatalf("Registering analyzer service failed: %v", err)
	}
	defer cleanup()

	addre, cleanup, err := testutil.CreatekRPCTestServer(&errDispatcher{}, "AnalyzerService")
	if err != nil {
		t.Fatalf("Registering analyzer service failed: %v", err)
	}
	defer cleanup()

	addrp, cleanup, err := testutil.CreatekRPCTestServer(&panicDispatcher{}, "AnalyzerService")
	if err != nil {
		t.Fatalf("Registering analyzer service failed: %v", err)
	}
	defer cleanup()

	tests := []struct {
		addrs  []string
		result map[string]strset.Set
	}{
		{[]string{addr0}, map[string]strset.Set{addr0: nil}},
		{[]string{addr2}, map[string]strset.Set{addr2: strset.New("Foo", "Bar")}},
		{[]string{addr0, addr2}, map[string]strset.Set{addr2: strset.New("Foo", "Bar"), addr0: nil}},
		{[]string{addrp, addr2}, map[string]strset.Set{addr2: strset.New("Foo", "Bar"), addrp: nil}},
		{[]string{addre, addr2}, map[string]strset.Set{addr2: strset.New("Foo", "Bar"), addre: nil}},
	}

	for _, test := range tests {
		driver := NewDriver(test.addrs)
		categories := driver.getAllCategories()

		if len(test.result) != len(categories) {
			t.Errorf("Incorrect number of results: got %v, want %v", categories, test.result)
		}

		for addr, cats := range test.result {
			if !strset.Equal(cats.ToSlice(), test.result[addr].ToSlice()) {
				t.Errorf("Incorrect categories for %s: got %v, want %v", addr, cats, test.result[addr])
			}
		}
	}
}

func TestCallAllAnalyzers(t *testing.T) {
	dispatcher := &fakeDispatcher{categories: []string{"Foo", "Bar"}, files: []string{"dir1/A.h", "dir1/A.cc"}}
	addr, cleanup, err := testutil.CreatekRPCTestServer(dispatcher, "AnalyzerService")
	if err != nil {
		t.Fatalf("Registering analyzer service failed: %v", err)
	}
	defer cleanup()

	driver := NewTestDriver(map[string]strset.Set{addr: strset.New("Foo", "Bar")})

	tests := []struct {
		files      []string
		categories []string
		expect     []*notepb.Note
	}{
		{
			// - SomeOtherCategory should be dropped; if it weren't fakeDispatcher would return an error.
			// - fakeDispatcher produces notes for files we didn't ask about; they should be dropped.
			[]string{"dir1/A.cc"},
			[]string{"Foo", "SomeOtherCategory"},
			[]*notepb.Note{
				&notepb.Note{
					Category:    proto.String("Foo"),
					Description: proto.String(""),
					Location:    testutil.CreateLocation("dir1/A.cc"),
				},
			},
		},
	}
	for _, test := range tests {
		cfg := &config{categories: test.categories}
		ctx := &ctxpb.ShipshapeContext{FilePath: test.files}

		ars := driver.callAllAnalyzers(cfg, ctx)
		var notes []*notepb.Note

		for _, ar := range ars {
			notes = append(notes, ar.Note...)
			if len(ar.Failure) > 0 {
				t.Errorf("Received failures from analyze call: %v", ar.Failure)
			}
		}

		ok, results := testutil.CheckNoteContainsContent(test.expect, notes)
		if !ok {
			t.Errorf("Incorrect notes for config %v: %s\n got %v, want %v", cfg, results, notes, test.expect)
		}
	}
}

func TestCallAllAnalyzersErrorCases(t *testing.T) {
	ctx := &ctxpb.ShipshapeContext{FilePath: []string{"dir1/A", "dir2/B"}}
	cfg := &config{categories: []string{"Foo"}}

	tests := []struct {
		response      *rpcpb.AnalyzeResponse
		expectNotes   []*notepb.Note
		expectFailure []*rpcpb.AnalysisFailure
	}{
		{ //analysis had a failure
			&rpcpb.AnalyzeResponse{
				Failure: []*rpcpb.AnalysisFailure{
					&rpcpb.AnalysisFailure{
						Category:       proto.String("Foo"),
						FailureMessage: proto.String("badbadbad"),
					},
				},
			},
			nil,
			[]*rpcpb.AnalysisFailure{
				&rpcpb.AnalysisFailure{
					Category:       proto.String("Foo"),
					FailureMessage: proto.String("badbadbad"),
				},
			},
		},
		{ //analysis had both failure and notes
			&rpcpb.AnalyzeResponse{
				Note: []*notepb.Note{
					&notepb.Note{
						Category:    proto.String("Foo"),
						Description: proto.String("A note"),
						Location:    testutil.CreateLocation("dir1/A"),
					},
					&notepb.Note{
						Category:    proto.String("Foo"),
						Description: proto.String("A note"),
						Location:    testutil.CreateLocation("dir1/A"),
					},
				},
				Failure: []*rpcpb.AnalysisFailure{
					&rpcpb.AnalysisFailure{
						Category:       proto.String("Foo"),
						FailureMessage: proto.String("badbadbad"),
					},
				},
			},
			[]*notepb.Note{
				&notepb.Note{
					Category:    proto.String("Foo"),
					Description: proto.String("A note"),
					Location:    testutil.CreateLocation("dir1/A"),
				},
				&notepb.Note{
					Category:    proto.String("Foo"),
					Description: proto.String("A note"),
					Location:    testutil.CreateLocation("dir1/A"),
				},
			},
			[]*rpcpb.AnalysisFailure{
				&rpcpb.AnalysisFailure{
					Category:       proto.String("Foo"),
					FailureMessage: proto.String("badbadbad"),
				},
			},
		},
	}
	for _, test := range tests {
		addr, cleanup, err := testutil.CreatekRPCTestServer(&fullFakeDispatcher{test.response}, "AnalyzerService")
		if err != nil {
			t.Fatalf("Registering analyzer service failed: %v", err)
		}
		defer cleanup()

		driver := NewTestDriver(map[string]strset.Set{addr: strset.New("Foo")})

		ars := driver.callAllAnalyzers(cfg, ctx)
		var notes []*notepb.Note
		var failures []*rpcpb.AnalysisFailure

		for _, ar := range ars {
			notes = append(notes, ar.Note...)
			failures = append(failures, ar.Failure...)
		}

		ok, results := testutil.CheckNoteContainsContent(test.expectNotes, notes)
		if !ok {
			t.Errorf("Incorrect notes for original response %v: %s\n got %v, want %v", test.response, results, notes, test.expectNotes)
		}
		ok, results = testutil.CheckFailureContainsContent(test.expectFailure, failures)
		if !ok {
			t.Errorf("Incorrect failures for original response %v: %s\n got %v, want %v", test.response, results, failures, test.expectFailure)
		}

	}
}

func TestFilterPaths(t *testing.T) {
	tests := []struct {
		label         string
		ignoreDirs    []string
		inputFiles    []string
		expectedFiles []string
	}{
		{
			"Empty ignore list",
			[]string{},
			[]string{"a", "b", "c", "d"},
			[]string{"a", "b", "c", "d"},
		},
		{
			"Ignore list matches some items",
			[]string{"dir1/", "dir2/"},
			[]string{"dir1/a", "dir3/a", "dir2/a"},
			[]string{"dir3/a"},
		},
		{
			"Ignore list does not match anything",
			[]string{"dir77/"},
			[]string{"dir1/a", "dir3/a", "dir2/a"},
			[]string{"dir1/a", "dir3/a", "dir2/a"},
		},
		{
			"Ignore list contains substring match",
			[]string{"dir1/"},
			[]string{"dir4/a", "dir4/dir1/a"},
			[]string{"dir4/a", "dir4/dir1/a"},
		},
		{
			"Ignore list matches everything",
			[]string{"dir1/"},
			[]string{"dir1/a", "dir1/b", "dir1/c"},
			nil,
		},
	}

	for _, test := range tests {
		out := filterPaths(test.ignoreDirs, test.inputFiles)
		if !reflect.DeepEqual(out, test.expectedFiles) {
			t.Fatalf("Error on %q: got %v, expected %v", test.label, out, test.expectedFiles)
		}
	}
}
<commit_msg>Existing driver tests pass now. We should add more tests for the stages, but will do that in a separate CL<commit_after>/*
 * Copyright 2014 Google Inc. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package service

import (
	"fmt"
	"reflect"
	"strings"
	"testing"

	strset "shipshape/util/strings"
	testutil "shipshape/util/test"
	"third_party/kythe/go/rpc/server"

	"code.google.com/p/goprotobuf/proto"

	notepb "shipshape/proto/note_proto"
	ctxpb "shipshape/proto/shipshape_context_proto"
	rpcpb "shipshape/proto/shipshape_rpc_proto"
)

type fakeDispatcher struct {
	categories []string
	files      []string
}

func (f fakeDispatcher) GetCategory(ctx server.Context, in *rpcpb.GetCategoryRequest) (*rpcpb.GetCategoryResponse, error) {
	return &rpcpb.GetCategoryResponse{
		Category: f.categories,
	}, nil
}

func (f fakeDispatcher) GetStage(ctx server.Context, in *rpcpb.GetStageRequest) (*rpcpb.GetStageResponse, error) {
	return &rpcpb.GetStageResponse{
		Stage: ctxpb.Stage_PRE_BUILD.Enum(),
	}, nil
}
func isSubset(a, b strset.Set) bool {
	return len(a.Intersect(b)) == len(a)
}

func (f fakeDispatcher) Analyze(ctx server.Context, in *rpcpb.AnalyzeRequest) (*rpcpb.AnalyzeResponse, error) {
	var nts []*notepb.Note

	// Assert that the analyzer was called with the right categories.
	if !isSubset(strset.New(in.Category...), strset.New(f.categories...)) {
		return nil, fmt.Errorf("Category mismatch: got %v, supports %v", in.Category, f.categories)
	}

	for _, file := range f.files {
		nts = append(nts, &notepb.Note{
			Category:    proto.String(f.categories[0]),
			Description: proto.String("Hello world"),
			Location:    testutil.CreateLocation(file),
		})
	}

	return &rpcpb.AnalyzeResponse{
		Note: nts,
	}, nil
}

type errDispatcher struct{}

func (errDispatcher) GetCategory(ctx server.Context, in *rpcpb.GetCategoryRequest) (*rpcpb.GetCategoryResponse, error) {
	return nil, fmt.Errorf("An error")
}

func (errDispatcher) GetStage(ctx server.Context, in *rpcpb.GetStageRequest) (*rpcpb.GetStageResponse, error) {
	return nil, fmt.Errorf("An error")
}

type panicDispatcher struct{}

func (panicDispatcher) GetCategory(ctx server.Context, in *rpcpb.GetCategoryRequest) (*rpcpb.GetCategoryResponse, error) {
	panic("panic")
}

func (panicDispatcher) GetStage(ctx server.Context, in *rpcpb.GetStageRequest) (*rpcpb.GetStageResponse, error) {
	panic("panic")
}

type fullFakeDispatcher struct {
	response *rpcpb.AnalyzeResponse
}

func (f fullFakeDispatcher) Analyze(ctx server.Context, in *rpcpb.AnalyzeRequest) (*rpcpb.AnalyzeResponse, error) {
	return f.response, nil
}

func TestGetServiceInfo(t *testing.T) {
	addr2, cleanup, err := testutil.CreatekRPCTestServer(&fakeDispatcher{[]string{"Foo", "Bar"}, nil}, "AnalyzerService")
	if err != nil {
		t.Fatalf("Registering analyzer service failed: %v", err)
	}
	defer cleanup()

	addr0, cleanup, err := testutil.CreatekRPCTestServer(&fakeDispatcher{nil, nil}, "AnalyzerService")
	if err != nil {
		t.Fatalf("Registering analyzer service failed: %v", err)
	}
	defer cleanup()

	addre, cleanup, err := testutil.CreatekRPCTestServer(&errDispatcher{}, "AnalyzerService")
	if err != nil {
		t.Fatalf("Registering analyzer service failed: %v", err)
	}
	defer cleanup()

	addrp, cleanup, err := testutil.CreatekRPCTestServer(&panicDispatcher{}, "AnalyzerService")
	if err != nil {
		t.Fatalf("Registering analyzer service failed: %v", err)
	}
	defer cleanup()

	tests := []struct {
		addrs  []string
		result map[string]strset.Set
	}{
		{[]string{addr0}, map[string]strset.Set{addr0: nil}},
		{[]string{addr2}, map[string]strset.Set{addr2: strset.New("Foo", "Bar")}},
		{[]string{addr0, addr2}, map[string]strset.Set{addr2: strset.New("Foo", "Bar"), addr0: nil}},
		{[]string{addrp, addr2}, map[string]strset.Set{addr2: strset.New("Foo", "Bar"), addrp: nil}},
		{[]string{addre, addr2}, map[string]strset.Set{addr2: strset.New("Foo", "Bar"), addre: nil}},
	}

	for _, test := range tests {
		driver := NewDriver(test.addrs)
		info := driver.getAllServiceInfo()

		if len(test.result) != len(info) {
			t.Errorf("Incorrect number of results: got %v, want %v", info, test.result)
		}

		for addr, expectCats := range test.result {
			if !strset.Equal(info[strings.TrimPrefix(addr, "http://")].categories.ToSlice(), expectCats.ToSlice()) {
				t.Errorf("Incorrect categories for %s: got %v, want %v", addr, info[addr].categories, expectCats)
			}
		}
	}
}

func TestCallAllAnalyzers(t *testing.T) {
	dispatcher := &fakeDispatcher{categories: []string{"Foo", "Bar"}, files: []string{"dir1/A.h", "dir1/A.cc"}}
	addr, cleanup, err := testutil.CreatekRPCTestServer(dispatcher, "AnalyzerService")
	if err != nil {
		t.Fatalf("Registering analyzer service failed: %v", err)
	}
	defer cleanup()

	driver := NewTestDriver([]serviceInfo{
		serviceInfo{addr, strset.New("Foo", "Bar"), ctxpb.Stage_PRE_BUILD},
	})

	tests := []struct {
		files      []string
		categories []string
		expect     []*notepb.Note
	}{
		{
			// - SomeOtherCategory should be dropped; if it weren't fakeDispatcher would return an error.
			// - fakeDispatcher produces notes for files we didn't ask about; they should be dropped.
			[]string{"dir1/A.cc"},
			[]string{"Foo", "SomeOtherCategory"},
			[]*notepb.Note{
				&notepb.Note{
					Category:    proto.String("Foo"),
					Description: proto.String(""),
					Location:    testutil.CreateLocation("dir1/A.cc"),
				},
			},
		},
	}
	for _, test := range tests {
		ctx := &ctxpb.ShipshapeContext{FilePath: test.files}

		ars := driver.callAllAnalyzers(strset.New(test.categories...), ctx, ctxpb.Stage_PRE_BUILD)
		var notes []*notepb.Note

		for _, ar := range ars {
			notes = append(notes, ar.Note...)
			if len(ar.Failure) > 0 {
				t.Errorf("Received failures from analyze call: %v", ar.Failure)
			}
		}

		ok, results := testutil.CheckNoteContainsContent(test.expect, notes)
		if !ok {
			t.Errorf("Incorrect notes for categories %v: %s\n got %v, want %v", test.categories, results, notes, test.expect)
		}
	}
}

func TestCallAllAnalyzersErrorCases(t *testing.T) {
	ctx := &ctxpb.ShipshapeContext{FilePath: []string{"dir1/A", "dir2/B"}}

	tests := []struct {
		response      *rpcpb.AnalyzeResponse
		expectNotes   []*notepb.Note
		expectFailure []*rpcpb.AnalysisFailure
	}{
		{ //analysis had a failure
			&rpcpb.AnalyzeResponse{
				Failure: []*rpcpb.AnalysisFailure{
					&rpcpb.AnalysisFailure{
						Category:       proto.String("Foo"),
						FailureMessage: proto.String("badbadbad"),
					},
				},
			},
			nil,
			[]*rpcpb.AnalysisFailure{
				&rpcpb.AnalysisFailure{
					Category:       proto.String("Foo"),
					FailureMessage: proto.String("badbadbad"),
				},
			},
		},
		{ //analysis had both failure and notes
			&rpcpb.AnalyzeResponse{
				Note: []*notepb.Note{
					&notepb.Note{
						Category:    proto.String("Foo"),
						Description: proto.String("A note"),
						Location:    testutil.CreateLocation("dir1/A"),
					},
					&notepb.Note{
						Category:    proto.String("Foo"),
						Description: proto.String("A note"),
						Location:    testutil.CreateLocation("dir1/A"),
					},
				},
				Failure: []*rpcpb.AnalysisFailure{
					&rpcpb.AnalysisFailure{
						Category:       proto.String("Foo"),
						FailureMessage: proto.String("badbadbad"),
					},
				},
			},
			[]*notepb.Note{
				&notepb.Note{
					Category:    proto.String("Foo"),
					Description: proto.String("A note"),
					Location:    testutil.CreateLocation("dir1/A"),
				},
				&notepb.Note{
					Category:    proto.String("Foo"),
					Description: proto.String("A note"),
					Location:    testutil.CreateLocation("dir1/A"),
				},
			},
			[]*rpcpb.AnalysisFailure{
				&rpcpb.AnalysisFailure{
					Category:       proto.String("Foo"),
					FailureMessage: proto.String("badbadbad"),
				},
			},
		},
	}
	for _, test := range tests {
		addr, cleanup, err := testutil.CreatekRPCTestServer(&fullFakeDispatcher{test.response}, "AnalyzerService")
		if err != nil {
			t.Fatalf("Registering analyzer service failed: %v", err)
		}
		defer cleanup()

		driver := NewTestDriver([]serviceInfo{
			serviceInfo{addr, strset.New("Foo"), ctxpb.Stage_PRE_BUILD},
		})

		ars := driver.callAllAnalyzers(strset.New("Foo"), ctx, ctxpb.Stage_PRE_BUILD)
		var notes []*notepb.Note
		var failures []*rpcpb.AnalysisFailure

		for _, ar := range ars {
			notes = append(notes, ar.Note...)
			failures = append(failures, ar.Failure...)
		}

		ok, results := testutil.CheckNoteContainsContent(test.expectNotes, notes)
		if !ok {
			t.Errorf("Incorrect notes for original response %v: %s\n got %v, want %v", test.response, results, notes, test.expectNotes)
		}
		ok, results = testutil.CheckFailureContainsContent(test.expectFailure, failures)
		if !ok {
			t.Errorf("Incorrect failures for original response %v: %s\n got %v, want %v", test.response, results, failures, test.expectFailure)
		}

	}
}

func TestFilterPaths(t *testing.T) {
	tests := []struct {
		label         string
		ignoreDirs    []string
		inputFiles    []string
		expectedFiles []string
	}{
		{
			"Empty ignore list",
			[]string{},
			[]string{"a", "b", "c", "d"},
			[]string{"a", "b", "c", "d"},
		},
		{
			"Ignore list matches some items",
			[]string{"dir1/", "dir2/"},
			[]string{"dir1/a", "dir3/a", "dir2/a"},
			[]string{"dir3/a"},
		},
		{
			"Ignore list does not match anything",
			[]string{"dir77/"},
			[]string{"dir1/a", "dir3/a", "dir2/a"},
			[]string{"dir1/a", "dir3/a", "dir2/a"},
		},
		{
			"Ignore list contains substring match",
			[]string{"dir1/"},
			[]string{"dir4/a", "dir4/dir1/a"},
			[]string{"dir4/a", "dir4/dir1/a"},
		},
		{
			"Ignore list matches everything",
			[]string{"dir1/"},
			[]string{"dir1/a", "dir1/b", "dir1/c"},
			nil,
		},
	}

	for _, test := range tests {
		out := filterPaths(test.ignoreDirs, test.inputFiles)
		if !reflect.DeepEqual(out, test.expectedFiles) {
			t.Fatalf("Error on %q: got %v, expected %v", test.label, out, test.expectedFiles)
		}
	}
}
<|endoftext|>


<commit_before>package gob

import (
	"bytes"
	"encoding/gob"
	"fmt"
	"io"
	"reflect"

	"github.com/dvyukov/go-fuzz/examples/fuzz"
)

type X struct {
	A int
	B string
	C float64
	D []byte
	E interface{}
	F complex128
	G []interface{}
	H *int
	I **int
	J *X
	K map[string]int
}

func init() {
	gob.Register(X{})
}

func Fuzz(data []byte) int {
	score := 0
	for _, ctor := range []func() interface{}{
		func() interface{} { return nil },
		func() interface{} { return new(int) },
		func() interface{} { return new(string) },
		func() interface{} { return new(float64) },
		func() interface{} { return new([]byte) },
		func() interface{} { return new(interface{}) },
		func() interface{} { return new(complex128) },
		func() interface{} { m := make(map[int]int); return &m },
		func() interface{} { m := make(map[string]interface{}); return &m },
		func() interface{} { return new(X) },
	} {
		v := ctor()
		dec := gob.NewDecoder(bytes.NewReader(data))
		if dec.Decode(v) != nil {
			continue
		}
		dec.Decode(ctor())
		score = 1
		if ctor() == nil {
			continue
		}
		b1 := new(bytes.Buffer)
		if err := gob.NewEncoder(b1).Encode(v); err != nil {
			panic(err)
		}
		v1 := reflect.ValueOf(ctor())
		err := gob.NewDecoder(bytes.NewReader(data)).DecodeValue(v1)
		if err != nil {
			panic(err)
		}
		if !fuzz.DeepEqual(v, v1.Interface()) {
			fmt.Printf("v0: %#v\n", reflect.ValueOf(v).Elem().Interface())
			fmt.Printf("v1: %#v\n", v1.Elem().Interface())
			panic(fmt.Sprintf("values not equal %T", v))
		}
		b2 := new(bytes.Buffer)
		err = gob.NewEncoder(b2).EncodeValue(v1)
		if err != nil {
			panic(err)
		}
		v2 := ctor()
		dec1 := gob.NewDecoder(b1)
		if err := dec1.Decode(v2); err != nil {
			panic(err)
		}
		if err := dec1.Decode(ctor()); err != io.EOF {
			panic(err)
		}
		if vv, ok := v.(*X); ok && vv.I != nil && (*vv.I == nil || **vv.I == 0) {
			// If input contains "I:42 I:null", then I will be in this weird state.
			// It is effectively nil, but DeepEqual does not handle such case.
			vv.I = nil
		}
		if !fuzz.DeepEqual(v, v2) {
			fmt.Printf("v0: %#v\n", reflect.ValueOf(v).Elem().Interface())
			fmt.Printf("v2: %#v\n", reflect.ValueOf(v2).Elem().Interface())
			panic(fmt.Sprintf("values not equal 2 %T", v))
		}
	}
	return score
}
<commit_msg>fix gob test (the weird pointers can appear in nested X structs)<commit_after>package gob

import (
	"bytes"
	"encoding/gob"
	"fmt"
	"io"
	"reflect"

	"github.com/dvyukov/go-fuzz/examples/fuzz"
)

type X struct {
	A int
	B string
	C float64
	D []byte
	E interface{}
	F complex128
	G []interface{}
	H *int
	I **int
	J *X
	K map[string]int
}

func init() {
	gob.Register(X{})
}

func Fuzz(data []byte) int {
	score := 0
	for _, ctor := range []func() interface{}{
		func() interface{} { return nil },
		func() interface{} { return new(int) },
		func() interface{} { return new(string) },
		func() interface{} { return new(float64) },
		func() interface{} { return new([]byte) },
		func() interface{} { return new(interface{}) },
		func() interface{} { return new(complex128) },
		func() interface{} { m := make(map[int]int); return &m },
		func() interface{} { m := make(map[string]interface{}); return &m },
		func() interface{} { return new(X) },
	} {
		v := ctor()
		dec := gob.NewDecoder(bytes.NewReader(data))
		if dec.Decode(v) != nil {
			continue
		}
		dec.Decode(ctor())
		score = 1
		if ctor() == nil {
			continue
		}
		b1 := new(bytes.Buffer)
		if err := gob.NewEncoder(b1).Encode(v); err != nil {
			panic(err)
		}
		v1 := reflect.ValueOf(ctor())
		err := gob.NewDecoder(bytes.NewReader(data)).DecodeValue(v1)
		if err != nil {
			panic(err)
		}
		if !fuzz.DeepEqual(v, v1.Interface()) {
			fmt.Printf("v0: %#v\n", reflect.ValueOf(v).Elem().Interface())
			fmt.Printf("v1: %#v\n", v1.Elem().Interface())
			panic(fmt.Sprintf("values not equal %T", v))
		}
		b2 := new(bytes.Buffer)
		err = gob.NewEncoder(b2).EncodeValue(v1)
		if err != nil {
			panic(err)
		}
		v2 := ctor()
		dec1 := gob.NewDecoder(b1)
		if err := dec1.Decode(v2); err != nil {
			panic(err)
		}
		if err := dec1.Decode(ctor()); err != io.EOF {
			panic(err)
		}
		if vv, ok := v.(*X); ok {
			fix(vv)
		}
		if !fuzz.DeepEqual(v, v2) {
			fmt.Printf("v0: %#v\n", reflect.ValueOf(v).Elem().Interface())
			fmt.Printf("v2: %#v\n", reflect.ValueOf(v2).Elem().Interface())
			panic(fmt.Sprintf("values not equal 2 %T", v))
		}
	}
	return score
}

func fix(vv *X) {
	// See https://github.com/golang/go/issues/11119
	if vv.I != nil && (*vv.I == nil || **vv.I == 0) {
		// If input contains "I:42 I:null", then I will be in this weird state.
		// It is effectively nil, but DeepEqual does not handle such case.
		vv.I = nil
	}
	if vv.H != nil && *vv.H == 0 {
		vv.H = nil
	}
	if vv.J != nil {
		fix(vv.J)
	}
}
<|endoftext|>


<commit_before>package test

import (
	"errors"
	"log"
	"os"
	"sync"
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"

	"github.com/anacrolix/torrent"
	"github.com/anacrolix/torrent/internal/testutil"
	"github.com/anacrolix/torrent/metainfo"
	"github.com/anacrolix/torrent/storage"
)

func justOneNetwork(cc *torrent.ClientConfig) {
	cc.DisableTCP = true
	cc.DisableIPv4 = true
}

func TestReceiveChunkStorageFailure(t *testing.T) {
	seederDataDir, metainfo := testutil.GreetingTestTorrent()
	defer os.RemoveAll(seederDataDir)
	seederClientConfig := torrent.TestingConfig()
	seederClientConfig.Debug = true
	justOneNetwork(seederClientConfig)
	seederClientStorage := storage.NewMMap(seederDataDir)
	defer seederClientStorage.Close()
	seederClientConfig.DefaultStorage = seederClientStorage
	seederClientConfig.Seed = true
	seederClientConfig.Debug = true
	seederClient, err := torrent.NewClient(seederClientConfig)
	require.NoError(t, err)
	defer testutil.ExportStatusWriter(seederClient, "s")()
	leecherClientConfig := torrent.TestingConfig()
	leecherClientConfig.Debug = true
	justOneNetwork(leecherClientConfig)
	leecherClient, err := torrent.NewClient(leecherClientConfig)
	require.NoError(t, err)
	defer testutil.ExportStatusWriter(leecherClient, "l")()
	info, err := metainfo.UnmarshalInfo()
	require.NoError(t, err)
	leecherStorage := diskFullStorage{
		pieces: make([]pieceState, info.NumPieces()),
		data:   make([]byte, info.TotalLength()),
	}
	defer leecherStorage.Close()
	leecherTorrent, new, err := leecherClient.AddTorrentSpec(&torrent.TorrentSpec{
		InfoHash: metainfo.HashInfoBytes(),
		Storage:  &leecherStorage,
	})
	leecherStorage.t = leecherTorrent
	require.NoError(t, err)
	assert.True(t, new)
	seederTorrent, err := seederClient.AddTorrent(metainfo)
	require.NoError(t, err)
	// Tell the seeder to find the leecher. Is it guaranteed seeders will always try to do this?
	seederTorrent.AddClientPeer(leecherClient)
	<-leecherTorrent.GotInfo()
	assertReadAllGreeting(t, leecherTorrent.NewReader())
}

type pieceState struct {
	complete bool
}

type diskFullStorage struct {
	pieces                        []pieceState
	t                             *torrent.Torrent
	defaultHandledWriteChunkError bool
	data                          []byte

	mu          sync.Mutex
	diskNotFull bool
}

func (me *diskFullStorage) Piece(p metainfo.Piece) storage.PieceImpl {
	return pieceImpl{
		mip:             p,
		diskFullStorage: me,
	}
}

func (me diskFullStorage) Close() error {
	return nil
}

func (d diskFullStorage) OpenTorrent(info *metainfo.Info, infoHash metainfo.Hash) (storage.TorrentImpl, error) {
	return &d, nil
}

type pieceImpl struct {
	mip metainfo.Piece
	*diskFullStorage
}

func (me pieceImpl) state() *pieceState {
	return &me.diskFullStorage.pieces[me.mip.Index()]
}

func (me pieceImpl) ReadAt(p []byte, off int64) (n int, err error) {
	off += me.mip.Offset()
	return copy(p, me.data[off:]), nil
}

func (me pieceImpl) WriteAt(p []byte, off int64) (int, error) {
	off += me.mip.Offset()
	if !me.defaultHandledWriteChunkError {
		go func() {
			me.t.SetOnWriteChunkError(func(err error) {
				log.Printf("got write chunk error to custom handler: %v", err)
				me.mu.Lock()
				me.diskNotFull = true
				me.mu.Unlock()
				me.t.AllowDataDownload()
			})
			me.t.AllowDataDownload()
		}()
		me.defaultHandledWriteChunkError = true
	}
	me.mu.Lock()
	defer me.mu.Unlock()
	if me.diskNotFull {
		return copy(me.data[off:], p), nil
	}
	return copy(me.data[off:], p[:1]), errors.New("disk full")
}

func (me pieceImpl) MarkComplete() error {
	me.state().complete = true
	return nil
}

func (me pieceImpl) MarkNotComplete() error {
	panic("implement me")
}

func (me pieceImpl) Completion() storage.Completion {
	return storage.Completion{
		Complete: me.state().complete,
		Ok:       true,
	}
}
<commit_msg>Use anacrolix/log in test<commit_after>package test

import (
	"errors"
	"os"
	"sync"
	"testing"

	"github.com/anacrolix/log"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"

	"github.com/anacrolix/torrent"
	"github.com/anacrolix/torrent/internal/testutil"
	"github.com/anacrolix/torrent/metainfo"
	"github.com/anacrolix/torrent/storage"
)

func justOneNetwork(cc *torrent.ClientConfig) {
	cc.DisableTCP = true
	cc.DisableIPv4 = true
}

func TestReceiveChunkStorageFailure(t *testing.T) {
	seederDataDir, metainfo := testutil.GreetingTestTorrent()
	defer os.RemoveAll(seederDataDir)
	seederClientConfig := torrent.TestingConfig()
	seederClientConfig.Debug = true
	justOneNetwork(seederClientConfig)
	seederClientStorage := storage.NewMMap(seederDataDir)
	defer seederClientStorage.Close()
	seederClientConfig.DefaultStorage = seederClientStorage
	seederClientConfig.Seed = true
	seederClientConfig.Debug = true
	seederClient, err := torrent.NewClient(seederClientConfig)
	require.NoError(t, err)
	defer testutil.ExportStatusWriter(seederClient, "s")()
	leecherClientConfig := torrent.TestingConfig()
	leecherClientConfig.Debug = true
	justOneNetwork(leecherClientConfig)
	leecherClient, err := torrent.NewClient(leecherClientConfig)
	require.NoError(t, err)
	defer testutil.ExportStatusWriter(leecherClient, "l")()
	info, err := metainfo.UnmarshalInfo()
	require.NoError(t, err)
	leecherStorage := diskFullStorage{
		pieces: make([]pieceState, info.NumPieces()),
		data:   make([]byte, info.TotalLength()),
	}
	defer leecherStorage.Close()
	leecherTorrent, new, err := leecherClient.AddTorrentSpec(&torrent.TorrentSpec{
		InfoHash: metainfo.HashInfoBytes(),
		Storage:  &leecherStorage,
	})
	leecherStorage.t = leecherTorrent
	require.NoError(t, err)
	assert.True(t, new)
	seederTorrent, err := seederClient.AddTorrent(metainfo)
	require.NoError(t, err)
	// Tell the seeder to find the leecher. Is it guaranteed seeders will always try to do this?
	seederTorrent.AddClientPeer(leecherClient)
	<-leecherTorrent.GotInfo()
	assertReadAllGreeting(t, leecherTorrent.NewReader())
}

type pieceState struct {
	complete bool
}

type diskFullStorage struct {
	pieces                        []pieceState
	t                             *torrent.Torrent
	defaultHandledWriteChunkError bool
	data                          []byte

	mu          sync.Mutex
	diskNotFull bool
}

func (me *diskFullStorage) Piece(p metainfo.Piece) storage.PieceImpl {
	return pieceImpl{
		mip:             p,
		diskFullStorage: me,
	}
}

func (me diskFullStorage) Close() error {
	return nil
}

func (d diskFullStorage) OpenTorrent(info *metainfo.Info, infoHash metainfo.Hash) (storage.TorrentImpl, error) {
	return &d, nil
}

type pieceImpl struct {
	mip metainfo.Piece
	*diskFullStorage
}

func (me pieceImpl) state() *pieceState {
	return &me.diskFullStorage.pieces[me.mip.Index()]
}

func (me pieceImpl) ReadAt(p []byte, off int64) (n int, err error) {
	off += me.mip.Offset()
	return copy(p, me.data[off:]), nil
}

func (me pieceImpl) WriteAt(p []byte, off int64) (int, error) {
	off += me.mip.Offset()
	if !me.defaultHandledWriteChunkError {
		go func() {
			me.t.SetOnWriteChunkError(func(err error) {
				log.Printf("got write chunk error to custom handler: %v", err)
				me.mu.Lock()
				me.diskNotFull = true
				me.mu.Unlock()
				me.t.AllowDataDownload()
			})
			me.t.AllowDataDownload()
		}()
		me.defaultHandledWriteChunkError = true
	}
	me.mu.Lock()
	defer me.mu.Unlock()
	if me.diskNotFull {
		return copy(me.data[off:], p), nil
	}
	return copy(me.data[off:], p[:1]), errors.New("disk full")
}

func (me pieceImpl) MarkComplete() error {
	me.state().complete = true
	return nil
}

func (me pieceImpl) MarkNotComplete() error {
	panic("implement me")
}

func (me pieceImpl) Completion() storage.Completion {
	return storage.Completion{
		Complete: me.state().complete,
		Ok:       true,
	}
}
<|endoftext|>


<commit_before>package sysvipc

import (
	"fmt"
	"os"
	"testing"
	"reflect"
)

var test_qname string = "ipc-transit-test-queue"

func TestSendRcv(t *testing.T) {
	defer func() {
		os.Remove(defaultTransitPath + test_qname)
	}()

	// How to create this message: http://play.golang.org/p/13OSJHd5xe
	// Info about seemingly fully dynamic marshal/unmarshal: http://stackoverflow.com/questions/19482612/go-golang-array-type-inside-struct-missing-type-composite-literal
	sendMessage := map[string]interface{}{
		"Name": "Wednesday",
		"Age":  6,
		"Parents": map[string]interface{}{
			"bee": "boo",
			"foo": map[string]interface{}{
				"hi": []string{"a", "b"},
			},
		},
	}
	sendErr := Send(sendMessage, test_qname)
	if sendErr != nil {
		t.Error(sendErr)
		return
	}
	m, receiveErr := Receive(test_qname)
	if receiveErr != nil {
		t.Error(receiveErr)
		return
	}
	msg := m.(map[string]interface{})
	for k, v := range msg {
		fmt.Println(k, " -> ", reflect.TypeOf(v))
		switch vv := v.(type) {
		case string:
			if k == "Name" {
				if v != "Wednesday" {
					t.Error(receiveErr)
				}
			}
			fmt.Println(k, "is string", vv)
		case float64:
			if k == "Age" {
				if v != 6.0 {  //very strange, even though it's an int in the json, it unmarshalled as a float
					t.Error(receiveErr)
				}
			}
			fmt.Println(k, "is float64", vv)
		case map[string]interface{}:
			fmt.Println(k, "is an array:")
			for i, u := range vv {
				fmt.Println(i, u)
			}
		default:
			fmt.Println(k, "is of a type I don't know how to handle", vv)
		}
	}
}
<commit_msg>fix package name and add realistic test<commit_after>package ipc-transit_test

import (
	"fmt"
	"os"
	"testing"
	"reflect"
)

var test_qname string = "ipc-transit-test-queue"

func TestSendRcv(t *testing.T) {
	defer func() {
		os.Remove(defaultTransitPath + test_qname)
	}()

	// How to create this message: http://play.golang.org/p/13OSJHd5xe
	// Info about seemingly fully dynamic marshal/unmarshal: http://stackoverflow.com/questions/19482612/go-golang-array-type-inside-struct-missing-type-composite-literal
	sendMessage := map[string]interface{}{
		"Name": "Wednesday",
		"Age":  6,
		"Parents": map[string]interface{}{
			"bee": "boo",
			"foo": map[string]interface{}{
				"hi": []string{"a", "b"},
			},
		},
	}
	sendErr := Send(sendMessage, test_qname)
	if sendErr != nil {
		t.Error(sendErr)
		return
	}
	m, receiveErr := Receive(test_qname)
	if receiveErr != nil {
		t.Error(receiveErr)
		return
	}
	msg := m.(map[string]interface{})
	for k, v := range msg {
		fmt.Println(k, " -> ", reflect.TypeOf(v))
		switch vv := v.(type) {
		case string:
			if k == "Name" {
				if v != "Wednesday" {
					t.Error(receiveErr)
				}
			}
			fmt.Println(k, "is string", vv)
		case float64:
			if k == "Age" {
				if v != 6.0 {  //very strange, even though it's an int in the json, it unmarshalled as a float
					t.Error(receiveErr)
				}
			}
			fmt.Println(k, "is float64", vv)
		case map[string]interface{}:
			fmt.Println(k, "is an array:")
			for i, u := range vv {
				fmt.Println(i, u)
			}
		default:
			fmt.Println(k, "is of a type I don't know how to handle", vv)
		}
	}
}
<|endoftext|>


<commit_before>package templatehelper

import (
	"strings"
	"text/template"
)

// FuncMap contains all the common string helpers
var (
	FuncMap = template.FuncMap{
		"Left": func(values ...interface{}) string {
			return values[0].(string)[:values[1].(int)]
		},
		"Mid": func(values ...interface{}) string {
			if len(values) > 2 {
				return values[0].(string)[values[1].(int):values[2].(int)]
			}
			return values[0].(string)[values[1].(int):]
		},
		"Right": func(values ...interface{}) string {
			return values[0].(string)[len(values[0].(string))-values[1].(int):]
		},
		"Last": func(values ...interface{}) string {
			return values[0].([]string)[len(values[0].([]string))-1]
		},
		// strings functions
		// "Compare":      strings.Compare, // 1.5+ only
		"Contains":     strings.Contains,
		"ContainsAny":  strings.ContainsAny,
		"Count":        strings.Count,
		"EqualFold":    strings.EqualFold,
		"HasPrefix":    strings.HasPrefix,
		"HasSuffix":    strings.HasSuffix,
		"Index":        strings.Index,
		"IndexAny":     strings.IndexAny,
		"Join":         strings.Join,
		"LastIndex":    strings.LastIndex,
		"LastIndexAny": strings.LastIndexAny,
		"Repeat":       strings.Repeat,
		"Replace":      strings.Replace,
		"Split":        strings.Split,
		"SplitAfter":   strings.SplitAfter,
		"SplitAfterN":  strings.SplitAfterN,
		"SplitN":       strings.SplitN,
		"Title":        strings.Title,
		"ToLower":      strings.ToLower,
		"ToTitle":      strings.ToTitle,
		"ToUpper":      strings.ToUpper,
		"Trim":         strings.Trim,
		"TrimLeft":     strings.TrimLeft,
		"TrimPrefix":   strings.TrimPrefix,
		"TrimRight":    strings.TrimRight,
		"TrimSpace":    strings.TrimSpace,
		"TrimSuffix":   strings.TrimSuffix,
	}
)
<commit_msg>Added AGPL header to templatehelper.go<commit_after>/*
 *    Copyright (C) 2017 Christian Muehlhaeuser
 *
 *    This program is free software: you can redistribute it and/or modify
 *    it under the terms of the GNU Affero General Public License as published
 *    by the Free Software Foundation, either version 3 of the License, or
 *    (at your option) any later version.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Affero General Public License for more details.
 *
 *    You should have received a copy of the GNU Affero General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 *    Authors:
 *      Christian Muehlhaeuser <muesli@gmail.com>
 */

// Package templatehelper provides a func-map of common template functions
package templatehelper

import (
	"strings"
	"text/template"
)

// FuncMap contains all the common string helpers
var (
	FuncMap = template.FuncMap{
		"Left": func(values ...interface{}) string {
			return values[0].(string)[:values[1].(int)]
		},
		"Mid": func(values ...interface{}) string {
			if len(values) > 2 {
				return values[0].(string)[values[1].(int):values[2].(int)]
			}
			return values[0].(string)[values[1].(int):]
		},
		"Right": func(values ...interface{}) string {
			return values[0].(string)[len(values[0].(string))-values[1].(int):]
		},
		"Last": func(values ...interface{}) string {
			return values[0].([]string)[len(values[0].([]string))-1]
		},
		// strings functions
		// "Compare":      strings.Compare, // 1.5+ only
		"Contains":     strings.Contains,
		"ContainsAny":  strings.ContainsAny,
		"Count":        strings.Count,
		"EqualFold":    strings.EqualFold,
		"HasPrefix":    strings.HasPrefix,
		"HasSuffix":    strings.HasSuffix,
		"Index":        strings.Index,
		"IndexAny":     strings.IndexAny,
		"Join":         strings.Join,
		"LastIndex":    strings.LastIndex,
		"LastIndexAny": strings.LastIndexAny,
		"Repeat":       strings.Repeat,
		"Replace":      strings.Replace,
		"Split":        strings.Split,
		"SplitAfter":   strings.SplitAfter,
		"SplitAfterN":  strings.SplitAfterN,
		"SplitN":       strings.SplitN,
		"Title":        strings.Title,
		"ToLower":      strings.ToLower,
		"ToTitle":      strings.ToTitle,
		"ToUpper":      strings.ToUpper,
		"Trim":         strings.Trim,
		"TrimLeft":     strings.TrimLeft,
		"TrimPrefix":   strings.TrimPrefix,
		"TrimRight":    strings.TrimRight,
		"TrimSpace":    strings.TrimSpace,
		"TrimSuffix":   strings.TrimSuffix,
	}
)
<|endoftext|>


<commit_before>package rwvfs

import (
	"code.google.com/p/go.tools/godoc/vfs"
	"fmt"
	"os"
	pathpkg "path"
	"path/filepath"
)

// OS returns an implementation of FileSystem reading from the tree rooted at
// root.
func OS(root string) FileSystem {
	return osFS{root, vfs.OS(root)}
}

type osFS struct {
	root string
	vfs.FileSystem
}

// resolve is from code.google.com/p/go.tools/godoc/vfs.
func (fs osFS) resolve(path string) string {
	// Clean the path so that it cannot possibly begin with ../.
	// If it did, the result of filepath.Join would be outside the
	// tree rooted at root.  We probably won't ever see a path
	// with .. in it, but be safe anyway.
	path = pathpkg.Clean("/" + path)

	return filepath.Join(string(fs.root), path)
}

// WriterOpen opens the file at path for writing, creating the file if it
// doesn't exist and truncating it otherwise.
func (fs osFS) OpenFile(path string, flag int) (ReadWriteSeekCloser, error) {
	f, err := os.OpenFile(fs.resolve(path), flag, 0644)
	if err != nil {
		return nil, err
	}

	fi, err := f.Stat()
	if err != nil {
		return nil, err
	}
	if fi.IsDir() {
		return nil, fmt.Errorf("Open: %s is a directory", path)
	}

	return f, nil
}
<commit_msg>Fix doc<commit_after>package rwvfs

import (
	"code.google.com/p/go.tools/godoc/vfs"
	"fmt"
	"os"
	pathpkg "path"
	"path/filepath"
)

// OS returns an implementation of FileSystem reading from the tree rooted at
// root.
func OS(root string) FileSystem {
	return osFS{root, vfs.OS(root)}
}

type osFS struct {
	root string
	vfs.FileSystem
}

// resolve is from code.google.com/p/go.tools/godoc/vfs.
func (fs osFS) resolve(path string) string {
	// Clean the path so that it cannot possibly begin with ../.
	// If it did, the result of filepath.Join would be outside the
	// tree rooted at root.  We probably won't ever see a path
	// with .. in it, but be safe anyway.
	path = pathpkg.Clean("/" + path)

	return filepath.Join(string(fs.root), path)
}

// OpenFile opens the file at path for writing, creating the file if it doesn't
// exist and truncating it otherwise.
func (fs osFS) OpenFile(path string, flag int) (ReadWriteSeekCloser, error) {
	f, err := os.OpenFile(fs.resolve(path), flag, 0644)
	if err != nil {
		return nil, err
	}

	fi, err := f.Stat()
	if err != nil {
		return nil, err
	}
	if fi.IsDir() {
		return nil, fmt.Errorf("Open: %s is a directory", path)
	}

	return f, nil
}
<|endoftext|>


<commit_before>package termui

type WrapPanel struct {
	BaseElement

	width, height int
	orientation   Orientation
	children      []Element
	childpos      map[Element]rect
}

type rect struct {
	x, y, width, height int
}

var _ Element = new(WrapPanel)

func NewWrapPanel(o Orientation) *WrapPanel {
	return &WrapPanel{
		orientation: o,
		childpos:    make(map[Element]rect),
	}
}

// Name returns the constant name of the WrapPanel for css styling.
func (v *WrapPanel) Name() string {
	return "wrappanel"
}

// Children returns all nested elements.
func (v *WrapPanel) Children() []Element {
	return v.children
}

// AddChild adds the given children to the WrapPanel
func (v *WrapPanel) AddChild(e ...Element) {
	v.children = append(v.children, e...)
	for _, c := range e {
		c.SetParent(v)
	}
}

func (v *WrapPanel) measureVertical(availableWidth, availableHeight int, arrange bool) (width int, height int) {
	cIdx := 0
	width = 0
	height = 0
	for cIdx < len(v.children) {
		colWidth := 0
		colHeight := 0
		cStart := cIdx
		cEnd := cIdx

		for i := cIdx; i < len(v.children); i++ {
			cw, ch := v.children[i].Measure(0, 0)

			if colHeight+ch > availableHeight {
				if colHeight > height {
					height = colHeight
				}
				width += colWidth
				break
			}

			if arrange {
				v.childpos[v.children[i]] = rect{
					x:      width,
					width:  cw,
					height: ch,
				}
			}

			if cw > colWidth {
				colWidth = cw
			}
			colHeight += ch
			cEnd = cIdx
			cIdx++
		}

		if arrange {
			y := 0
			for i := cStart; i <= cEnd; i++ {
				child := v.children[i]
				rect := v.childpos[child]
				child.Arrange(rect.width, rect.height)
				rect.y = y
				y += rect.height
				v.childpos[child] = rect
			}
		}
	}
	return
}

func (v *WrapPanel) measureHorizontal(availableWidth, availableHeight int, arrange bool) (width int, height int) {
	cIdx := 0
	width = 0
	height = 0
	for cIdx < len(v.children) {
		colWidth := 0
		colHeight := 0
		cStart := cIdx
		cEnd := cIdx

		for i := cIdx; i < len(v.children); i++ {
			cw, ch := v.children[i].Measure(0, 0)

			if colWidth+cw > availableWidth {
				if colWidth > width {
					width = colWidth
				}
				height += colHeight
				break
			}

			if arrange {
				v.childpos[v.children[i]] = rect{
					y:      height,
					width:  cw,
					height: ch,
				}
			}

			if ch > colHeight {
				colHeight = ch
			}
			colWidth += cw
			cEnd = cIdx
			cIdx++
		}

		if arrange {
			x := 0
			for i := cStart; i <= cEnd; i++ {
				child := v.children[i]
				rect := v.childpos[child]
				child.Arrange(rect.width, rect.height)
				rect.x = x
				x += rect.width
				v.childpos[child] = rect
			}
		}
	}
	return
}

// Measure gets the "wanted" size of the element based on the available size
func (v *WrapPanel) Measure(availableWidth, availableHeight int) (width int, height int) {
	if v.orientation == Vertical {
		return v.measureVertical(availableWidth, availableHeight, false)
	} else {
		return v.measureHorizontal(availableWidth, availableHeight, false)
	}
}

// Arrange sets the final size for the Element end tells it to Arrange itself
func (v *WrapPanel) Arrange(finalWidth, finalHeight int) {
	v.width, v.height = finalWidth, finalHeight
	if v.orientation == Vertical {
		v.measureVertical(finalWidth, finalHeight, true)
	} else {
		v.measureHorizontal(finalWidth, finalHeight, true)
	}
}

// Render renders the element on the given Renderer
func (v *WrapPanel) Render(rn Renderer) {
	for el, pos := range v.childpos {
		rn.RenderChild(el, pos.width, pos.height, pos.x, pos.y)
	}
}

// Width returns the width of the WrapPanel
func (v *WrapPanel) Width() int {
	return v.width
}

// Height returns the height of the WrapPanel.
func (v *WrapPanel) Height() int {
	return v.height
}
<commit_msg>fixed measure of wrappanel and added clear func.<commit_after>package termui

type WrapPanel struct {
	BaseElement

	width, height int
	orientation   Orientation
	children      []Element
	childpos      map[Element]rect
}

type rect struct {
	x, y, width, height int
}

var _ Element = new(WrapPanel)

func NewWrapPanel(o Orientation) *WrapPanel {
	return &WrapPanel{
		orientation: o,
		childpos:    make(map[Element]rect),
	}
}

// Name returns the constant name of the WrapPanel for css styling.
func (v *WrapPanel) Name() string {
	return "wrappanel"
}

// Children returns all nested elements.
func (v *WrapPanel) Children() []Element {
	return v.children
}

// AddChild adds the given children to the WrapPanel
func (v *WrapPanel) AddChild(e ...Element) {
	v.children = append(v.children, e...)
	for _, c := range e {
		c.SetParent(v)
	}
}

// Removes all children.
func (v *WrapPanel) Clear() {
	for _, c := range v.children {
		c.SetParent(nil)
	}
	v.childpos = make(map[Element]rect)
	v.children = nil
}

func (v *WrapPanel) measureVertical(availableWidth, availableHeight int, arrange bool) (width int, height int) {
	cIdx := 0
	width = 0
	height = 0
	for cIdx < len(v.children) {
		colWidth := 0
		colHeight := 0
		cStart := cIdx
		cEnd := cIdx

		for i := cIdx; i < len(v.children); i++ {
			cw, ch := v.children[i].Measure(0, 0)

			if colHeight+ch > availableHeight && availableHeight != 0 {
				if colHeight > height {
					height = colHeight
				}
				width += colWidth
				break
			}

			if arrange {
				v.childpos[v.children[i]] = rect{
					x:      width,
					width:  cw,
					height: ch,
				}
			}

			if cw > colWidth {
				colWidth = cw
			}
			colHeight += ch
			cEnd = cIdx
			cIdx++
		}

		if arrange {
			y := 0
			for i := cStart; i <= cEnd; i++ {
				child := v.children[i]
				rect := v.childpos[child]
				child.Arrange(rect.width, rect.height)
				rect.y = y
				y += rect.height
				v.childpos[child] = rect
			}
		}
	}
	return
}

func (v *WrapPanel) measureHorizontal(availableWidth, availableHeight int, arrange bool) (width int, height int) {
	cIdx := 0
	width = 0
	height = 0
	for cIdx < len(v.children) {
		colWidth := 0
		colHeight := 0
		cStart := cIdx
		cEnd := cIdx

		for i := cIdx; i < len(v.children); i++ {
			cw, ch := v.children[i].Measure(0, 0)

			if colWidth+cw > availableWidth && availableWidth != 0 {
				if colWidth > width {
					width = colWidth
				}
				height += colHeight
				break
			}

			if arrange {
				v.childpos[v.children[i]] = rect{
					y:      height,
					width:  cw,
					height: ch,
				}
			}

			if ch > colHeight {
				colHeight = ch
			}
			colWidth += cw
			cEnd = cIdx
			cIdx++
		}

		if arrange {
			x := 0
			for i := cStart; i <= cEnd; i++ {
				child := v.children[i]
				rect := v.childpos[child]
				child.Arrange(rect.width, rect.height)
				rect.x = x
				x += rect.width
				v.childpos[child] = rect
			}
		}
	}
	return
}

// Measure gets the "wanted" size of the element based on the available size
func (v *WrapPanel) Measure(availableWidth, availableHeight int) (width int, height int) {
	if v.orientation == Vertical {
		return v.measureVertical(availableWidth, availableHeight, false)
	} else {
		return v.measureHorizontal(availableWidth, availableHeight, false)
	}
}

// Arrange sets the final size for the Element end tells it to Arrange itself
func (v *WrapPanel) Arrange(finalWidth, finalHeight int) {
	v.width, v.height = finalWidth, finalHeight
	if v.orientation == Vertical {
		v.measureVertical(finalWidth, finalHeight, true)
	} else {
		v.measureHorizontal(finalWidth, finalHeight, true)
	}
}

// Render renders the element on the given Renderer
func (v *WrapPanel) Render(rn Renderer) {
	for el, pos := range v.childpos {
		rn.RenderChild(el, pos.width, pos.height, pos.x, pos.y)
	}
}

// Width returns the width of the WrapPanel
func (v *WrapPanel) Width() int {
	return v.width
}

// Height returns the height of the WrapPanel.
func (v *WrapPanel) Height() int {
	return v.height
}
<|endoftext|>


<commit_before>package skip_list

import "math/rand"

type SkipList struct {
	head *element
}

type element struct {
	data Data
	next []*element
}

type Data struct {
	Key   string
	Value int
}

const p = 0.25

func NewSkipList() *SkipList {
	return &SkipList{&element{}}
}

func (l *SkipList) Get(key string) (value int, ok bool) {
	e := l.head
	layer := len(l.head.next) - 1
	for layer >= 0 {
		n := e.next[layer]
		if n == nil || n.data.Key > key {
			// Search the next lower layer.
			layer--
			continue
		}
		if n.data.Key == key {
			return n.data.Value, true
		}
		// Continue searching the current layer.
		e = n
	}
	return 0, false
}

func (l *SkipList) Set(key string, value int) {
	var prevs []*element
	e := l.head
	layer := len(l.head.next) - 1
	for layer >= 0 {
		n := e.next[layer]
		if n == nil || n.data.Key > key {
			// Keep a reference to the previous node; will be useful if it ends with an insertion.
			prevs = append(prevs, e)
			// Search the next lower layer.
			layer--
			continue
		}
		if n.data.Key == key {
			n.data.Value = value
			return
		}
		// Continue searching the current layer.
		e = n
	}
	newElement := &element{Data{key, value}, nil}
	insert(l.head, prevs, newElement)
}

func insert(head *element, prevs []*element, e *element) {
	for layer := 0; layer == 0 || coinflip(); layer++ {
		var prev *element
		if layer < len(prevs) {
			// The element will be inserted in a preexisting layer.
			// prevs was built in reverse order.
			prev = prevs[len(prevs)-1-layer]
		} else {
			// The element will be inserted in a new empty layer.
			head.next = append(head.next, nil)
			prev = head
		}
		e.next = append(e.next, prev.next[layer])
		prev.next[layer] = e
	}
}

func (l *SkipList) Del(key string) {
	e := l.head
	layer := len(l.head.next) - 1
	for layer >= 0 {
		n := e.next[layer]
		if n == nil || n.data.Key > key {
			// Search the next lower layer.
			layer--
			continue
		}
		if n.data.Key == key {
			e.next[layer] = n.next[layer]
			if l.head.next[layer] == nil {
				// Delete empty layer.
				l.head.next = l.head.next[:len(l.head.next)-1]
			}
			// Continue deletion at the next lower layer.
			layer--
			continue
		}
		// Continue searching the current layer.
		e = n
	}
}

func (l *SkipList) All() []Data {
	a := []Data{}
	if len(l.head.next) < 1 {
		return a
	}
	for e := l.head.next[0]; e != nil; e = e.next[0] {
		a = append(a, e.data)
	}
	return a
}

func coinflip() bool {
	return rand.Float32() < p
}
<commit_msg>[skip_list/go] Reword comments<commit_after>package skip_list

import "math/rand"

type SkipList struct {
	head *element
}

type element struct {
	data Data
	next []*element
}

type Data struct {
	Key   string
	Value int
}

const p = 0.25

func NewSkipList() *SkipList {
	return &SkipList{&element{}}
}

func (l *SkipList) Get(key string) (value int, ok bool) {
	e := l.head
	layer := len(l.head.next) - 1
	for layer >= 0 {
		n := e.next[layer]
		if n == nil || n.data.Key > key {
			// Search the next lower layer.
			layer--
			continue
		}
		if n.data.Key == key {
			return n.data.Value, true
		}
		// Keep searching the current layer.
		e = n
	}
	return 0, false
}

func (l *SkipList) Set(key string, value int) {
	var prevs []*element
	e := l.head
	layer := len(l.head.next) - 1
	for layer >= 0 {
		n := e.next[layer]
		if n == nil || n.data.Key > key {
			// Keep a reference to the previous node; will be useful if it ends with an insertion.
			prevs = append(prevs, e)
			// Search the next lower layer.
			layer--
			continue
		}
		if n.data.Key == key {
			n.data.Value = value
			return
		}
		// Keep searching the current layer.
		e = n
	}
	newElement := &element{Data{key, value}, nil}
	insert(l.head, prevs, newElement)
}

func insert(head *element, prevs []*element, e *element) {
	for layer := 0; layer == 0 || coinflip(); layer++ {
		var prev *element
		if layer < len(prevs) {
			// The element will be inserted in a preexisting layer.
			// prevs was built in reverse order.
			prev = prevs[len(prevs)-1-layer]
		} else {
			// The element will be inserted in a new empty layer.
			head.next = append(head.next, nil)
			prev = head
		}
		e.next = append(e.next, prev.next[layer])
		prev.next[layer] = e
	}
}

func (l *SkipList) Del(key string) {
	e := l.head
	layer := len(l.head.next) - 1
	for layer >= 0 {
		n := e.next[layer]
		if n == nil || n.data.Key > key {
			// Search the next lower layer.
			layer--
			continue
		}
		if n.data.Key == key {
			e.next[layer] = n.next[layer]
			if l.head.next[layer] == nil {
				// Delete empty layer.
				l.head.next = l.head.next[:len(l.head.next)-1]
			}
			// Continue deletion at the next lower layer.
			layer--
			continue
		}
		// Keep searching the current layer.
		e = n
	}
}

func (l *SkipList) All() []Data {
	a := []Data{}
	if len(l.head.next) < 1 {
		return a
	}
	for e := l.head.next[0]; e != nil; e = e.next[0] {
		a = append(a, e.data)
	}
	return a
}

func coinflip() bool {
	return rand.Float32() < p
}
<|endoftext|>


<commit_before>package xlogutils
import (
	"fmt"
	"errors"
)

const XLOG_SEG_SIZE_16MB = 16 * 1024 * 1024
const XLOG_SEG_SIZE_DEFAULT = XLOG_SEG_SIZE_16MB

var xlog_seg_size uint = XLOG_SEG_SIZE_DEFAULT
var XLogSegmentsPerXLogId uint

var ParseError = errors.New("Parse error")

func init() {
	SetXLogSegSize(XLOG_SEG_SIZE_16MB)
}

func XLogSegSize() (uint) {
	return xlog_seg_size
}

func SetXLogSegSize(size uint) {
	xlog_seg_size = size
	XLogSegmentsPerXLogId = 0x100000000 / xlog_seg_size
}

type Location struct {
	XLogId uint
	Offset uint
}

type Filename struct {
	Timeline uint
	XLogId uint
	Segment uint
	IsHistory bool
}

func FilenameFromString(fname string) (*Filename, error) {
	ret := Filename{}
	if r, err := fmt.Sscanf(fname, "%08X%08X%08X", &ret.Timeline, &ret.XLogId, &ret.Segment); r != 3 || err != nil {
		if r, err := fmt.Sscan("%08X.history", &ret.Timeline); r == 1 && err == nil {
			ret.Segment = 0
			ret.XLogId = 0
			ret.IsHistory = true
			return &ret, nil
		} else {
			return nil, ParseError
		}
	}
	return &ret, nil
}

func LocationFromString(loc string) (*Location, error) {
	ret := Location{}
	if r, err := fmt.Sscanf(loc, "%X/%X", &ret.XLogId, &ret.Offset); r != 2 || err != nil {
		return nil, ParseError
	}

	return &ret, nil
}

func (this *Location) Filename(tli uint) *Filename {
	ret := Filename{}
	ret.Timeline = tli
	ret.XLogId = this.XLogId
	ret.Segment = this.Offset / xlog_seg_size
	return &ret
}

func (this *Location) Int() uint64 {
	return uint64(this.XLogId) * 0x100000000 + uint64(this.Offset)
}

func (this *Location) Diff(other *Location) uint64 {
	if this.Int() > other.Int() {
		return this.Int() - other.Int()
	} else {
		return other.Int() - this.Int()
	}
}

func (this *Filename) String() string {
	if (!this.IsHistory) {
		return fmt.Sprintf("%08x%08X%08X", this.Timeline, this.XLogId, this.Segment)
	} else {
		return fmt.Sprintf("%08X.history", this.Timeline)
	}
}

func (this *Filename) Next() *Filename {
	ret := Filename{}
	ret.Timeline = this.Timeline
	ret.XLogId = this.XLogId
	ret.Segment = this.Segment
	ret.Segment++
	if ret.Segment == XLogSegmentsPerXLogId {
		ret.Segment = 0
		ret.XLogId++
	}
	return &ret
}

func (this *Filename) Prev() *Filename {
	ret := Filename{}
	ret.Timeline = this.Timeline
	ret.XLogId = this.XLogId
	ret.Segment = this.Segment
	if ret.Segment == 0 {
		ret.Segment = XLogSegmentsPerXLogId
		ret.XLogId--
	}
	ret.Segment--
	return &ret
}<commit_msg>Added File.Location()<commit_after>package xlogutils
import (
	"fmt"
	"errors"
)

const XLOG_SEG_SIZE_16MB = 16 * 1024 * 1024
const XLOG_SEG_SIZE_DEFAULT = XLOG_SEG_SIZE_16MB

var xlog_seg_size uint = XLOG_SEG_SIZE_DEFAULT
var XLogSegmentsPerXLogId uint

var ParseError = errors.New("Parse error")

func init() {
	SetXLogSegSize(XLOG_SEG_SIZE_16MB)
}

func XLogSegSize() (uint) {
	return xlog_seg_size
}

func SetXLogSegSize(size uint) {
	xlog_seg_size = size
	XLogSegmentsPerXLogId = 0x100000000 / xlog_seg_size
}

type Location struct {
	XLogId uint
	Offset uint
}

type Filename struct {
	Timeline uint
	XLogId uint
	Segment uint
	IsHistory bool
}

func FilenameFromString(fname string) (*Filename, error) {
	ret := Filename{}
	if r, err := fmt.Sscanf(fname, "%08X%08X%08X", &ret.Timeline, &ret.XLogId, &ret.Segment); r != 3 || err != nil {
		if r, err := fmt.Sscan("%08X.history", &ret.Timeline); r == 1 && err == nil {
			ret.Segment = 0
			ret.XLogId = 0
			ret.IsHistory = true
			return &ret, nil
		} else {
			return nil, ParseError
		}
	}
	return &ret, nil
}

func LocationFromString(loc string) (*Location, error) {
	ret := Location{}
	if r, err := fmt.Sscanf(loc, "%X/%X", &ret.XLogId, &ret.Offset); r != 2 || err != nil {
		return nil, ParseError
	}

	return &ret, nil
}

func (this *Location) Filename(tli uint) *Filename {
	ret := Filename{}
	ret.Timeline = tli
	ret.XLogId = this.XLogId
	ret.Segment = this.Offset / xlog_seg_size
	return &ret
}

func (this *Location) Int() uint64 {
	return uint64(this.XLogId) * 0x100000000 + uint64(this.Offset)
}

func (this *Location) Diff(other *Location) uint64 {
	if this.Int() > other.Int() {
		return this.Int() - other.Int()
	} else {
		return other.Int() - this.Int()
	}
}

func (this *Filename) String() string {
	if (!this.IsHistory) {
		return fmt.Sprintf("%08x%08X%08X", this.Timeline, this.XLogId, this.Segment)
	} else {
		return fmt.Sprintf("%08X.history", this.Timeline)
	}
}

func (this *Filename) Location() *Location {
	ret := Location{}
	ret.XLogId = this.XLogId
	ret.Offset = this.Segment * xlog_seg_size
	return &ret
}

func (this *Filename) Next() *Filename {
	ret := Filename{}
	ret.Timeline = this.Timeline
	ret.XLogId = this.XLogId
	ret.Segment = this.Segment
	ret.Segment++
	if ret.Segment == XLogSegmentsPerXLogId {
		ret.Segment = 0
		ret.XLogId++
	}
	return &ret
}

func (this *Filename) Prev() *Filename {
	ret := Filename{}
	ret.Timeline = this.Timeline
	ret.XLogId = this.XLogId
	ret.Segment = this.Segment
	if ret.Segment == 0 {
		ret.Segment = XLogSegmentsPerXLogId
		ret.XLogId--
	}
	ret.Segment--
	return &ret
}<|endoftext|>