<commit_before>package observability

import (
	"context"
	"fmt"
	"net/http"
	"strconv"

	"github.com/go-gorp/gorp"
	"github.com/gorilla/mux"
	"go.opencensus.io/trace"

	"github.com/ovh/cds/engine/api/cache"
	"github.com/ovh/cds/sdk/log"
	"github.com/ovh/cds/sdk/tracingutils"
)

// Tags contants
const (
	TagWorkflow           = "workflow"
	TagWorkflowRun        = "workflow_run"
	TagProjectKey         = "project_key"
	TagWorkflowNodeRun    = "workflow_node_run"
	TagWorkflowNodeJobRun = "workflow_node_job_run"
	TagJob                = "job"
	TagWorkflowNode       = "workflow_node"
	TagPipelineID         = "pipeline_id"
	TagPipelineDeep       = "pipeline_deep"
	TagWorker             = "worker"
)

// LinkTo a traceID
func LinkTo(ctx context.Context, traceID [16]byte) {
	s := Current(ctx)
	if s == nil {
		return
	}

	s.AddLink(
		trace.Link{
			TraceID: trace.TraceID(traceID),
		},
	)
}

// Current return the current span
func Current(ctx context.Context, tags ...trace.Attribute) *trace.Span {
	if ctx == nil {
		return nil
	}
	span := trace.FromContext(ctx)
	if span == nil {
		return nil
	}
	if len(tags) > 0 {
		span.AddAttributes(tags...)
	}
	return span
}

// Tag is helper function to instanciate trace.Attribute
func Tag(key string, value interface{}) trace.Attribute {
	return trace.StringAttribute(key, fmt.Sprintf("%v", value))
}

// Span start a new span from the parent context
func Span(ctx context.Context, name string, tags ...trace.Attribute) (context.Context, func()) {
	if ctx == nil {
		return nil, func() {}
	}
	var span *trace.Span
	ctx, span = trace.StartSpan(ctx, name)
	if len(tags) > 0 {
		span.AddAttributes(tags...)
	}
	ctx = tracingutils.SpanContextToContext(ctx, span.SpanContext())
	return ctx, span.End
}

func findPrimaryKeyFromRequest(req *http.Request, db gorp.SqlExecutor, store cache.Store) (string, bool) {
	vars := mux.Vars(req)
	pkey := vars["key"]
	if pkey == "" {
		pkey = vars["permProjectKey"]
	}

	if pkey == "" {
		id, _ := strconv.ParseInt(vars["id"], 10, 64)
		//The ID found may be a node run job, let's try to find the project key behing
		if id <= 0 {
			id, _ = strconv.ParseInt(vars["permID"], 10, 64)
		}
		if id != 0 {
			var err error
			cacheKey := cache.Key("api:FindProjetKeyForNodeRunJob:", fmt.Sprintf("%v", id))
			if !store.Get(cacheKey, &pkey) {
				pkey, err = findProjetKeyForNodeRunJob(db, id)
				if err != nil {
					log.Error("tracingMiddleware> %v", err)
					return "", false
				}
				store.SetWithTTL(cacheKey, pkey, 60*15)
			}
		}
	}

	return pkey, pkey != ""
}
<commit_msg>fix(hatchery): ctxhttp nil context (#3182)<commit_after>package observability

import (
	"context"
	"fmt"
	"net/http"
	"strconv"

	"github.com/go-gorp/gorp"
	"github.com/gorilla/mux"
	"go.opencensus.io/trace"

	"github.com/ovh/cds/engine/api/cache"
	"github.com/ovh/cds/sdk/log"
	"github.com/ovh/cds/sdk/tracingutils"
)

// Tags contants
const (
	TagWorkflow           = "workflow"
	TagWorkflowRun        = "workflow_run"
	TagProjectKey         = "project_key"
	TagWorkflowNodeRun    = "workflow_node_run"
	TagWorkflowNodeJobRun = "workflow_node_job_run"
	TagJob                = "job"
	TagWorkflowNode       = "workflow_node"
	TagPipelineID         = "pipeline_id"
	TagPipelineDeep       = "pipeline_deep"
	TagWorker             = "worker"
)

// LinkTo a traceID
func LinkTo(ctx context.Context, traceID [16]byte) {
	s := Current(ctx)
	if s == nil {
		return
	}

	s.AddLink(
		trace.Link{
			TraceID: trace.TraceID(traceID),
		},
	)
}

// Current return the current span
func Current(ctx context.Context, tags ...trace.Attribute) *trace.Span {
	if ctx == nil {
		return nil
	}
	span := trace.FromContext(ctx)
	if span == nil {
		return nil
	}
	if len(tags) > 0 {
		span.AddAttributes(tags...)
	}
	return span
}

// Tag is helper function to instanciate trace.Attribute
func Tag(key string, value interface{}) trace.Attribute {
	return trace.StringAttribute(key, fmt.Sprintf("%v", value))
}

// Span start a new span from the parent context
func Span(ctx context.Context, name string, tags ...trace.Attribute) (context.Context, func()) {
	if ctx == nil {
		return context.Background(), func() {}
	}
	var span *trace.Span
	ctx, span = trace.StartSpan(ctx, name)
	if len(tags) > 0 {
		span.AddAttributes(tags...)
	}
	ctx = tracingutils.SpanContextToContext(ctx, span.SpanContext())
	return ctx, span.End
}

func findPrimaryKeyFromRequest(req *http.Request, db gorp.SqlExecutor, store cache.Store) (string, bool) {
	vars := mux.Vars(req)
	pkey := vars["key"]
	if pkey == "" {
		pkey = vars["permProjectKey"]
	}

	if pkey == "" {
		id, _ := strconv.ParseInt(vars["id"], 10, 64)
		//The ID found may be a node run job, let's try to find the project key behing
		if id <= 0 {
			id, _ = strconv.ParseInt(vars["permID"], 10, 64)
		}
		if id != 0 {
			var err error
			cacheKey := cache.Key("api:FindProjetKeyForNodeRunJob:", fmt.Sprintf("%v", id))
			if !store.Get(cacheKey, &pkey) {
				pkey, err = findProjetKeyForNodeRunJob(db, id)
				if err != nil {
					log.Error("tracingMiddleware> %v", err)
					return "", false
				}
				store.SetWithTTL(cacheKey, pkey, 60*15)
			}
		}
	}

	return pkey, pkey != ""
}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build freebsd openbsd netbsd darwin

/*
Package fsnotify implements filesystem notification.

Example:
    watcher, err := fsnotify.NewWatcher()
    if err != nil {
        log.Fatal(err)
    }
    err = watcher.Watch("/tmp")
    if err != nil {
        log.Fatal(err)
    }
    for {
        select {
        case ev := <-watcher.Event:
            log.Println("event:", ev)
        case err := <-watcher.Error:
            log.Println("error:", err)
        }
    }

*/
package fsnotify

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"syscall"
)

type FileEvent struct {
	mask   uint32 // Mask of events
	Name   string // File name (optional)
	create bool   // set by fsnotify package if found new file
}

// IsCreate reports whether the FileEvent was triggerd by a creation
func (e *FileEvent) IsCreate() bool { return e.create }

// IsDelete reports whether the FileEvent was triggerd by a delete
func (e *FileEvent) IsDelete() bool { return (e.mask & NOTE_DELETE) == NOTE_DELETE }

// IsModify reports whether the FileEvent was triggerd by a file modification
func (e *FileEvent) IsModify() bool { return ((e.mask & NOTE_WRITE) == NOTE_WRITE || (e.mask & NOTE_ATTRIB) == NOTE_ATTRIB) }

// IsRename reports whether the FileEvent was triggerd by a change name
func (e *FileEvent) IsRename() bool { return (e.mask & NOTE_RENAME) == NOTE_RENAME }

type Watcher struct {
	kq       int                 // File descriptor (as returned by the kqueue() syscall)
	watches  map[string]int      // Map of watched file diescriptors (key: path)
	paths    map[int]string      // Map of watched paths (key: watch descriptor)
	finfo    map[int]os.FileInfo // Map of file information (isDir, isReg; key: watch descriptor)
	Error    chan error          // Errors are sent on this channel
	Event    chan *FileEvent     // Events are returned on this channel
	done     chan bool           // Channel for sending a "quit message" to the reader goroutine
	isClosed bool                // Set to true when Close() is first called
	kbuf     [1]syscall.Kevent_t // An event buffer for Add/Remove watch
}

// NewWatcher creates and returns a new kevent instance using kqueue(2)
func NewWatcher() (*Watcher, error) {
	fd, errno := syscall.Kqueue()
	if fd == -1 {
		return nil, os.NewSyscallError("kqueue", errno)
	}
	w := &Watcher{
		kq:      fd,
		watches: make(map[string]int),
		paths:   make(map[int]string),
		finfo:   make(map[int]os.FileInfo),
		Event:   make(chan *FileEvent),
		Error:   make(chan error),
		done:    make(chan bool, 1),
	}

	go w.readEvents()
	return w, nil
}

// Close closes a kevent watcher instance
// It sends a message to the reader goroutine to quit and removes all watches
// associated with the kevent instance
func (w *Watcher) Close() error {
	if w.isClosed {
		return nil
	}
	w.isClosed = true

	// Send "quit" message to the reader goroutine
	w.done <- true
	for path := range w.watches {
		w.RemoveWatch(path)
	}

	return nil
}

// AddWatch adds path to the watched file set.
// The flags are interpreted as described in kevent(2).
func (w *Watcher) addWatch(path string, flags uint32) error {
	if w.isClosed {
		return errors.New("kevent instance already closed")
	}

	watchEntry := &w.kbuf[0]
	watchEntry.Fflags = flags

	watchfd, found := w.watches[path]
	if !found {
		fd, errno := syscall.Open(path, syscall.O_NONBLOCK|syscall.O_RDONLY, 0700)
		if fd == -1 {
			return errno
		}
		watchfd = fd

		w.watches[path] = watchfd
		w.paths[watchfd] = path

		fi, _ := os.Stat(path)
		w.finfo[watchfd] = fi
		if fi.IsDir() {
			w.watchDirectoryFiles(path)
		}
	}
	syscall.SetKevent(watchEntry, watchfd, syscall.EVFILT_VNODE, syscall.EV_ADD|syscall.EV_CLEAR)

	wd, errno := syscall.Kevent(w.kq, w.kbuf[:], nil, nil)
	if wd == -1 {
		return errno
	} else if (watchEntry.Flags & syscall.EV_ERROR) == syscall.EV_ERROR {
		return errors.New("kevent add error")
	}

	return nil
}

// Watch adds path to the watched file set, watching all events.
func (w *Watcher) Watch(path string) error {
	return w.addWatch(path, NOTE_ALLEVENTS)
}

// RemoveWatch removes path from the watched file set.
func (w *Watcher) RemoveWatch(path string) error {
	watchfd, ok := w.watches[path]
	if !ok {
		return errors.New(fmt.Sprintf("can't remove non-existent kevent watch for: %s", path))
	}
	syscall.Close(watchfd)
	watchEntry := &w.kbuf[0]
	syscall.SetKevent(watchEntry, w.watches[path], syscall.EVFILT_VNODE, syscall.EV_DELETE)
	success, errno := syscall.Kevent(w.kq, w.kbuf[:], nil, nil)
	if success == -1 {
		return os.NewSyscallError("kevent_rm_watch", errno)
	} else if (watchEntry.Flags & syscall.EV_ERROR) == syscall.EV_ERROR {
		return errors.New("kevent rm error")
	}
	delete(w.watches, path)
	return nil
}

// readEvents reads from the kqueue file descriptor, converts the
// received events into Event objects and sends them via the Event channel
func (w *Watcher) readEvents() {
	var (
		eventbuf [10]syscall.Kevent_t // Event buffer
		events   []syscall.Kevent_t   // Received events
		twait    *syscall.Timespec    // Time to block waiting for events
		n        int                  // Number of events returned from kevent
		errno    error                // Syscall errno
	)
	events = eventbuf[0:0]
	twait = new(syscall.Timespec)
	*twait = syscall.NsecToTimespec(keventWaitTime)

	for {
		if len(events) == 0 {
			n, errno = syscall.Kevent(w.kq, nil, eventbuf[:], twait)
			events = eventbuf[0:n]
		}
		// See if there is a message on the "done" channel
		var done bool
		select {
		case done = <-w.done:
		default:
		}

		// If "done" message is received
		if done {
			errno := syscall.Close(w.kq)
			if errno != nil {
				w.Error <- os.NewSyscallError("close", errno)
			}
			close(w.Event)
			close(w.Error)
			return
		}
		if n < 0 {
			w.Error <- os.NewSyscallError("kevent", errno)
			continue
		}

		// Timeout, no big deal
		if n == 0 {
			continue
		}

		// Flush the events we recieved to the events channel
		for len(events) > 0 {
			fileEvent := new(FileEvent)
			watchEvent := &events[0]
			fileEvent.mask = uint32(watchEvent.Fflags)
			fileEvent.Name = w.paths[int(watchEvent.Ident)]

			fileInfo := w.finfo[int(watchEvent.Ident)]
			if fileInfo.IsDir() && fileEvent.IsModify() {
				w.sendDirectoryChangeEvents(fileEvent.Name)
			} else {
				// Send the event on the events channel
				w.Event <- fileEvent
			}

			// Move to next event
			events = events[1:]
		}
	}
}

func (w *Watcher) watchDirectoryFiles(dirPath string) {
	// Get all files
	files, err := ioutil.ReadDir(dirPath)
	if err != nil {
		w.Error <- err
	}

	// Search for new files
	for _, fileInfo := range files {
		if fileInfo.IsDir() == false {
			filePath := path.Join(dirPath, fileInfo.Name())
			// Watch file to mimic linux fsnotify
			e := w.addWatch(filePath, NOTE_DELETE|NOTE_WRITE|NOTE_RENAME)
			if e != nil {
				w.Error <- e
			}
		}
	}
}

// sendDirectoryEvents searches the directory for newly created files
// and sends them over the event channel. This functionality is to have
// the BSD version of fsnotify mach linux fsnotify which provides a
// create event for files created in a watched directory.
func (w *Watcher) sendDirectoryChangeEvents(dirPath string) {
	w.watchDirectoryFiles(dirPath)
	// Get all files
	files, err := ioutil.ReadDir(dirPath)
	if err != nil {
		w.Error <- err
	}

	// Search for new files
	for _, fileInfo := range files {
		if fileInfo.IsDir() == false {
			filePath := path.Join(dirPath, fileInfo.Name())
			if w.watches[filePath] == 0 {
				// Send create event
				fileEvent := new(FileEvent)
				fileEvent.Name = filePath
				fileEvent.create = true
				w.Event <- fileEvent
			}
		}
	}
}

const (
	// Flags (from <sys/event.h>)
	NOTE_DELETE = 0x0001 /* vnode was removed */
	NOTE_WRITE  = 0x0002 /* data contents changed */
	NOTE_EXTEND = 0x0004 /* size increased */
	NOTE_ATTRIB = 0x0008 /* attributes changed */
	NOTE_LINK   = 0x0010 /* link count changed */
	NOTE_RENAME = 0x0020 /* vnode was renamed */
	NOTE_REVOKE = 0x0040 /* vnode access was revoked */

	// Watch all events
	NOTE_ALLEVENTS = NOTE_DELETE | NOTE_WRITE | NOTE_ATTRIB | NOTE_RENAME

	// Block for 100 ms on each call to kevent
	keventWaitTime = 100e6
)
<commit_msg>BSD - Fix issue reported by robfig<commit_after>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build freebsd openbsd netbsd darwin

/*
Package fsnotify implements filesystem notification.

Example:
    watcher, err := fsnotify.NewWatcher()
    if err != nil {
        log.Fatal(err)
    }
    err = watcher.Watch("/tmp")
    if err != nil {
        log.Fatal(err)
    }
    for {
        select {
        case ev := <-watcher.Event:
            log.Println("event:", ev)
        case err := <-watcher.Error:
            log.Println("error:", err)
        }
    }

*/
package fsnotify

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"syscall"
)

type FileEvent struct {
	mask   uint32 // Mask of events
	Name   string // File name (optional)
	create bool   // set by fsnotify package if found new file
}

// IsCreate reports whether the FileEvent was triggerd by a creation
func (e *FileEvent) IsCreate() bool { return e.create }

// IsDelete reports whether the FileEvent was triggerd by a delete
func (e *FileEvent) IsDelete() bool { return (e.mask & NOTE_DELETE) == NOTE_DELETE }

// IsModify reports whether the FileEvent was triggerd by a file modification
func (e *FileEvent) IsModify() bool {
	return ((e.mask&NOTE_WRITE) == NOTE_WRITE || (e.mask&NOTE_ATTRIB) == NOTE_ATTRIB)
}

// IsRename reports whether the FileEvent was triggerd by a change name
func (e *FileEvent) IsRename() bool { return (e.mask & NOTE_RENAME) == NOTE_RENAME }

type Watcher struct {
	kq       int                 // File descriptor (as returned by the kqueue() syscall)
	watches  map[string]int      // Map of watched file diescriptors (key: path)
	paths    map[int]string      // Map of watched paths (key: watch descriptor)
	finfo    map[int]os.FileInfo // Map of file information (isDir, isReg; key: watch descriptor)
	Error    chan error          // Errors are sent on this channel
	Event    chan *FileEvent     // Events are returned on this channel
	done     chan bool           // Channel for sending a "quit message" to the reader goroutine
	isClosed bool                // Set to true when Close() is first called
	kbuf     [1]syscall.Kevent_t // An event buffer for Add/Remove watch
}

// NewWatcher creates and returns a new kevent instance using kqueue(2)
func NewWatcher() (*Watcher, error) {
	fd, errno := syscall.Kqueue()
	if fd == -1 {
		return nil, os.NewSyscallError("kqueue", errno)
	}
	w := &Watcher{
		kq:      fd,
		watches: make(map[string]int),
		paths:   make(map[int]string),
		finfo:   make(map[int]os.FileInfo),
		Event:   make(chan *FileEvent),
		Error:   make(chan error),
		done:    make(chan bool, 1),
	}

	go w.readEvents()
	return w, nil
}

// Close closes a kevent watcher instance
// It sends a message to the reader goroutine to quit and removes all watches
// associated with the kevent instance
func (w *Watcher) Close() error {
	if w.isClosed {
		return nil
	}
	w.isClosed = true

	// Send "quit" message to the reader goroutine
	w.done <- true
	for path := range w.watches {
		w.RemoveWatch(path)
	}

	return nil
}

// AddWatch adds path to the watched file set.
// The flags are interpreted as described in kevent(2).
func (w *Watcher) addWatch(path string, flags uint32) error {
	if w.isClosed {
		return errors.New("kevent instance already closed")
	}

	watchEntry := &w.kbuf[0]
	watchEntry.Fflags = flags

	watchfd, found := w.watches[path]
	if !found {
		fd, errno := syscall.Open(path, syscall.O_NONBLOCK|syscall.O_RDONLY, 0700)
		if fd == -1 {
			return errno
		}
		watchfd = fd

		w.watches[path] = watchfd
		w.paths[watchfd] = path

		fi, _ := os.Stat(path)
		w.finfo[watchfd] = fi
		if fi.IsDir() {
			w.watchDirectoryFiles(path)
		}
	}
	syscall.SetKevent(watchEntry, watchfd, syscall.EVFILT_VNODE, syscall.EV_ADD|syscall.EV_CLEAR)

	wd, errno := syscall.Kevent(w.kq, w.kbuf[:], nil, nil)
	if wd == -1 {
		return errno
	} else if (watchEntry.Flags & syscall.EV_ERROR) == syscall.EV_ERROR {
		return errors.New("kevent add error")
	}

	return nil
}

// Watch adds path to the watched file set, watching all events.
func (w *Watcher) Watch(path string) error {
	return w.addWatch(path, NOTE_ALLEVENTS)
}

// RemoveWatch removes path from the watched file set.
func (w *Watcher) RemoveWatch(path string) error {
	watchfd, ok := w.watches[path]
	if !ok {
		return errors.New(fmt.Sprintf("can't remove non-existent kevent watch for: %s", path))
	}
	syscall.Close(watchfd)
	watchEntry := &w.kbuf[0]
	syscall.SetKevent(watchEntry, w.watches[path], syscall.EVFILT_VNODE, syscall.EV_DELETE)
	success, errno := syscall.Kevent(w.kq, w.kbuf[:], nil, nil)
	if success == -1 {
		return os.NewSyscallError("kevent_rm_watch", errno)
	} else if (watchEntry.Flags & syscall.EV_ERROR) == syscall.EV_ERROR {
		return errors.New("kevent rm error")
	}
	delete(w.watches, path)
	return nil
}

// readEvents reads from the kqueue file descriptor, converts the
// received events into Event objects and sends them via the Event channel
func (w *Watcher) readEvents() {
	var (
		eventbuf [10]syscall.Kevent_t // Event buffer
		events   []syscall.Kevent_t   // Received events
		twait    *syscall.Timespec    // Time to block waiting for events
		n        int                  // Number of events returned from kevent
		errno    error                // Syscall errno
	)
	events = eventbuf[0:0]
	twait = new(syscall.Timespec)
	*twait = syscall.NsecToTimespec(keventWaitTime)

	for {
		// See if there is a message on the "done" channel
		var done bool
		select {
		case done = <-w.done:
		default:
		}

		// If "done" message is received
		if done {
			errno := syscall.Close(w.kq)
			if errno != nil {
				w.Error <- os.NewSyscallError("close", errno)
			}
			close(w.Event)
			close(w.Error)
			return
		}

		// Get new events
		if len(events) == 0 {
			n, errno = syscall.Kevent(w.kq, nil, eventbuf[:], twait)

			// EINTR is okay, basically the syscall was interrupted before
			// timeout expired.
			if errno != nil && errno != syscall.EINTR {
				w.Error <- os.NewSyscallError("kevent", errno)
				continue
			} else {
				events = eventbuf[0:n]
			}
		}

		// Timeout, no big deal
		if n == 0 {
			continue
		}

		// Flush the events we recieved to the events channel
		for len(events) > 0 {
			fileEvent := new(FileEvent)
			watchEvent := &events[0]
			fileEvent.mask = uint32(watchEvent.Fflags)
			fileEvent.Name = w.paths[int(watchEvent.Ident)]

			fileInfo := w.finfo[int(watchEvent.Ident)]
			if fileInfo.IsDir() && fileEvent.IsModify() {
				w.sendDirectoryChangeEvents(fileEvent.Name)
			} else {
				// Send the event on the events channel
				w.Event <- fileEvent
			}

			// Move to next event
			events = events[1:]
		}
	}
}

func (w *Watcher) watchDirectoryFiles(dirPath string) {
	// Get all files
	files, err := ioutil.ReadDir(dirPath)
	if err != nil {
		w.Error <- err
	}

	// Search for new files
	for _, fileInfo := range files {
		if fileInfo.IsDir() == false {
			filePath := path.Join(dirPath, fileInfo.Name())
			// Watch file to mimic linux fsnotify
			e := w.addWatch(filePath, NOTE_DELETE|NOTE_WRITE|NOTE_RENAME)
			if e != nil {
				w.Error <- e
			}
		}
	}
}

// sendDirectoryEvents searches the directory for newly created files
// and sends them over the event channel. This functionality is to have
// the BSD version of fsnotify mach linux fsnotify which provides a
// create event for files created in a watched directory.
func (w *Watcher) sendDirectoryChangeEvents(dirPath string) {
	w.watchDirectoryFiles(dirPath)
	// Get all files
	files, err := ioutil.ReadDir(dirPath)
	if err != nil {
		w.Error <- err
	}

	// Search for new files
	for _, fileInfo := range files {
		if fileInfo.IsDir() == false {
			filePath := path.Join(dirPath, fileInfo.Name())
			if w.watches[filePath] == 0 {
				// Send create event
				fileEvent := new(FileEvent)
				fileEvent.Name = filePath
				fileEvent.create = true
				w.Event <- fileEvent
			}
		}
	}
}

const (
	// Flags (from <sys/event.h>)
	NOTE_DELETE = 0x0001 /* vnode was removed */
	NOTE_WRITE  = 0x0002 /* data contents changed */
	NOTE_EXTEND = 0x0004 /* size increased */
	NOTE_ATTRIB = 0x0008 /* attributes changed */
	NOTE_LINK   = 0x0010 /* link count changed */
	NOTE_RENAME = 0x0020 /* vnode was renamed */
	NOTE_REVOKE = 0x0040 /* vnode access was revoked */

	// Watch all events
	NOTE_ALLEVENTS = NOTE_DELETE | NOTE_WRITE | NOTE_ATTRIB | NOTE_RENAME

	// Block for 100 ms on each call to kevent
	keventWaitTime = 100e6
)
<|endoftext|>


<commit_before>package kubernetes

import (
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"github.com/davidhiendl/telegraf-docker-sd/app/logger"
	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/types"
	"github.com/davidhiendl/telegraf-docker-sd/app/utils"
	"bytes"
)

func (backend *KubernetesBackend) processPodsOnCurrentKubeNode() error {

	pods, err := backend.client.CoreV1().Pods("").List(metav1.ListOptions{})
	if err != nil {
		return err
	}

	logger.Debugf(LOG_PREFIX + " there are %d trackedPods in the cluster", len(pods.Items))

	uidToPod := make(map[types.UID]*corev1.Pod)

	// process trackedPods
	for key, _ := range pods.Items {

		// create explicit reference to value as value from range would be a copy that cannot be referenced
		pod := &pods.Items[key]

		// skip trackedPods on other nodes
		if pod.Spec.NodeName != backend.node.Name {
			continue
		}

		// map trackedPods by UID for cleanup later on
		uidToPod[pod.UID] = pod

		// check if already tracked and if it should be tracked
		_, ok := backend.trackedPods[pod.UID]
		shouldTrack := backend.shouldTrackPod(pod)
		if !ok && shouldTrack {
			backend.startTrackingPod(pod)
		}
	}

	logger.Debugf(LOG_PREFIX + " there are %d trackedPods on current node", len(uidToPod))

	// check for trackedPods that are still tracked but shouldn't be
	for uid, trackedPod := range backend.trackedPods {
		pod, ok := uidToPod[uid]

		shouldTrack := backend.shouldTrackPod(pod)
		if !ok || !shouldTrack {
			logger.Debugf(LOG_PREFIX + " stopping to track pod: found_in_cluster=%v, shouldTrack=%v", ok, shouldTrack)
			backend.cleanupTrackedPod(trackedPod)
		}
	}

	return nil
}

func (backend *KubernetesBackend) cleanupTrackedPod(tp *TrackedPod) {
	logger.Infof(LOG_PREFIX + "[%v] cleaning up no longer tracked pod", tp.Name)
	utils.RemoveConfigFile(tp.ConfigFile())
	delete(backend.trackedPods, tp.UID)
}

func (backend *KubernetesBackend) shouldTrackPod(pod *corev1.Pod) bool {
	if pod.Status.Phase != corev1.PodRunning {
		return false
	}
	// TODO check if pod is using HostPorts/HostNetworking and if it is reachable (agent on host => yes, agent in cluster: probably no?)
	return true
}

// initialize pod tracking
func (backend *KubernetesBackend) startTrackingPod(pod *corev1.Pod) {
	logger.Debugf(LOG_PREFIX + "[%v] starting to track pod", pod.Name)
	podIP := pod.Status.PodIP

	logger.Debugf(LOG_PREFIX + "[%v] detected IP: %v", pod.Name, podIP)

	// create object
	trackedPod := NewTrackedPod(backend, pod)

	// process templates
	backend.executeTemplatesAgainstTrackedPod(trackedPod)

	// add to map
	backend.trackedPods[trackedPod.UID] = trackedPod
	backend.telegrafReloader.RequestReload()
}

// process templates for pod and write config file
func (backend *KubernetesBackend) executeTemplatesAgainstTrackedPod(trackedPod *TrackedPod) {
	// process template(s) for container
	configBuffer := new(bytes.Buffer)
	for _, template := range backend.templates {
		logger.Debugf(LOG_PREFIX + "[%v] running against template: %v", trackedPod.Name, template.FileName)
		err := template.Execute(configBuffer, trackedPod)
		if err != nil {
			logger.Fatalf(LOG_PREFIX + "[%v] error during template execution: %+v", trackedPod.Name, err)
		}
	}

	// write config
	utils.WriteConfigFile(trackedPod.ConfigFile(), configBuffer.String())
}
<commit_msg>corrected log statements<commit_after>package kubernetes

import (
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"github.com/davidhiendl/telegraf-docker-sd/app/logger"
	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/types"
	"github.com/davidhiendl/telegraf-docker-sd/app/utils"
	"bytes"
)

func (backend *KubernetesBackend) processPodsOnCurrentKubeNode() error {

	pods, err := backend.client.CoreV1().Pods("").List(metav1.ListOptions{})
	if err != nil {
		return err
	}

	logger.Debugf(LOG_PREFIX + " there are %d pods in the cluster", len(pods.Items))

	uidToPod := make(map[types.UID]*corev1.Pod)

	// process trackedPods
	for key, _ := range pods.Items {

		// create explicit reference to value as value from range would be a copy that cannot be referenced
		pod := &pods.Items[key]

		// skip trackedPods on other nodes
		if pod.Spec.NodeName != backend.node.Name {
			continue
		}

		// map trackedPods by UID for cleanup later on
		uidToPod[pod.UID] = pod

		// check if already tracked and if it should be tracked
		_, ok := backend.trackedPods[pod.UID]
		shouldTrack := backend.shouldTrackPod(pod)
		if !ok && shouldTrack {
			backend.startTrackingPod(pod)
		}
	}

	logger.Debugf(LOG_PREFIX + " there are %d pods on current node", len(uidToPod))

	// check for trackedPods that are still tracked but shouldn't be
	for uid, trackedPod := range backend.trackedPods {
		pod, ok := uidToPod[uid]

		shouldTrack := backend.shouldTrackPod(pod)
		if !ok || !shouldTrack {
			logger.Debugf(LOG_PREFIX + " stopping to track pod: found_in_cluster=%v, shouldTrack=%v", ok, shouldTrack)
			backend.cleanupTrackedPod(trackedPod)
		}
	}

	return nil
}

func (backend *KubernetesBackend) cleanupTrackedPod(tp *TrackedPod) {
	logger.Infof(LOG_PREFIX + "[%v] cleaning up no longer tracked pod", tp.Name)
	utils.RemoveConfigFile(tp.ConfigFile())
	delete(backend.trackedPods, tp.UID)
}

func (backend *KubernetesBackend) shouldTrackPod(pod *corev1.Pod) bool {
	if pod.Status.Phase != corev1.PodRunning {
		return false
	}
	// TODO check if pod is using HostPorts/HostNetworking and if it is reachable (agent on host => yes, agent in cluster: probably no?)
	return true
}

// initialize pod tracking
func (backend *KubernetesBackend) startTrackingPod(pod *corev1.Pod) {
	logger.Debugf(LOG_PREFIX + "[%v] starting to track pod", pod.Name)
	podIP := pod.Status.PodIP

	logger.Debugf(LOG_PREFIX + "[%v] detected IP: %v", pod.Name, podIP)

	// create object
	trackedPod := NewTrackedPod(backend, pod)

	// process templates
	backend.executeTemplatesAgainstTrackedPod(trackedPod)

	// add to map
	backend.trackedPods[trackedPod.UID] = trackedPod
	backend.telegrafReloader.RequestReload()
}

// process templates for pod and write config file
func (backend *KubernetesBackend) executeTemplatesAgainstTrackedPod(trackedPod *TrackedPod) {
	// process template(s) for container
	configBuffer := new(bytes.Buffer)
	for _, template := range backend.templates {
		logger.Debugf(LOG_PREFIX + "[%v] running against template: %v", trackedPod.Name, template.FileName)
		err := template.Execute(configBuffer, trackedPod)
		if err != nil {
			logger.Fatalf(LOG_PREFIX + "[%v] error during template execution: %+v", trackedPod.Name, err)
		}
	}

	// write config
	utils.WriteConfigFile(trackedPod.ConfigFile(), configBuffer.String())
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/jbeda/geom"
	"io"
	"math"
	"os"
	"strings"
)

////////////////////////////////////////////////////////////////////////////
// SVG serialization helper
type SVG struct {
	Writer io.Writer
}

func NewSVG(w io.Writer) *SVG {
	return &SVG{w}
}

func (svg *SVG) printf(format string, a ...interface{}) (n int, errno error) {
	return fmt.Fprintf(svg.Writer, format, a...)
}

// BUGBUG: not quoting aware
func extraparams(s []string) string {
	ep := ""
	for i := 0; i < len(s); i++ {
		if strings.Index(s[i], "=") > 0 {
			ep += (s[i]) + " "
		} else if len(s[i]) > 0 {
			ep += fmt.Sprintf("style='%s' ", s[i])
		}
	}
	return ep
}

func onezero(b bool) string {
	if b {
		return "1"
	}
	return "0"
}

func (svg *SVG) Start(viewBox geom.Rect, s ...string) {
	svg.printf(`<?xml version="1.0"?>
<svg version="1.1"
     viewBox="%f %f %f %f"
     xmlns="http://www.w3.org/2000/svg" %s>
`, viewBox.Min.X, viewBox.Min.Y, viewBox.Width(), viewBox.Height(), extraparams(s))
}

func (svg *SVG) End() {
	svg.printf("</svg>")
}

func (svg *SVG) Line(p1 geom.Coord, p2 geom.Coord, s ...string) {
	svg.printf("<line x1='%f' y1='%f' x2='%f' y2='%f' %s/>\n", p1.X, p1.Y, p2.X, p2.Y, extraparams(s))
}

func (svg *SVG) Circle(c geom.Coord, r float64, s ...string) {
	svg.printf("<circle cx='%f' cy='%f' r='%f' %s/>\n", c.X, c.Y, r, extraparams(s))
}

func (svg *SVG) CircularArc(p1 geom.Coord, p2 geom.Coord, r float64, largeArc bool, sweep bool, s ...string) {
	svg.printf("<path d='M%f,%f A%f,%f 0 %s,%s %f,%f' %s/>\n",
		p1.X, p1.Y, r, r, onezero(largeArc), onezero(sweep), p2.X, p2.Y, extraparams(s))
}

////////////////////////////////////////////////////////////////////////////
// Penrose stuff

// Penrose primitive
type PenrosePrimitive interface {
	Draw(s *SVG)
	Deflate() []PenrosePrimitive
}

const (
	c1 = math.Phi - 1.0
	c2 = 2.0 - math.Phi
)

type halfKite struct {
	*geom.Triangle
}

func (me halfKite) Draw(s *SVG) {
	s.Line(me.B, me.C, "stroke: black")
	s.Line(me.C, me.A, "stroke: black")

	rB := me.A.Minus(me.C).Magnitude()
	rA := rB * 1.0 / math.Phi

	aA1 := me.B.Minus(me.A).Unit().Times(rA).Plus(me.A)
	aA2 := me.C.Minus(me.A).Unit().Times(rA).Plus(me.A)
	s.CircularArc(aA1, aA2, rA, false,
		geom.VertexAngle(me.A, me.B, me.C) < 0, "stroke:green")

	aB1 := me.C.Minus(me.B).Unit().Times(rB).Plus(me.B)
	aB2 := me.A.Minus(me.B).Unit().Times(rB).Plus(me.B)
	s.CircularArc(aB1, aB2, rB, false,
		geom.VertexAngle(me.C, me.B, me.A) > 0, "stroke:red")

}

func (me halfKite) Deflate() []PenrosePrimitive {
	r := make([]PenrosePrimitive, 3)
	d := me.A.Times(c1).Plus(me.B.Times(c2))
	e := me.B.Times(c1).Plus(me.C.Times(c2))
	r[0] = halfKite{&geom.Triangle{d, me.C, me.A}}
	r[1] = halfKite{&geom.Triangle{d, me.C, e}}
	r[2] = halfDart{&geom.Triangle{me.B, e, d}}
	return r
}

type halfDart struct {
	*geom.Triangle
}

func (me halfDart) Draw(s *SVG) {
	s.Line(me.B, me.C, "stroke: black")
	s.Line(me.C, me.A, "stroke: black")

	r := me.A.Minus(me.C).Magnitude()
	rA := r / math.Pow(math.Phi, 2)
	rB := r / math.Pow(math.Phi, 3)

	aA1 := me.B.Minus(me.A).Unit().Times(rA).Plus(me.A)
	aA2 := me.C.Minus(me.A).Unit().Times(rA).Plus(me.A)
	s.CircularArc(aA1, aA2, rA, false,
		geom.VertexAngle(me.A, me.B, me.C) < 0, "stroke:red")

	aB1 := me.C.Minus(me.B).Unit().Times(rB).Plus(me.B)
	aB2 := me.A.Minus(me.B).Unit().Times(rB).Plus(me.B)
	s.CircularArc(aB1, aB2, rB, false,
		geom.VertexAngle(me.C, me.B, me.A) > 0, "stroke:green")
}

func (me halfDart) Deflate() []PenrosePrimitive {
	r := make([]PenrosePrimitive, 2)
	d := me.A.Times(c2).Plus(me.C.Times(c1))
	r[0] = halfDart{&geom.Triangle{me.C, d, me.B}}
	r[1] = halfKite{&geom.Triangle{me.B, me.A, d}}
	return r
}

func degToRads(d float64) float64 {
	return d * math.Pi / 180.0
}

// Starting shape "constants"
var HalfKite = halfKite{
	&geom.Triangle{
		geom.Coord{0, 0},
		geom.Coord{math.Phi * math.Cos(degToRads(72)), math.Phi * math.Sin(degToRads(72))},
		geom.Coord{1, 0},
	},
}

var HalfDart = halfDart{
	&geom.Triangle{
		geom.Coord{1, 0},
		geom.Coord{c1 * math.Cos(degToRads(36)), c1 * math.Sin(degToRads(36))},
		geom.Coord{0, 0},
	},
}

func Sun() []PenrosePrimitive {
	r := make([]PenrosePrimitive, 0, 10)
	for i := 0; i < 5; i++ {
		r = append(r, halfKite{
			&geom.Triangle{
				geom.Coord{math.Cos(degToRads(float64(72 * i))), math.Sin(degToRads(float64(72 * i)))},
				geom.Coord{0, 0},
				geom.Coord{math.Cos(degToRads(float64(36 + 72.0*i))), math.Sin(degToRads(float64(36 + 72.0*i)))},
			},
		})
		r = append(r, halfKite{
			&geom.Triangle{
				geom.Coord{math.Cos(degToRads(float64(72 * i))), math.Sin(degToRads(float64(72 * i)))},
				geom.Coord{0, 0},
				geom.Coord{math.Cos(degToRads(float64(-36 + 72.0*i))), math.Sin(degToRads(float64(-36 + 72.0*i)))},
			},
		})
	}
	return r
}

func BoundsOfPrimitiveSlice(ps []PenrosePrimitive) geom.Rect {
	bounds := ps[0].(geom.Bounded).Bounds()
	for _, p := range ps[1:] {
		bounds.ExpandToContainRect(p.(geom.Bounded).Bounds())
	}
	return bounds
}

////////////////////////////////////////////////////////////////////////////
func main() {
	shapes := Sun()
	bounds := BoundsOfPrimitiveSlice(shapes)

	for i := 0; i < 5; i++ {
		newShapes := make([]PenrosePrimitive, 0, 3*len(shapes))
		for _, shape := range shapes {
			newShapes = append(newShapes, shape.Deflate()...)
		}
		shapes = newShapes
	}

	s := NewSVG(os.Stdout)
	s.Start(bounds, "stroke-width: 0.1%; stroke-linecap: round; fill: none")
	for _, shape := range shapes {
		shape.Draw(s)
	}
	s.End()
}
<commit_msg>Double stroke for red lines<commit_after>package main

import (
	"fmt"
	"github.com/jbeda/geom"
	"io"
	"math"
	"os"
	"strings"
)

const (
	DEFAULT_STYLE        = "stroke-width: 0.002; stroke-linecap: round; fill: none"
	DOUBLE_STROKE_OFFSET = 0.002
	CUT_STYLE            = "stroke: black"
	MARK1_STYLE          = "stroke: green"
	MARK2_STYLE          = "stroke: red"
)

////////////////////////////////////////////////////////////////////////////
// SVG serialization helper
type SVG struct {
	Writer io.Writer
}

func NewSVG(w io.Writer) *SVG {
	return &SVG{w}
}

func (svg *SVG) printf(format string, a ...interface{}) (n int, errno error) {
	return fmt.Fprintf(svg.Writer, format, a...)
}

// BUGBUG: not quoting aware
func extraparams(s []string) string {
	ep := ""
	for i := 0; i < len(s); i++ {
		if strings.Index(s[i], "=") > 0 {
			ep += (s[i]) + " "
		} else if len(s[i]) > 0 {
			ep += fmt.Sprintf("style='%s' ", s[i])
		}
	}
	return ep
}

func onezero(b bool) string {
	if b {
		return "1"
	}
	return "0"
}

func (svg *SVG) Start(viewBox geom.Rect, s ...string) {
	svg.printf(`<?xml version="1.0"?>
<svg version="1.1"
     viewBox="%f %f %f %f"
     xmlns="http://www.w3.org/2000/svg" %s>
`, viewBox.Min.X, viewBox.Min.Y, viewBox.Width(), viewBox.Height(), extraparams(s))
}

func (svg *SVG) End() {
	svg.printf("</svg>")
}

func (svg *SVG) Line(p1 geom.Coord, p2 geom.Coord, s ...string) {
	svg.printf("<line x1='%f' y1='%f' x2='%f' y2='%f' %s/>\n", p1.X, p1.Y, p2.X, p2.Y, extraparams(s))
}

func (svg *SVG) Circle(c geom.Coord, r float64, s ...string) {
	svg.printf("<circle cx='%f' cy='%f' r='%f' %s/>\n", c.X, c.Y, r, extraparams(s))
}

func (svg *SVG) CircularArc(c geom.Coord, v1 geom.Coord, v2 geom.Coord, r float64, s ...string) {
	p1 := v1.Minus(c).Unit().Times(r).Plus(c)
	p2 := v2.Minus(c).Unit().Times(r).Plus(c)
	a := geom.VertexAngle(v1, c, v2)
	largeArc := a > math.Pi
	sweep := a > 0
	svg.printf("<path d='M%f,%f A%f,%f 0 %s,%s %f,%f' %s/>\n",
		p1.X, p1.Y, r, r, onezero(largeArc), onezero(sweep), p2.X, p2.Y, extraparams(s))
}

////////////////////////////////////////////////////////////////////////////
// Penrose stuff

// Penrose primitive
type PenrosePrimitive interface {
	Draw(s *SVG)
	Deflate() []PenrosePrimitive
}

const (
	c1 = math.Phi - 1.0
	c2 = 2.0 - math.Phi
)

type halfKite struct {
	*geom.Triangle
}

func (me halfKite) Draw(s *SVG) {
	s.Line(me.B, me.C, CUT_STYLE)
	s.Line(me.C, me.A, CUT_STYLE)

	rB := me.A.Minus(me.C).Magnitude()
	rA := rB * 1.0 / math.Phi

	s.CircularArc(me.A, me.B, me.C, rA+DOUBLE_STROKE_OFFSET, MARK2_STYLE)
	s.CircularArc(me.A, me.B, me.C, rA-DOUBLE_STROKE_OFFSET, MARK2_STYLE)
	s.CircularArc(me.B, me.C, me.A, rB, MARK1_STYLE)
}

func (me halfKite) Deflate() []PenrosePrimitive {
	r := make([]PenrosePrimitive, 3)
	d := me.A.Times(c1).Plus(me.B.Times(c2))
	e := me.B.Times(c1).Plus(me.C.Times(c2))
	r[0] = halfKite{&geom.Triangle{d, me.C, me.A}}
	r[1] = halfKite{&geom.Triangle{d, me.C, e}}
	r[2] = halfDart{&geom.Triangle{me.B, e, d}}
	return r
}

type halfDart struct {
	*geom.Triangle
}

func (me halfDart) Draw(s *SVG) {
	s.Line(me.B, me.C, CUT_STYLE)
	s.Line(me.C, me.A, CUT_STYLE)

	r := me.A.Minus(me.C).Magnitude()
	rA := r / math.Pow(math.Phi, 2)
	rB := r / math.Pow(math.Phi, 3)

	s.CircularArc(me.A, me.B, me.C, rA, MARK1_STYLE)
	s.CircularArc(me.B, me.C, me.A, rB+DOUBLE_STROKE_OFFSET, MARK2_STYLE)
	s.CircularArc(me.B, me.C, me.A, rB-DOUBLE_STROKE_OFFSET, MARK2_STYLE)
}

func (me halfDart) Deflate() []PenrosePrimitive {
	r := make([]PenrosePrimitive, 2)
	d := me.A.Times(c2).Plus(me.C.Times(c1))
	r[0] = halfDart{&geom.Triangle{me.C, d, me.B}}
	r[1] = halfKite{&geom.Triangle{me.B, me.A, d}}
	return r
}

func degToRads(d float64) float64 {
	return d * math.Pi / 180.0
}

// Starting shape "constants"
var HalfKite = halfKite{
	&geom.Triangle{
		geom.Coord{0, 0},
		geom.Coord{math.Phi * math.Cos(degToRads(72)), math.Phi * math.Sin(degToRads(72))},
		geom.Coord{1, 0},
	},
}

var HalfDart = halfDart{
	&geom.Triangle{
		geom.Coord{1, 0},
		geom.Coord{c1 * math.Cos(degToRads(36)), c1 * math.Sin(degToRads(36))},
		geom.Coord{0, 0},
	},
}

func Sun() []PenrosePrimitive {
	r := make([]PenrosePrimitive, 0, 10)
	for i := 0; i < 5; i++ {
		r = append(r, halfKite{
			&geom.Triangle{
				geom.Coord{math.Cos(degToRads(float64(72 * i))), math.Sin(degToRads(float64(72 * i)))},
				geom.Coord{0, 0},
				geom.Coord{math.Cos(degToRads(float64(36 + 72.0*i))), math.Sin(degToRads(float64(36 + 72.0*i)))},
			},
		})
		r = append(r, halfKite{
			&geom.Triangle{
				geom.Coord{math.Cos(degToRads(float64(72 * i))), math.Sin(degToRads(float64(72 * i)))},
				geom.Coord{0, 0},
				geom.Coord{math.Cos(degToRads(float64(-36 + 72.0*i))), math.Sin(degToRads(float64(-36 + 72.0*i)))},
			},
		})
	}
	return r
}

func BoundsOfPrimitiveSlice(ps []PenrosePrimitive) geom.Rect {
	bounds := ps[0].(geom.Bounded).Bounds()
	for _, p := range ps[1:] {
		bounds.ExpandToContainRect(p.(geom.Bounded).Bounds())
	}
	return bounds
}

////////////////////////////////////////////////////////////////////////////
func main() {
	shapes := Sun()
	bounds := BoundsOfPrimitiveSlice(shapes)

	for i := 0; i < 5; i++ {
		newShapes := make([]PenrosePrimitive, 0, 3*len(shapes))
		for _, shape := range shapes {
			newShapes = append(newShapes, shape.Deflate()...)
		}
		shapes = newShapes
	}

	s := NewSVG(os.Stdout)
	s.Start(bounds, DEFAULT_STYLE)
	for _, shape := range shapes {
		shape.Draw(s)
	}
	s.End()
}
<|endoftext|>


<commit_before>package service

import (
	"net/http"
	"sync"
	"time"

	commonModel "github.com/Cepave/open-falcon-backend/common/model"
	commonQueue "github.com/Cepave/open-falcon-backend/common/queue"
	commonSling "github.com/Cepave/open-falcon-backend/common/sling"
	"github.com/Cepave/open-falcon-backend/modules/hbs/cache"
	"github.com/Cepave/open-falcon-backend/modules/nqm-mng/model"
	"github.com/dghubble/sling"
)

type AgentHeartbeatService struct {
	sync.WaitGroup
	safeQ            *commonQueue.Queue
	qConfig          *commonQueue.Config
	started          bool
	slingInit        *sling.Sling
	rowsAffectedCnt  int64
	agentsDroppedCnt int64
	agentsPutCnt     int64
}

func NewAgentHeartbeatService(config *commonQueue.Config) *AgentHeartbeatService {
	return &AgentHeartbeatService{
		safeQ:     commonQueue.New(),
		qConfig:   config,
		slingInit: NewSlingBase().Post("api/v1/agent/heartbeat"),
	}
}

func (s *AgentHeartbeatService) Start() {
	if s.started {
		return
	}
	s.started = true

	s.Add(1)
	go func() {
		defer s.Done()

		for {
			if !s.started {
				break
			}

			s.consumeHeartbeatQueue(100*time.Millisecond, false)

			if !s.started {
				break
			}

			time.Sleep(5 * time.Second)
		}

		s.consumeHeartbeatQueue(0, true)
	}()
}

func (s *AgentHeartbeatService) consumeHeartbeatQueue(waitForQueue time.Duration, logFlag bool) {
	for {
		var elementType *model.AgentHeartbeat
		absArray := s.safeQ.DrainNWithDurationByType(s.qConfig, elementType)
		agents := absArray.([]*model.AgentHeartbeat)
		agentsNum := len(agents)
		if agentsNum == 0 {
			break
		}

		s.heartbeat(agents)
		if logFlag {
			logger.Infof("Flushing [%d] agents", agentsNum)
		}
	}
}

func (s *AgentHeartbeatService) Stop() {
	if !s.started {
		return
	}

	s.started = false
	logger.Infof("Stopping AgentHeartbeatService. Size of queue: [%d]", s.CurrentSize())

	/**
	 * Waiting for queue to be processed
	 */
	s.Wait()
	s.safeQ = nil
}

func (s *AgentHeartbeatService) Put(req *commonModel.AgentReportRequest) {
	if !s.started {
		return
	}
	now := time.Now().Unix()
	cache.Agents.Put(req, now)
	agent := &model.AgentHeartbeat{
		Hostname:      req.Hostname,
		IP:            req.IP,
		AgentVersion:  req.AgentVersion,
		PluginVersion: req.PluginVersion,
		UpdateTime:    now,
	}
	s.safeQ.Enqueue(agent)
	s.agentsPutCnt++
}

func (s *AgentHeartbeatService) CurrentSize() int {
	return s.safeQ.Len()
}

func (s *AgentHeartbeatService) CumulativeAgentsDropped() int64 {
	return s.agentsDroppedCnt
}

func (s *AgentHeartbeatService) CumulativeAgentsPut() int64 {
	return s.agentsPutCnt
}

func (s *AgentHeartbeatService) CumulativeRowsAffected() int64 {
	return s.rowsAffectedCnt
}

func (s *AgentHeartbeatService) heartbeat(agents []*model.AgentHeartbeat) {
	param := struct {
		UpdateOnly bool `json:"update_only"`
	}{updateOnlyFlag}
	s.slingInit = s.slingInit.BodyJSON(agents).QueryStruct(&param)

	res := model.AgentHeartbeatResult{}
	err := commonSling.ToSlintExt(s.slingInit).DoReceive(http.StatusOK, &res)
	if err != nil {
		s.agentsDroppedCnt += int64(len(agents))
		logger.Errorln("Heartbeat:", err)
		return
	}

	s.rowsAffectedCnt += res.RowsAffected
}
<commit_msg>[OWL-1695] Extract heartbeat call for ease of testing.<commit_after>package service

import (
	"net/http"
	"sync"
	"time"

	commonModel "github.com/Cepave/open-falcon-backend/common/model"
	commonQueue "github.com/Cepave/open-falcon-backend/common/queue"
	commonSling "github.com/Cepave/open-falcon-backend/common/sling"
	"github.com/Cepave/open-falcon-backend/modules/hbs/cache"
	"github.com/Cepave/open-falcon-backend/modules/nqm-mng/model"
	"github.com/dghubble/sling"
)

type AgentHeartbeatService struct {
	sync.WaitGroup
	safeQ            *commonQueue.Queue
	qConfig          *commonQueue.Config
	started          bool
	agentsPutCnt     int64
	heartbeatCall    func([]*model.AgentHeartbeat, *sling.Sling) (int64, int64)
	slingInit        *sling.Sling
	rowsAffectedCnt  int64
	agentsDroppedCnt int64
}

func NewAgentHeartbeatService(config *commonQueue.Config) *AgentHeartbeatService {
	return &AgentHeartbeatService{
		safeQ:         commonQueue.New(),
		qConfig:       config,
		heartbeatCall: heartbeat,
		slingInit:     NewSlingBase().Post("api/v1/agent/heartbeat"),
	}
}

func (s *AgentHeartbeatService) Start() {
	if s.started {
		return
	}
	s.started = true

	s.Add(1)
	go func() {
		defer s.Done()

		for {
			if !s.started {
				break
			}

			s.consumeHeartbeatQueue(100*time.Millisecond, false)

			if !s.started {
				break
			}

			time.Sleep(5 * time.Second)
		}

		s.consumeHeartbeatQueue(0, true)
	}()
}

func (s *AgentHeartbeatService) consumeHeartbeatQueue(waitForQueue time.Duration, logFlag bool) {
	for {
		var elementType *model.AgentHeartbeat
		absArray := s.safeQ.DrainNWithDurationByType(s.qConfig, elementType)
		agents := absArray.([]*model.AgentHeartbeat)
		agentsNum := len(agents)
		if agentsNum == 0 {
			break
		}

		r, d := s.heartbeatCall(agents, s.slingInit)
		s.rowsAffectedCnt += r
		s.agentsDroppedCnt += d
		if logFlag {
			logger.Infof("Flushing [%d] agents", agentsNum)
		}
	}
}

func (s *AgentHeartbeatService) Stop() {
	if !s.started {
		return
	}

	s.started = false
	logger.Infof("Stopping AgentHeartbeatService. Size of queue: [%d]", s.CurrentSize())

	/**
	 * Waiting for queue to be processed
	 */
	s.Wait()
	s.safeQ = nil
}

func (s *AgentHeartbeatService) Put(req *commonModel.AgentReportRequest) {
	if !s.started {
		return
	}
	now := time.Now().Unix()
	cache.Agents.Put(req, now)
	agent := &model.AgentHeartbeat{
		Hostname:      req.Hostname,
		IP:            req.IP,
		AgentVersion:  req.AgentVersion,
		PluginVersion: req.PluginVersion,
		UpdateTime:    now,
	}
	s.safeQ.Enqueue(agent)
	s.agentsPutCnt++
}

func (s *AgentHeartbeatService) CurrentSize() int {
	return s.safeQ.Len()
}

func (s *AgentHeartbeatService) CumulativeAgentsDropped() int64 {
	return s.agentsDroppedCnt
}

func (s *AgentHeartbeatService) CumulativeAgentsPut() int64 {
	return s.agentsPutCnt
}

func (s *AgentHeartbeatService) CumulativeRowsAffected() int64 {
	return s.rowsAffectedCnt
}

func heartbeat(agents []*model.AgentHeartbeat, slingAPI *sling.Sling) (rowsAffectedCnt int64, agentsDroppedCnt int64) {
	param := struct {
		UpdateOnly bool `json:"update_only"`
	}{updateOnlyFlag}
	req := slingAPI.BodyJSON(agents).QueryStruct(&param)

	res := model.AgentHeartbeatResult{}
	err := commonSling.ToSlintExt(req).DoReceive(http.StatusOK, &res)
	if err != nil {
		logger.Errorln("[Heartbeat]", err)
		return 0, int64(len(agents))
	}

	return res.RowsAffected, 0
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"

	"io/ioutil"

	"fmt"

	"bytes"
)

func mapKeyPairs(input [][]byte, buffer []string) []map[string][]byte {
	tokenMapS := []map[string][]byte{}
	for _, pv := range input {
		tokenMap := map[string]*json.RawMessage{}

		err := json.Unmarshal(pv, &tokenMap)
		if err != nil {
			fmt.Println("Failed to parse one of the JSON token files.")
		}

		tempMap := map[string][]byte{}

		for k, v := range tokenMap {
			j, err := json.Marshal(&v)
			checkError(err)
			tempMap[buffer[0]+k+buffer[1]] = j
		}
		tokenMapS = append(tokenMapS, tempMap)
	}
	return tokenMapS
}

func detokenize(input []byte, tokenMap []map[string][]byte) []byte {
	overrideCompiled := map[string][]byte{}
	for _, v := range tokenMap {
		for mk, mv := range v {
			overrideCompiled[mk] = mv
		}
	}
	for k, v := range overrideCompiled {
		input = bytes.Replace(input, []byte(k), v, -1)
		input = bytes.Replace(input, []byte("\\u003c"), []byte("<"), -1)
		input = bytes.Replace(input, []byte("\\u003e"), []byte(">"), -1)
		input = bytes.Replace(input, []byte("\\u0026"), []byte("&"), -1)
		input = bytes.Replace(input, []byte("\\\\\\"), []byte("\\"), -1)

	}
	return input
}

func loadFile(path string) []byte {
	file, err := ioutil.ReadFile(path)
	checkError(err)
	return file
}

func outputToFile(path string, data []byte) {
	err := ioutil.WriteFile(path, data, 0644)
	checkError(err)
}

func outputToStdout(data []byte) {
	fmt.Print(string(data))
}
<commit_msg>tweak replace<commit_after>package main

import (
	"encoding/json"

	"io/ioutil"

	"fmt"

	"bytes"
)

func mapKeyPairs(input [][]byte, buffer []string) []map[string][]byte {
	tokenMapS := []map[string][]byte{}
	for _, pv := range input {
		tokenMap := map[string]*json.RawMessage{}

		err := json.Unmarshal(pv, &tokenMap)
		if err != nil {
			fmt.Println("Failed to parse one of the JSON token files.")
		}

		tempMap := map[string][]byte{}

		for k, v := range tokenMap {
			j, err := json.Marshal(&v)
			checkError(err)
			tempMap[buffer[0]+k+buffer[1]] = j
		}
		tokenMapS = append(tokenMapS, tempMap)
	}
	return tokenMapS
}

func detokenize(input []byte, tokenMap []map[string][]byte) []byte {
	overrideCompiled := map[string][]byte{}
	for _, v := range tokenMap {
		for mk, mv := range v {
			overrideCompiled[mk] = mv
		}
	}
	for k, v := range overrideCompiled {
		input = bytes.Replace(input, []byte(k), v, -1)
		input = bytes.Replace(input, []byte("\\u003c"), []byte("<"), -1)
		input = bytes.Replace(input, []byte("\\u003e"), []byte(">"), -1)
		input = bytes.Replace(input, []byte("\\u0026"), []byte("&"), -1)
		input = bytes.Replace(input, []byte(`\\`), []byte(`\`), -1)

	}
	return input
}

func loadFile(path string) []byte {
	file, err := ioutil.ReadFile(path)
	checkError(err)
	return file
}

func outputToFile(path string, data []byte) {
	err := ioutil.WriteFile(path, data, 0644)
	checkError(err)
}

func outputToStdout(data []byte) {
	fmt.Print(string(data))
}
<|endoftext|>


<commit_before>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package provisioner_test

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"time"

	gc "launchpad.net/gocheck"

	"launchpad.net/juju-core/agent"
	agenttools "launchpad.net/juju-core/agent/tools"
	"launchpad.net/juju-core/constraints"
	"launchpad.net/juju-core/container/lxc"
	"launchpad.net/juju-core/container/lxc/mock"
	"launchpad.net/juju-core/environs"
	"launchpad.net/juju-core/environs/config"
	"launchpad.net/juju-core/instance"
	instancetest "launchpad.net/juju-core/instance/testing"
	jujutesting "launchpad.net/juju-core/juju/testing"
	"launchpad.net/juju-core/names"
	"launchpad.net/juju-core/provider"
	"launchpad.net/juju-core/state"
	coretesting "launchpad.net/juju-core/testing"
	jc "launchpad.net/juju-core/testing/checkers"
	coretools "launchpad.net/juju-core/tools"
	"launchpad.net/juju-core/version"
	"launchpad.net/juju-core/worker/provisioner"
)

type lxcSuite struct {
	lxc.TestSuite
	events chan mock.Event
}

type lxcBrokerSuite struct {
	lxcSuite
	broker      environs.InstanceBroker
	agentConfig agent.Config
}

var _ = gc.Suite(&lxcBrokerSuite{})

func (s *lxcSuite) SetUpSuite(c *gc.C) {
	s.LoggingSuite.SetUpSuite(c)
	s.TestSuite.SetUpSuite(c)
}

func (s *lxcSuite) TearDownSuite(c *gc.C) {
	s.TestSuite.TearDownSuite(c)
	s.LoggingSuite.TearDownSuite(c)
}

func (s *lxcSuite) SetUpTest(c *gc.C) {
	s.LoggingSuite.SetUpTest(c)
	s.TestSuite.SetUpTest(c)
	s.events = make(chan mock.Event)
	go func() {
		for event := range s.events {
			c.Output(3, fmt.Sprintf("lxc event: <%s, %s>", event.Action, event.InstanceId))
		}
	}()
	s.TestSuite.Factory.AddListener(s.events)
}

func (s *lxcSuite) TearDownTest(c *gc.C) {
	close(s.events)
	s.TestSuite.TearDownTest(c)
	s.LoggingSuite.TearDownTest(c)
}

func (s *lxcBrokerSuite) SetUpTest(c *gc.C) {
	s.lxcSuite.SetUpTest(c)
	tools := &coretools.Tools{
		Version: version.MustParseBinary("2.3.4-foo-bar"),
		URL:     "http://tools.testing.invalid/2.3.4-foo-bar.tgz",
	}
	config := coretesting.EnvironConfig(c)
	var err error
	s.agentConfig, err = agent.NewAgentConfig(
		agent.AgentConfigParams{
			DataDir:      "/not/used/here",
			Tag:          "tag",
			Password:     "dummy-secret",
			Nonce:        "nonce",
			APIAddresses: []string{"10.0.0.1:1234"},
			CACert:       []byte(coretesting.CACert),
		})
	c.Assert(err, gc.IsNil)
	s.broker = provisioner.NewLxcBroker(config, tools, s.agentConfig)
}

func (s *lxcBrokerSuite) startInstance(c *gc.C, machineId string) instance.Instance {
	stateInfo := jujutesting.FakeStateInfo(machineId)
	apiInfo := jujutesting.FakeAPIInfo(machineId)

	series := "series"
	nonce := "fake-nonce"
	cons := constraints.Value{}
	lxc, _, err := provider.StartInstance(s.broker, machineId, nonce, series, cons, stateInfo, apiInfo)
	c.Assert(err, gc.IsNil)
	return lxc
}

func (s *lxcBrokerSuite) TestStartInstance(c *gc.C) {
	machineId := "1/lxc/0"
	lxc := s.startInstance(c, machineId)
	c.Assert(lxc.Id(), gc.Equals, instance.Id("juju-machine-1-lxc-0"))
	c.Assert(s.lxcContainerDir(lxc), jc.IsDirectory)
	s.assertInstances(c, lxc)
	// Uses default network config
	lxcConfContents, err := ioutil.ReadFile(filepath.Join(s.ContainerDir, string(lxc.Id()), "lxc.conf"))
	c.Assert(err, gc.IsNil)
	c.Assert(string(lxcConfContents), jc.Contains, "lxc.network.type = veth")
	c.Assert(string(lxcConfContents), jc.Contains, "lxc.network.link = lxcbr0")
}

func (s *lxcBrokerSuite) TestStartInstanceWithBridgeEnviron(c *gc.C) {
	s.agentConfig.SetValue(agent.LxcBridge, "br0")
	machineId := "1/lxc/0"
	lxc := s.startInstance(c, machineId)
	c.Assert(lxc.Id(), gc.Equals, instance.Id("juju-machine-1-lxc-0"))
	c.Assert(s.lxcContainerDir(lxc), jc.IsDirectory)
	s.assertInstances(c, lxc)
	// Uses default network config
	lxcConfContents, err := ioutil.ReadFile(filepath.Join(s.ContainerDir, string(lxc.Id()), "lxc.conf"))
	c.Assert(err, gc.IsNil)
	c.Assert(string(lxcConfContents), jc.Contains, "lxc.network.type = veth")
	c.Assert(string(lxcConfContents), jc.Contains, "lxc.network.link = br0")
}

func (s *lxcBrokerSuite) TestStopInstance(c *gc.C) {
	lxc0 := s.startInstance(c, "1/lxc/0")
	lxc1 := s.startInstance(c, "1/lxc/1")
	lxc2 := s.startInstance(c, "1/lxc/2")

	err := s.broker.StopInstances([]instance.Instance{lxc0})
	c.Assert(err, gc.IsNil)
	s.assertInstances(c, lxc1, lxc2)
	c.Assert(s.lxcContainerDir(lxc0), jc.DoesNotExist)
	c.Assert(s.lxcRemovedContainerDir(lxc0), jc.IsDirectory)

	err = s.broker.StopInstances([]instance.Instance{lxc1, lxc2})
	c.Assert(err, gc.IsNil)
	s.assertInstances(c)
}

func (s *lxcBrokerSuite) TestAllInstances(c *gc.C) {
	lxc0 := s.startInstance(c, "1/lxc/0")
	lxc1 := s.startInstance(c, "1/lxc/1")
	s.assertInstances(c, lxc0, lxc1)

	err := s.broker.StopInstances([]instance.Instance{lxc1})
	c.Assert(err, gc.IsNil)
	lxc2 := s.startInstance(c, "1/lxc/2")
	s.assertInstances(c, lxc0, lxc2)
}

func (s *lxcBrokerSuite) assertInstances(c *gc.C, inst ...instance.Instance) {
	results, err := s.broker.AllInstances()
	c.Assert(err, gc.IsNil)
	instancetest.MatchInstances(c, results, inst...)
}

func (s *lxcBrokerSuite) lxcContainerDir(inst instance.Instance) string {
	return filepath.Join(s.ContainerDir, string(inst.Id()))
}

func (s *lxcBrokerSuite) lxcRemovedContainerDir(inst instance.Instance) string {
	return filepath.Join(s.RemovedDir, string(inst.Id()))
}

type lxcProvisionerSuite struct {
	CommonProvisionerSuite
	lxcSuite
	machineId string
	events    chan mock.Event
}

var _ = gc.Suite(&lxcProvisionerSuite{})

func (s *lxcProvisionerSuite) SetUpSuite(c *gc.C) {
	s.CommonProvisionerSuite.SetUpSuite(c)
	s.lxcSuite.SetUpSuite(c)
}

func (s *lxcProvisionerSuite) TearDownSuite(c *gc.C) {
	s.lxcSuite.TearDownSuite(c)
	s.CommonProvisionerSuite.TearDownSuite(c)
}

func (s *lxcProvisionerSuite) SetUpTest(c *gc.C) {
	s.CommonProvisionerSuite.SetUpTest(c)
	s.lxcSuite.SetUpTest(c)
	// Write the tools file.
	toolsDir := agenttools.SharedToolsDir(s.DataDir(), version.Current)
	c.Assert(os.MkdirAll(toolsDir, 0755), gc.IsNil)
	urlPath := filepath.Join(toolsDir, "downloaded-url.txt")
	err := ioutil.WriteFile(urlPath, []byte("http://testing.invalid/tools"), 0644)
	c.Assert(err, gc.IsNil)

	// The lxc provisioner actually needs the machine it is being created on
	// to be in state, in order to get the watcher.
	m, err := s.State.AddMachine(config.DefaultSeries, state.JobHostUnits)
	c.Assert(err, gc.IsNil)
	s.machineId = m.Id()

	s.events = make(chan mock.Event, 25)
	s.Factory.AddListener(s.events)
}

func (s *lxcProvisionerSuite) expectStarted(c *gc.C, machine *state.Machine) string {
	s.State.StartSync()
	event := <-s.events
	c.Assert(event.Action, gc.Equals, mock.Started)
	err := machine.Refresh()
	c.Assert(err, gc.IsNil)
	s.waitInstanceId(c, machine, instance.Id(event.InstanceId))
	return event.InstanceId
}

func (s *lxcProvisionerSuite) expectStopped(c *gc.C, instId string) {
	s.State.StartSync()
	event := <-s.events
	c.Assert(event.Action, gc.Equals, mock.Stopped)
	c.Assert(event.InstanceId, gc.Equals, instId)
}

func (s *lxcProvisionerSuite) expectNoEvents(c *gc.C) {
	select {
	case event := <-s.events:
		c.Fatalf("unexpected event %#v", event)
	case <-time.After(coretesting.ShortWait):
		return
	}
}

func (s *lxcProvisionerSuite) TearDownTest(c *gc.C) {
	close(s.events)
	s.lxcSuite.TearDownTest(c)
	s.CommonProvisionerSuite.TearDownTest(c)
}

func (s *lxcProvisionerSuite) newLxcProvisioner(c *gc.C) *provisioner.Provisioner {
	machineTag := names.MachineTag(s.machineId)
	agentConfig := s.AgentConfigForTag(c, machineTag)
	return provisioner.NewProvisioner(provisioner.LXC, s.State, s.machineId, agentConfig)
}

func (s *lxcProvisionerSuite) TestProvisionerStartStop(c *gc.C) {
	p := s.newLxcProvisioner(c)
	c.Assert(p.Stop(), gc.IsNil)
}

func (s *lxcProvisionerSuite) TestDoesNotStartEnvironMachines(c *gc.C) {
	p := s.newLxcProvisioner(c)
	defer stop(c, p)

	// Check that an instance is not provisioned when the machine is created.
	_, err := s.State.AddMachine(config.DefaultSeries, state.JobHostUnits)
	c.Assert(err, gc.IsNil)

	s.expectNoEvents(c)
}

func (s *lxcProvisionerSuite) addContainer(c *gc.C) *state.Machine {
	params := state.AddMachineParams{
		ParentId:      s.machineId,
		ContainerType: instance.LXC,
		Series:        config.DefaultSeries,
		Jobs:          []state.MachineJob{state.JobHostUnits},
	}
	container, err := s.State.AddMachineWithConstraints(&params)
	c.Assert(err, gc.IsNil)
	return container
}

func (s *lxcProvisionerSuite) TestContainerStartedAndStopped(c *gc.C) {
	p := s.newLxcProvisioner(c)
	defer stop(c, p)

	container := s.addContainer(c)

	instId := s.expectStarted(c, container)

	// ...and removed, along with the machine, when the machine is Dead.
	c.Assert(container.EnsureDead(), gc.IsNil)
	s.expectStopped(c, instId)
	s.waitRemoved(c, container)
}
<commit_msg>Forgot some simplification.<commit_after>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package provisioner_test

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"time"

	gc "launchpad.net/gocheck"

	"launchpad.net/juju-core/agent"
	agenttools "launchpad.net/juju-core/agent/tools"
	"launchpad.net/juju-core/constraints"
	"launchpad.net/juju-core/container/lxc"
	"launchpad.net/juju-core/container/lxc/mock"
	"launchpad.net/juju-core/environs"
	"launchpad.net/juju-core/environs/config"
	"launchpad.net/juju-core/instance"
	instancetest "launchpad.net/juju-core/instance/testing"
	jujutesting "launchpad.net/juju-core/juju/testing"
	"launchpad.net/juju-core/names"
	"launchpad.net/juju-core/provider"
	"launchpad.net/juju-core/state"
	coretesting "launchpad.net/juju-core/testing"
	jc "launchpad.net/juju-core/testing/checkers"
	coretools "launchpad.net/juju-core/tools"
	"launchpad.net/juju-core/version"
	"launchpad.net/juju-core/worker/provisioner"
)

type lxcSuite struct {
	lxc.TestSuite
	events chan mock.Event
}

type lxcBrokerSuite struct {
	lxcSuite
	broker      environs.InstanceBroker
	agentConfig agent.Config
}

var _ = gc.Suite(&lxcBrokerSuite{})

func (s *lxcSuite) SetUpTest(c *gc.C) {
	s.TestSuite.SetUpTest(c)
	s.events = make(chan mock.Event)
	go func() {
		for event := range s.events {
			c.Output(3, fmt.Sprintf("lxc event: <%s, %s>", event.Action, event.InstanceId))
		}
	}()
	s.TestSuite.Factory.AddListener(s.events)
}

func (s *lxcSuite) TearDownTest(c *gc.C) {
	close(s.events)
	s.TestSuite.TearDownTest(c)
}

func (s *lxcBrokerSuite) SetUpTest(c *gc.C) {
	s.lxcSuite.SetUpTest(c)
	tools := &coretools.Tools{
		Version: version.MustParseBinary("2.3.4-foo-bar"),
		URL:     "http://tools.testing.invalid/2.3.4-foo-bar.tgz",
	}
	config := coretesting.EnvironConfig(c)
	var err error
	s.agentConfig, err = agent.NewAgentConfig(
		agent.AgentConfigParams{
			DataDir:      "/not/used/here",
			Tag:          "tag",
			Password:     "dummy-secret",
			Nonce:        "nonce",
			APIAddresses: []string{"10.0.0.1:1234"},
			CACert:       []byte(coretesting.CACert),
		})
	c.Assert(err, gc.IsNil)
	s.broker = provisioner.NewLxcBroker(config, tools, s.agentConfig)
}

func (s *lxcBrokerSuite) startInstance(c *gc.C, machineId string) instance.Instance {
	stateInfo := jujutesting.FakeStateInfo(machineId)
	apiInfo := jujutesting.FakeAPIInfo(machineId)

	series := "series"
	nonce := "fake-nonce"
	cons := constraints.Value{}
	lxc, _, err := provider.StartInstance(s.broker, machineId, nonce, series, cons, stateInfo, apiInfo)
	c.Assert(err, gc.IsNil)
	return lxc
}

func (s *lxcBrokerSuite) TestStartInstance(c *gc.C) {
	machineId := "1/lxc/0"
	lxc := s.startInstance(c, machineId)
	c.Assert(lxc.Id(), gc.Equals, instance.Id("juju-machine-1-lxc-0"))
	c.Assert(s.lxcContainerDir(lxc), jc.IsDirectory)
	s.assertInstances(c, lxc)
	// Uses default network config
	lxcConfContents, err := ioutil.ReadFile(filepath.Join(s.ContainerDir, string(lxc.Id()), "lxc.conf"))
	c.Assert(err, gc.IsNil)
	c.Assert(string(lxcConfContents), jc.Contains, "lxc.network.type = veth")
	c.Assert(string(lxcConfContents), jc.Contains, "lxc.network.link = lxcbr0")
}

func (s *lxcBrokerSuite) TestStartInstanceWithBridgeEnviron(c *gc.C) {
	s.agentConfig.SetValue(agent.LxcBridge, "br0")
	machineId := "1/lxc/0"
	lxc := s.startInstance(c, machineId)
	c.Assert(lxc.Id(), gc.Equals, instance.Id("juju-machine-1-lxc-0"))
	c.Assert(s.lxcContainerDir(lxc), jc.IsDirectory)
	s.assertInstances(c, lxc)
	// Uses default network config
	lxcConfContents, err := ioutil.ReadFile(filepath.Join(s.ContainerDir, string(lxc.Id()), "lxc.conf"))
	c.Assert(err, gc.IsNil)
	c.Assert(string(lxcConfContents), jc.Contains, "lxc.network.type = veth")
	c.Assert(string(lxcConfContents), jc.Contains, "lxc.network.link = br0")
}

func (s *lxcBrokerSuite) TestStopInstance(c *gc.C) {
	lxc0 := s.startInstance(c, "1/lxc/0")
	lxc1 := s.startInstance(c, "1/lxc/1")
	lxc2 := s.startInstance(c, "1/lxc/2")

	err := s.broker.StopInstances([]instance.Instance{lxc0})
	c.Assert(err, gc.IsNil)
	s.assertInstances(c, lxc1, lxc2)
	c.Assert(s.lxcContainerDir(lxc0), jc.DoesNotExist)
	c.Assert(s.lxcRemovedContainerDir(lxc0), jc.IsDirectory)

	err = s.broker.StopInstances([]instance.Instance{lxc1, lxc2})
	c.Assert(err, gc.IsNil)
	s.assertInstances(c)
}

func (s *lxcBrokerSuite) TestAllInstances(c *gc.C) {
	lxc0 := s.startInstance(c, "1/lxc/0")
	lxc1 := s.startInstance(c, "1/lxc/1")
	s.assertInstances(c, lxc0, lxc1)

	err := s.broker.StopInstances([]instance.Instance{lxc1})
	c.Assert(err, gc.IsNil)
	lxc2 := s.startInstance(c, "1/lxc/2")
	s.assertInstances(c, lxc0, lxc2)
}

func (s *lxcBrokerSuite) assertInstances(c *gc.C, inst ...instance.Instance) {
	results, err := s.broker.AllInstances()
	c.Assert(err, gc.IsNil)
	instancetest.MatchInstances(c, results, inst...)
}

func (s *lxcBrokerSuite) lxcContainerDir(inst instance.Instance) string {
	return filepath.Join(s.ContainerDir, string(inst.Id()))
}

func (s *lxcBrokerSuite) lxcRemovedContainerDir(inst instance.Instance) string {
	return filepath.Join(s.RemovedDir, string(inst.Id()))
}

type lxcProvisionerSuite struct {
	CommonProvisionerSuite
	lxcSuite
	machineId string
	events    chan mock.Event
}

var _ = gc.Suite(&lxcProvisionerSuite{})

func (s *lxcProvisionerSuite) SetUpSuite(c *gc.C) {
	s.CommonProvisionerSuite.SetUpSuite(c)
	s.lxcSuite.SetUpSuite(c)
}

func (s *lxcProvisionerSuite) TearDownSuite(c *gc.C) {
	s.lxcSuite.TearDownSuite(c)
	s.CommonProvisionerSuite.TearDownSuite(c)
}

func (s *lxcProvisionerSuite) SetUpTest(c *gc.C) {
	s.CommonProvisionerSuite.SetUpTest(c)
	s.lxcSuite.SetUpTest(c)
	// Write the tools file.
	toolsDir := agenttools.SharedToolsDir(s.DataDir(), version.Current)
	c.Assert(os.MkdirAll(toolsDir, 0755), gc.IsNil)
	urlPath := filepath.Join(toolsDir, "downloaded-url.txt")
	err := ioutil.WriteFile(urlPath, []byte("http://testing.invalid/tools"), 0644)
	c.Assert(err, gc.IsNil)

	// The lxc provisioner actually needs the machine it is being created on
	// to be in state, in order to get the watcher.
	m, err := s.State.AddMachine(config.DefaultSeries, state.JobHostUnits)
	c.Assert(err, gc.IsNil)
	s.machineId = m.Id()

	s.events = make(chan mock.Event, 25)
	s.Factory.AddListener(s.events)
}

func (s *lxcProvisionerSuite) expectStarted(c *gc.C, machine *state.Machine) string {
	s.State.StartSync()
	event := <-s.events
	c.Assert(event.Action, gc.Equals, mock.Started)
	err := machine.Refresh()
	c.Assert(err, gc.IsNil)
	s.waitInstanceId(c, machine, instance.Id(event.InstanceId))
	return event.InstanceId
}

func (s *lxcProvisionerSuite) expectStopped(c *gc.C, instId string) {
	s.State.StartSync()
	event := <-s.events
	c.Assert(event.Action, gc.Equals, mock.Stopped)
	c.Assert(event.InstanceId, gc.Equals, instId)
}

func (s *lxcProvisionerSuite) expectNoEvents(c *gc.C) {
	select {
	case event := <-s.events:
		c.Fatalf("unexpected event %#v", event)
	case <-time.After(coretesting.ShortWait):
		return
	}
}

func (s *lxcProvisionerSuite) TearDownTest(c *gc.C) {
	close(s.events)
	s.lxcSuite.TearDownTest(c)
	s.CommonProvisionerSuite.TearDownTest(c)
}

func (s *lxcProvisionerSuite) newLxcProvisioner(c *gc.C) *provisioner.Provisioner {
	machineTag := names.MachineTag(s.machineId)
	agentConfig := s.AgentConfigForTag(c, machineTag)
	return provisioner.NewProvisioner(provisioner.LXC, s.State, s.machineId, agentConfig)
}

func (s *lxcProvisionerSuite) TestProvisionerStartStop(c *gc.C) {
	p := s.newLxcProvisioner(c)
	c.Assert(p.Stop(), gc.IsNil)
}

func (s *lxcProvisionerSuite) TestDoesNotStartEnvironMachines(c *gc.C) {
	p := s.newLxcProvisioner(c)
	defer stop(c, p)

	// Check that an instance is not provisioned when the machine is created.
	_, err := s.State.AddMachine(config.DefaultSeries, state.JobHostUnits)
	c.Assert(err, gc.IsNil)

	s.expectNoEvents(c)
}

func (s *lxcProvisionerSuite) addContainer(c *gc.C) *state.Machine {
	params := state.AddMachineParams{
		ParentId:      s.machineId,
		ContainerType: instance.LXC,
		Series:        config.DefaultSeries,
		Jobs:          []state.MachineJob{state.JobHostUnits},
	}
	container, err := s.State.AddMachineWithConstraints(&params)
	c.Assert(err, gc.IsNil)
	return container
}

func (s *lxcProvisionerSuite) TestContainerStartedAndStopped(c *gc.C) {
	p := s.newLxcProvisioner(c)
	defer stop(c, p)

	container := s.addContainer(c)

	instId := s.expectStarted(c, container)

	// ...and removed, along with the machine, when the machine is Dead.
	c.Assert(container.EnsureDead(), gc.IsNil)
	s.expectStopped(c, instId)
	s.waitRemoved(c, container)
}
<|endoftext|>


<commit_before>package urbandictionary

import (
	"fmt"
	"github.com/0x263b/Porygon2"
	"github.com/0x263b/Porygon2/web"
	"net/url"
	"regexp"
	"strconv"
)

const (
	urbanURL = "http://api.urbandictionary.com/v0/define?term=%s"
)

type DefinitionResults struct {
	Tags       []string `json:"tags"`
	ResultType string   `json:"result_type"`
	List       []struct {
		Defid       int    `json:"defid"`
		Word        string `json:"word"`
		Author      string `json:"author"`
		Permalink   string `json:"permalink"`
		Definition  string `json:"definition"`
		Example     string `json:"example"`
		ThumbsUp    int    `json:"thumbs_up"`
		ThumbsDown  int    `json:"thumbs_down"`
		CurrentVote string `json:"current_vote"`
	} `json:"list"`
	Sounds []interface{} `json:"sounds"`
}

func urban(command *bot.Cmd, matches []string) (msg string, err error) {
	var i int64 = 0
	if matches[1] != "" {
		i, _ = strconv.ParseInt(matches[1], 10, 64)
		i = i - 1
	}

	results := &DefinitionResults{}
	err = web.GetJSON(fmt.Sprintf(urbanURL, url.QueryEscape(matches[2])), results)
	if err != nil {
		return fmt.Sprintf("Urban Dictionary | %s | (No definition found)", matches[2]), nil
	}
	if results.ResultType == "no_results" {
		return fmt.Sprintf("Urban Dictionary | %s | (No definition found)", matches[2]), nil
	}

	word := results.List[i].Word
	definition := results.List[i].Definition
	permalink := results.List[i].Permalink
	short := web.ShortenURL(permalink)

	reg := regexp.MustCompile("\\s+")
	definition = reg.ReplaceAllString(definition, " ") // Strip tabs and newlines

	if len(definition) > 180 {
		definition = fmt.Sprintf("%s...", definition[0:180])
	}

	output := fmt.Sprintf("Urban Dictionary | %s | %s | %s", word, short, definition)

	return output, nil
}

func init() {
	bot.RegisterCommand(
		"^u(?:r(?:ban)?)? (?:([1-7]{1}) )?(.+)$",
		urban)
}
<commit_msg>Check if match is out of range<commit_after>package urbandictionary

import (
	"fmt"
	"github.com/0x263b/Porygon2"
	"github.com/0x263b/Porygon2/web"
	"net/url"
	"regexp"
	"strconv"
)

const (
	urbanURL = "http://api.urbandictionary.com/v0/define?term=%s"
)

type DefinitionResults struct {
	Tags       []string `json:"tags"`
	ResultType string   `json:"result_type"`
	List       []struct {
		Defid       int    `json:"defid"`
		Word        string `json:"word"`
		Author      string `json:"author"`
		Permalink   string `json:"permalink"`
		Definition  string `json:"definition"`
		Example     string `json:"example"`
		ThumbsUp    int    `json:"thumbs_up"`
		ThumbsDown  int    `json:"thumbs_down"`
		CurrentVote string `json:"current_vote"`
	} `json:"list"`
	Sounds []interface{} `json:"sounds"`
}

func urban(command *bot.Cmd, matches []string) (msg string, err error) {
	var i int64 = 0
	if matches[1] != "" {
		i, _ = strconv.ParseInt(matches[1], 10, 64)
		i = i - 1
	}

	results := &DefinitionResults{}
	err = web.GetJSON(fmt.Sprintf(urbanURL, url.QueryEscape(matches[2])), results)
	if err != nil {
		return fmt.Sprintf("Urban Dictionary | %s | (No definition found)", matches[2]), nil
	}
	if results.ResultType == "no_results" {
		return fmt.Sprintf("Urban Dictionary | %s | (No definition found)", matches[2]), nil
	}
	if int(i+1) > len(results.List) {
		return fmt.Sprintf("Urban Dictionary | %s | (No definition found)", matches[2]), nil
	}

	word := results.List[i].Word
	definition := results.List[i].Definition
	permalink := results.List[i].Permalink
	short := web.ShortenURL(permalink)

	reg := regexp.MustCompile("\\s+")
	definition = reg.ReplaceAllString(definition, " ") // Strip tabs and newlines

	if len(definition) > 180 {
		definition = fmt.Sprintf("%s...", definition[0:180])
	}

	output := fmt.Sprintf("Urban Dictionary | %s | %s | %s", word, short, definition)

	return output, nil
}

func init() {
	bot.RegisterCommand(
		"^u(?:r(?:ban)?)? (?:([1-7]{1}) )?(.+)$",
		urban)
}
<|endoftext|>


<commit_before>package scanner

import (
	"bytes"
	"errors"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/calmh/syncthing/lamport"
	"github.com/calmh/syncthing/protocol"
)

type Walker struct {
	// Dir is the base directory for the walk
	Dir string
	// BlockSize controls the size of the block used when hashing.
	BlockSize int
	// If IgnoreFile is not empty, it is the name used for the file that holds ignore patterns.
	IgnoreFile string
	// If TempNamer is not nil, it is used to ignore tempory files when walking.
	TempNamer TempNamer
	// If CurrentFiler is not nil, it is queried for the current file before rescanning.
	CurrentFiler CurrentFiler
	// If Suppressor is not nil, it is queried for supression of modified files.
	// Suppressed files will be returned with empty metadata and the Suppressed flag set.
	// Requires CurrentFiler to be set.
	Suppressor Suppressor
}

type TempNamer interface {
	// Temporary returns a temporary name for the filed referred to by filepath.
	TempName(path string) string
	// IsTemporary returns true if path refers to the name of temporary file.
	IsTemporary(path string) bool
}

type Suppressor interface {
	// Supress returns true if the update to the named file should be ignored.
	Suppress(name string, fi os.FileInfo) (bool, bool)
}

type CurrentFiler interface {
	// CurrentFile returns the file as seen at last scan.
	CurrentFile(name string) File
}

// Walk returns the list of files found in the local repository by scanning the
// file system. Files are blockwise hashed.
func (w *Walker) Walk() (files []File, ignore map[string][]string, err error) {
	if debug {
		l.Debugln("Walk", w.Dir, w.BlockSize, w.IgnoreFile)
	}

	err = checkDir(w.Dir)
	if err != nil {
		return
	}

	t0 := time.Now()

	ignore = make(map[string][]string)
	hashFiles := w.walkAndHashFiles(&files, ignore)

	filepath.Walk(w.Dir, w.loadIgnoreFiles(w.Dir, ignore))
	filepath.Walk(w.Dir, hashFiles)

	if debug {
		t1 := time.Now()
		d := t1.Sub(t0).Seconds()
		l.Debugf("Walk in %.02f ms, %.0f files/s", d*1000, float64(len(files))/d)
	}

	err = checkDir(w.Dir)
	return
}

// CleanTempFiles removes all files that match the temporary filename pattern.
func (w *Walker) CleanTempFiles() {
	filepath.Walk(w.Dir, w.cleanTempFile)
}

func (w *Walker) loadIgnoreFiles(dir string, ign map[string][]string) filepath.WalkFunc {
	return func(p string, info os.FileInfo, err error) error {
		if err != nil {
			return nil
		}

		rn, err := filepath.Rel(dir, p)
		if err != nil {
			return nil
		}

		if pn, sn := filepath.Split(rn); sn == w.IgnoreFile {
			pn := strings.Trim(pn, "/")
			bs, _ := ioutil.ReadFile(p)
			lines := bytes.Split(bs, []byte("\n"))
			var patterns []string
			for _, line := range lines {
				if len(line) > 0 {
					patterns = append(patterns, string(line))
				}
			}
			ign[pn] = patterns
		}

		return nil
	}
}

func (w *Walker) walkAndHashFiles(res *[]File, ign map[string][]string) filepath.WalkFunc {
	return func(p string, info os.FileInfo, err error) error {
		if err != nil {
			if debug {
				l.Debugln("error:", p, info, err)
			}
			return nil
		}

		rn, err := filepath.Rel(w.Dir, p)
		if err != nil {
			if debug {
				l.Debugln("rel error:", p, err)
			}
			return nil
		}

		if rn == "." {
			return nil
		}

		if w.TempNamer != nil && w.TempNamer.IsTemporary(rn) {
			// A temporary file
			if debug {
				l.Debugln("temporary:", rn)
			}
			return nil
		}

		if _, sn := filepath.Split(rn); sn == w.IgnoreFile {
			// An ignore-file; these are ignored themselves
			if debug {
				l.Debugln("ignorefile:", rn)
			}
			return nil
		}

		if w.ignoreFile(ign, rn) {
			// An ignored file
			if debug {
				l.Debugln("ignored:", rn)
			}
			if info.IsDir() {
				return filepath.SkipDir
			}
			return nil
		}

		if info.Mode().IsDir() {
			if w.CurrentFiler != nil {
				cf := w.CurrentFiler.CurrentFile(rn)
				if cf.Modified == info.ModTime().Unix() && cf.Flags == uint32(info.Mode()&os.ModePerm|protocol.FlagDirectory) {
					if debug {
						l.Debugln("unchanged:", cf)
					}
					*res = append(*res, cf)
				} else {
					f := File{
						Name:     rn,
						Version:  lamport.Default.Tick(0),
						Flags:    uint32(info.Mode()&os.ModePerm) | protocol.FlagDirectory,
						Modified: info.ModTime().Unix(),
					}
					if debug {
						l.Debugln("dir:", cf, f)
					}
					*res = append(*res, f)
				}
				return nil
			}
		}

		if info.Mode().IsRegular() {
			if w.CurrentFiler != nil {
				cf := w.CurrentFiler.CurrentFile(rn)
				if cf.Flags&protocol.FlagDeleted == 0 && cf.Modified == info.ModTime().Unix() {
					if debug {
						l.Debugln("unchanged:", cf)
					}
					*res = append(*res, cf)
					return nil
				}

				if w.Suppressor != nil {
					if cur, prev := w.Suppressor.Suppress(rn, info); cur && !prev {
						l.Infof("Changes to %q are being temporarily suppressed because it changes too frequently.", p)
						cf.Suppressed = true
						cf.Version++
						if debug {
							l.Debugln("suppressed:", cf)
						}
						*res = append(*res, cf)
						return nil
					} else if prev && !cur {
						l.Infof("Changes to %q are no longer suppressed.", p)
					}
				}
			}

			fd, err := os.Open(p)
			if err != nil {
				if debug {
					l.Debugln("open:", p, err)
				}
				return nil
			}
			defer fd.Close()

			t0 := time.Now()
			blocks, err := Blocks(fd, w.BlockSize)
			if err != nil {
				if debug {
					l.Debugln("hash error:", rn, err)
				}
				return nil
			}
			if debug {
				t1 := time.Now()
				l.Debugln("hashed:", rn, ";", len(blocks), "blocks;", info.Size(), "bytes;", int(float64(info.Size())/1024/t1.Sub(t0).Seconds()), "KB/s")
			}
			f := File{
				Name:     rn,
				Version:  lamport.Default.Tick(0),
				Size:     info.Size(),
				Flags:    uint32(info.Mode()),
				Modified: info.ModTime().Unix(),
				Blocks:   blocks,
			}
			*res = append(*res, f)
		}

		return nil
	}
}

func (w *Walker) cleanTempFile(path string, info os.FileInfo, err error) error {
	if err != nil {
		return err
	}
	if info.Mode()&os.ModeType == 0 && w.TempNamer.IsTemporary(path) {
		os.Remove(path)
	}
	return nil
}

func (w *Walker) ignoreFile(patterns map[string][]string, file string) bool {
	first, last := filepath.Split(file)
	for prefix, pats := range patterns {
		if len(prefix) == 0 || prefix == first || strings.HasPrefix(first, prefix+"/") {
			for _, pattern := range pats {
				if match, _ := filepath.Match(pattern, last); match {
					return true
				}
			}
		}
	}
	return false
}

func checkDir(dir string) error {
	if info, err := os.Stat(dir); err != nil {
		return err
	} else if !info.IsDir() {
		return errors.New(dir + ": not a directory")
	}
	return nil
}
<commit_msg>Also sync changes to only permission bits (fixes #237)<commit_after>package scanner

import (
	"bytes"
	"errors"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/calmh/syncthing/lamport"
	"github.com/calmh/syncthing/protocol"
)

type Walker struct {
	// Dir is the base directory for the walk
	Dir string
	// BlockSize controls the size of the block used when hashing.
	BlockSize int
	// If IgnoreFile is not empty, it is the name used for the file that holds ignore patterns.
	IgnoreFile string
	// If TempNamer is not nil, it is used to ignore tempory files when walking.
	TempNamer TempNamer
	// If CurrentFiler is not nil, it is queried for the current file before rescanning.
	CurrentFiler CurrentFiler
	// If Suppressor is not nil, it is queried for supression of modified files.
	// Suppressed files will be returned with empty metadata and the Suppressed flag set.
	// Requires CurrentFiler to be set.
	Suppressor Suppressor
}

type TempNamer interface {
	// Temporary returns a temporary name for the filed referred to by filepath.
	TempName(path string) string
	// IsTemporary returns true if path refers to the name of temporary file.
	IsTemporary(path string) bool
}

type Suppressor interface {
	// Supress returns true if the update to the named file should be ignored.
	Suppress(name string, fi os.FileInfo) (bool, bool)
}

type CurrentFiler interface {
	// CurrentFile returns the file as seen at last scan.
	CurrentFile(name string) File
}

// Walk returns the list of files found in the local repository by scanning the
// file system. Files are blockwise hashed.
func (w *Walker) Walk() (files []File, ignore map[string][]string, err error) {
	if debug {
		l.Debugln("Walk", w.Dir, w.BlockSize, w.IgnoreFile)
	}

	err = checkDir(w.Dir)
	if err != nil {
		return
	}

	t0 := time.Now()

	ignore = make(map[string][]string)
	hashFiles := w.walkAndHashFiles(&files, ignore)

	filepath.Walk(w.Dir, w.loadIgnoreFiles(w.Dir, ignore))
	filepath.Walk(w.Dir, hashFiles)

	if debug {
		t1 := time.Now()
		d := t1.Sub(t0).Seconds()
		l.Debugf("Walk in %.02f ms, %.0f files/s", d*1000, float64(len(files))/d)
	}

	err = checkDir(w.Dir)
	return
}

// CleanTempFiles removes all files that match the temporary filename pattern.
func (w *Walker) CleanTempFiles() {
	filepath.Walk(w.Dir, w.cleanTempFile)
}

func (w *Walker) loadIgnoreFiles(dir string, ign map[string][]string) filepath.WalkFunc {
	return func(p string, info os.FileInfo, err error) error {
		if err != nil {
			return nil
		}

		rn, err := filepath.Rel(dir, p)
		if err != nil {
			return nil
		}

		if pn, sn := filepath.Split(rn); sn == w.IgnoreFile {
			pn := strings.Trim(pn, "/")
			bs, _ := ioutil.ReadFile(p)
			lines := bytes.Split(bs, []byte("\n"))
			var patterns []string
			for _, line := range lines {
				if len(line) > 0 {
					patterns = append(patterns, string(line))
				}
			}
			ign[pn] = patterns
		}

		return nil
	}
}

func (w *Walker) walkAndHashFiles(res *[]File, ign map[string][]string) filepath.WalkFunc {
	return func(p string, info os.FileInfo, err error) error {
		if err != nil {
			if debug {
				l.Debugln("error:", p, info, err)
			}
			return nil
		}

		rn, err := filepath.Rel(w.Dir, p)
		if err != nil {
			if debug {
				l.Debugln("rel error:", p, err)
			}
			return nil
		}

		if rn == "." {
			return nil
		}

		if w.TempNamer != nil && w.TempNamer.IsTemporary(rn) {
			// A temporary file
			if debug {
				l.Debugln("temporary:", rn)
			}
			return nil
		}

		if _, sn := filepath.Split(rn); sn == w.IgnoreFile {
			// An ignore-file; these are ignored themselves
			if debug {
				l.Debugln("ignorefile:", rn)
			}
			return nil
		}

		if w.ignoreFile(ign, rn) {
			// An ignored file
			if debug {
				l.Debugln("ignored:", rn)
			}
			if info.IsDir() {
				return filepath.SkipDir
			}
			return nil
		}

		if info.Mode().IsDir() {
			if w.CurrentFiler != nil {
				cf := w.CurrentFiler.CurrentFile(rn)
				if cf.Modified == info.ModTime().Unix() && cf.Flags == uint32(info.Mode()&os.ModePerm|protocol.FlagDirectory) {
					if debug {
						l.Debugln("unchanged:", cf)
					}
					*res = append(*res, cf)
				} else {
					f := File{
						Name:     rn,
						Version:  lamport.Default.Tick(0),
						Flags:    uint32(info.Mode()&os.ModePerm) | protocol.FlagDirectory,
						Modified: info.ModTime().Unix(),
					}
					if debug {
						l.Debugln("dir:", cf, f)
					}
					*res = append(*res, f)
				}
				return nil
			}
		}

		if info.Mode().IsRegular() {
			if w.CurrentFiler != nil {
				cf := w.CurrentFiler.CurrentFile(rn)
				if cf.Flags&protocol.FlagDeleted == 0 && cf.Modified == info.ModTime().Unix() && cf.Flags == uint32(info.Mode()&os.ModePerm) {
					if debug {
						l.Debugln("unchanged:", cf)
					}
					*res = append(*res, cf)
					return nil
				}

				if w.Suppressor != nil {
					if cur, prev := w.Suppressor.Suppress(rn, info); cur && !prev {
						l.Infof("Changes to %q are being temporarily suppressed because it changes too frequently.", p)
						cf.Suppressed = true
						cf.Version++
						if debug {
							l.Debugln("suppressed:", cf)
						}
						*res = append(*res, cf)
						return nil
					} else if prev && !cur {
						l.Infof("Changes to %q are no longer suppressed.", p)
					}
				}
			}

			fd, err := os.Open(p)
			if err != nil {
				if debug {
					l.Debugln("open:", p, err)
				}
				return nil
			}
			defer fd.Close()

			t0 := time.Now()
			blocks, err := Blocks(fd, w.BlockSize)
			if err != nil {
				if debug {
					l.Debugln("hash error:", rn, err)
				}
				return nil
			}
			if debug {
				t1 := time.Now()
				l.Debugln("hashed:", rn, ";", len(blocks), "blocks;", info.Size(), "bytes;", int(float64(info.Size())/1024/t1.Sub(t0).Seconds()), "KB/s")
			}
			f := File{
				Name:     rn,
				Version:  lamport.Default.Tick(0),
				Size:     info.Size(),
				Flags:    uint32(info.Mode()),
				Modified: info.ModTime().Unix(),
				Blocks:   blocks,
			}
			*res = append(*res, f)
		}

		return nil
	}
}

func (w *Walker) cleanTempFile(path string, info os.FileInfo, err error) error {
	if err != nil {
		return err
	}
	if info.Mode()&os.ModeType == 0 && w.TempNamer.IsTemporary(path) {
		os.Remove(path)
	}
	return nil
}

func (w *Walker) ignoreFile(patterns map[string][]string, file string) bool {
	first, last := filepath.Split(file)
	for prefix, pats := range patterns {
		if len(prefix) == 0 || prefix == first || strings.HasPrefix(first, prefix+"/") {
			for _, pattern := range pats {
				if match, _ := filepath.Match(pattern, last); match {
					return true
				}
			}
		}
	}
	return false
}

func checkDir(dir string) error {
	if info, err := os.Stat(dir); err != nil {
		return err
	} else if !info.IsDir() {
		return errors.New(dir + ": not a directory")
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Hajime Hoshi
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package vorbis provides Ogg/Vorbis decoder.
package vorbis

import (
	"io"
	"runtime"

	"github.com/hajimehoshi/ebiten/audio"
	"github.com/jfreymuth/oggvorbis"
)

type decoded struct {
	data       []float32
	totalBytes int
	posInBytes int
	source     io.Closer
	decoder    *oggvorbis.Reader
}

func (d *decoded) readUntil(posInBytes int) error {
	c := 0
	for len(d.data) < posInBytes/2 {
		buffer := make([]float32, 8192)
		n, err := d.decoder.Read(buffer)
		if n > 0 {
			d.data = append(d.data, buffer[:n]...)
		}
		if err == io.EOF {
			if err := d.source.Close(); err != nil {
				return err
			}
			break
		}
		if err != nil {
			return err
		}
		c++
		if c%4 == 0 {
			runtime.Gosched()
		}
	}
	return nil
}

func (d *decoded) Read(b []byte) (int, error) {
	total := d.totalBytes
	l := total - d.posInBytes
	if l > len(b) {
		l = len(b)
	}
	if l < 0 {
		return 0, io.EOF
	}
	// l must be even so that d.posInBytes is always even.
	l = l / 2 * 2
	if err := d.readUntil(d.posInBytes + l); err != nil {
		return 0, err
	}
	for i := 0; i < l/2; i++ {
		f := d.data[d.posInBytes/2+i]
		s := int16(f * (1<<15 - 1))
		b[2*i] = uint8(s)
		b[2*i+1] = uint8(s >> 8)
	}
	d.posInBytes += l
	if d.posInBytes == total {
		return l, io.EOF
	}
	return l, nil
}

func (d *decoded) Seek(offset int64, whence int) (int64, error) {
	next := int64(0)
	switch whence {
	case io.SeekStart:
		next = offset
	case io.SeekCurrent:
		next = int64(d.posInBytes) + offset
	case io.SeekEnd:
		next = int64(d.totalBytes) + offset
	}
	// pos should be always even
	next = next / 2 * 2
	d.posInBytes = int(next)
	if err := d.readUntil(d.posInBytes); err != nil {
		return 0, err
	}
	return next, nil
}

func (d *decoded) Close() error {
	runtime.SetFinalizer(d, nil)
	return nil
}

func (d *decoded) Size() int64 {
	return int64(d.totalBytes)
}

// decode accepts an ogg stream and returns a decorded stream.
func decode(in audio.ReadSeekCloser) (*decoded, int, int, error) {
	// TODO: Lazy evaluation
	r, err := oggvorbis.NewReader(in)
	if err != nil {
		return nil, 0, 0, err
	}
	d := &decoded{
		data:       []float32{},
		totalBytes: int(r.Length()) * 4, // TODO: What if length is 0?
		posInBytes: 0,
		source:     in,
		decoder:    r,
	}
	runtime.SetFinalizer(d, (*decoded).Close)
	return d, r.Channels(), r.SampleRate(), nil
}
<commit_msg>audio/vorbis: Read some data before actual using (#297)<commit_after>// Copyright 2015 Hajime Hoshi
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package vorbis provides Ogg/Vorbis decoder.
package vorbis

import (
	"io"
	"runtime"

	"github.com/hajimehoshi/ebiten/audio"
	"github.com/jfreymuth/oggvorbis"
)

type decoded struct {
	data       []float32
	totalBytes int
	posInBytes int
	source     io.Closer
	decoder    *oggvorbis.Reader
}

func (d *decoded) readUntil(posInBytes int) error {
	c := 0
	buffer := make([]float32, 8192)
	for len(d.data) < posInBytes/2 {
		n, err := d.decoder.Read(buffer)
		if n > 0 {
			d.data = append(d.data, buffer[:n]...)
		}
		if err == io.EOF {
			if err := d.source.Close(); err != nil {
				return err
			}
			break
		}
		if err != nil {
			return err
		}
		c++
		if c%4 == 0 {
			runtime.Gosched()
		}
	}
	return nil
}

func (d *decoded) Read(b []uint8) (int, error) {
	total := d.totalBytes
	l := total - d.posInBytes
	if l > len(b) {
		l = len(b)
	}
	if l < 0 {
		return 0, io.EOF
	}
	// l must be even so that d.posInBytes is always even.
	l = l / 2 * 2
	if err := d.readUntil(d.posInBytes + l); err != nil {
		return 0, err
	}
	for i := 0; i < l/2; i++ {
		f := d.data[d.posInBytes/2+i]
		s := int16(f * (1<<15 - 1))
		b[2*i] = uint8(s)
		b[2*i+1] = uint8(s >> 8)
	}
	d.posInBytes += l
	if d.posInBytes == total {
		return l, io.EOF
	}
	return l, nil
}

func (d *decoded) Seek(offset int64, whence int) (int64, error) {
	next := int64(0)
	switch whence {
	case io.SeekStart:
		next = offset
	case io.SeekCurrent:
		next = int64(d.posInBytes) + offset
	case io.SeekEnd:
		next = int64(d.totalBytes) + offset
	}
	// pos should be always even
	next = next / 2 * 2
	d.posInBytes = int(next)
	if err := d.readUntil(d.posInBytes); err != nil {
		return 0, err
	}
	return next, nil
}

func (d *decoded) Close() error {
	runtime.SetFinalizer(d, nil)
	return nil
}

func (d *decoded) Size() int64 {
	return int64(d.totalBytes)
}

// decode accepts an ogg stream and returns a decorded stream.
func decode(in audio.ReadSeekCloser) (*decoded, int, int, error) {
	// TODO: Lazy evaluation
	r, err := oggvorbis.NewReader(in)
	if err != nil {
		return nil, 0, 0, err
	}
	d := &decoded{
		data:       make([]float32, 0, r.Length()),
		totalBytes: int(r.Length()) * 4, // TODO: What if length is 0?
		posInBytes: 0,
		source:     in,
		decoder:    r,
	}
	runtime.SetFinalizer(d, (*decoded).Close)
	if _, err := d.Read(make([]uint8, 65536)); err != nil {
		return nil, 0, 0, err
	}
	if _, err := d.Seek(0, io.SeekStart); err != nil {
		return nil, 0, 0, err
	}
	return d, r.Channels(), r.SampleRate(), nil
}
<|endoftext|>