<commit_before>// This is free and unencumbered software released into the public
// domain.  For more information, see <http://unlicense.org> or the
// accompanying UNLICENSE file.

package editor

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/nelsam/gxui"
	"github.com/nelsam/gxui/math"
	"github.com/nelsam/gxui/mixins"
	"github.com/nelsam/gxui/themes/basic"
	"github.com/nelsam/vidar/commander/input"
	"github.com/nelsam/vidar/fsw"
	"github.com/nelsam/vidar/theme"
)

type CodeEditor struct {
	mixins.CodeEditor

	theme       *basic.Theme
	syntaxTheme theme.Theme
	driver      gxui.Driver

	lock         sync.RWMutex
	lastModified time.Time
	hasChanges   bool
	filepath     string

	watcher fsw.Watcher

	selections      []gxui.TextSelection
	scrollPositions math.Point
	layers          []input.SyntaxLayer

	renamed  bool
	onRename func(newPath string)
}

func (e *CodeEditor) Init(driver gxui.Driver, theme *basic.Theme, syntaxTheme theme.Theme, font gxui.Font, file, headerText string) {
	e.theme = theme
	e.syntaxTheme = syntaxTheme
	e.driver = driver

	e.CodeEditor.Init(e, driver, theme, font)
	e.CodeEditor.SetScrollBarEnabled(true)
	e.SetDesiredWidth(math.MaxSize.W)
	e.watcherSetup()

	// TODO: move to hooks on the input.Handler
	e.OnTextChanged(func(changes []gxui.TextBoxEdit) {
		e.hasChanges = true
	})
	e.filepath = file
	e.open(headerText)

	e.SetTextColor(theme.TextBoxDefaultStyle.FontColor)
	e.SetMargin(math.Spacing{L: 3, T: 3, R: 3, B: 3})
	e.SetPadding(math.Spacing{L: 3, T: 3, R: 3, B: 3})
	e.SetBorderPen(gxui.TransparentPen)
}

func (e *CodeEditor) DataChanged(recreate bool) {
	e.List.DataChanged(recreate)
}

func (e *CodeEditor) Carets() []int {
	return e.Controller().Carets()
}

func (e *CodeEditor) SetCarets(carets ...int) {
	if len(carets) == 0 {
		e.Controller().ClearSelections()
		return
	}
	var sel []gxui.TextSelection
	for _, c := range carets {
		sel = append(sel, gxui.CreateTextSelection(c, c, true))
	}
	e.Controller().SetSelections(sel)
}

func (e *CodeEditor) OnRename(callback func(newPath string)) {
	e.onRename = callback
}

func (e *CodeEditor) open(headerText string) {
	go e.watch()
	e.load(headerText)
}

func (e *CodeEditor) watcherSetup() {
	var err error
	e.watcher, err = fsw.New()
	if err != nil {
		log.Printf("Error creating new watcher: %s", err)
	}
}

func (e *CodeEditor) startWatch() error {
	err := e.watcher.Add(e.filepath)
	if os.IsNotExist(err) {
		err = e.waitForFileCreate()
		if err != nil {
			return err
		}
		return e.startWatch()
	}
	return nil
}

func (e *CodeEditor) watch() {
	if e.watcher == nil {
		return
	}
	err := e.startWatch()
	if err != nil {
		log.Printf("Error trying to watch %s for changes: %s", e.filepath, err)
		return
	}
	defer e.watcher.Remove(e.filepath)
	fileDir := filepath.Dir(e.filepath)
	err = e.watcher.Add(fileDir)
	if err != nil {
		log.Printf("Error trying to watch %s for changes: %s", fileDir, err)
		return
	}
	defer e.watcher.Remove(fileDir)
	for {
		ev, err := e.watcher.Next()
		if err != nil {
			log.Printf("Error from watcher: %s", err)
			return
		}
		if ev.Path != e.filepath {
			if !e.renamed || ev.Op != fsw.Create {
				continue
			}
			e.filepath = ev.Path
			e.onRename(e.filepath)
			e.open("")
			return
		}
		switch ev.Op {
		case fsw.Write:
			e.load("")
		case fsw.Rename:
			e.renamed = true
		case fsw.Remove:
			e.open("")
			return
		}
	}
}

func (e *CodeEditor) waitForFileCreate() error {
	dir := filepath.Dir(e.filepath)
	if err := os.MkdirAll(dir, 0750|os.ModeDir); err != nil {
		return err
	}
	if err := e.watcher.Add(dir); err != nil {
		return err
	}
	defer e.watcher.Remove(dir)

	for {
		ev, err := e.watcher.Next()
		if err != nil {
			return err
		}
		if ev.Path == e.filepath && ev.Op&fsw.Create == fsw.Create {
			return nil
		}
	}
}

func (e *CodeEditor) load(headerText string) {
	f, err := os.Open(e.filepath)
	if os.IsNotExist(err) {
		e.driver.Call(func() {
			e.SetText(headerText)
		})
		return
	}
	if err != nil {
		log.Printf("Error opening file %s: %s", e.filepath, err)
		return
	}
	defer f.Close()
	finfo, err := f.Stat()
	if err != nil {
		log.Printf("Error stating file %s: %s", e.filepath, err)
		return
	}
	e.setLastModified(finfo.ModTime())
	b, err := ioutil.ReadAll(f)
	if err != nil {
		log.Printf("Error reading file %s: %s", e.filepath, err)
		return
	}
	newText := string(b)
	if !strings.HasPrefix(newText, headerText) {
		log.Printf("%s: header text does not match requested header text", e.filepath)
	}
	e.driver.Call(func() {
		if e.Text() == newText {
			return
		}
		e.SetText(newText)
		if len(e.selections) > 0 {
			e.restorePositions()
		}
	})
}

func (e *CodeEditor) HasChanges() bool {
	return e.hasChanges
}

func (e *CodeEditor) LastKnownMTime() time.Time {
	e.lock.RLock()
	defer e.lock.RUnlock()
	return e.lastModified
}

func (e *CodeEditor) setLastModified(value time.Time) {
	e.lock.Lock()
	defer e.lock.Unlock()
	e.lastModified = value
}

func (e *CodeEditor) Filepath() string {
	return e.filepath
}

func (e *CodeEditor) FlushedChanges() {
	e.hasChanges = false
	e.setLastModified(time.Now())
}

func (e *CodeEditor) Elements() []interface{} {
	return []interface{}{
		e.Controller(),
	}
}

func (e *CodeEditor) SetSyntaxLayers(layers []input.SyntaxLayer) {
	defer e.syntaxTheme.Rainbow.Reset()
	sort.Slice(layers, func(i, j int) bool {
		return layers[i].Construct < layers[j].Construct
	})
	e.layers = layers
	gLayers := make(gxui.CodeSyntaxLayers, 0, len(layers))
	for _, l := range layers {
		highlight, found := e.syntaxTheme.Constructs[l.Construct]
		if !found {
			highlight = e.syntaxTheme.Rainbow.Next()
		}
		gLayer := gxui.CreateCodeSyntaxLayer()
		gLayer.SetColor(gxui.Color(highlight.Foreground))
		gLayer.SetBackgroundColor(gxui.Color(highlight.Background))
		for _, s := range l.Spans {
			gLayer.Add(s.Start, s.End-s.Start)
		}
		gLayers = append(gLayers, gLayer)
	}
	e.CodeEditor.SetSyntaxLayers(gLayers)
}

func (e *CodeEditor) SyntaxLayers() []input.SyntaxLayer {
	return e.layers
}

func (e *CodeEditor) Paint(c gxui.Canvas) {
	e.CodeEditor.Paint(c)

	if e.HasFocus() {
		r := e.Size().Rect()
		c.DrawRoundedRect(r, 3, 3, 3, 3, e.theme.FocusedStyle.Pen, e.theme.FocusedStyle.Brush)
	}
}

func (e *CodeEditor) storePositions() {
	e.selections = e.Controller().SelectionSlice()
	e.scrollPositions = math.Point{
		X: e.HorizOffset(),
		Y: e.ScrollOffset(),
	}
}

func (e *CodeEditor) restorePositions() {
	e.Controller().SetSelections(e.selections)
	e.SetHorizOffset(e.scrollPositions.X)
	e.SetScrollOffset(e.scrollPositions.Y)
}

func (e *CodeEditor) KeyPress(event gxui.KeyboardEvent) bool {
	defer e.storePositions()
	if event.Modifier != 0 && event.Modifier != gxui.ModShift {
		// ctrl-, alt-, and cmd-/super- keybindings should be dealt
		// with by the commands mapped to key bindings.
		return false
	}
	switch event.Key {
	case gxui.KeyPageUp, gxui.KeyPageDown:
		// These are all bindings that the TextBox handles fine.
		return e.TextBox.KeyPress(event)
	case gxui.KeyTab:
		// TODO: Gain knowledge about scope, so we know how much to indent.
		switch {
		case event.Modifier.Shift():
			e.Controller().UnindentSelection()
		default:
			e.Controller().IndentSelection()
		}
		return true
	case gxui.KeyEscape:
		// TODO: Keep track of some sort of concept of a "focused" caret and
		// focus that.
		e.Controller().SetCaret(e.Controller().FirstCaret())
	}
	return false
}

func (e *CodeEditor) KeyStroke(event gxui.KeyStrokeEvent) (consume bool) {
	return false
}

func (e *CodeEditor) CreateLine(theme gxui.Theme, index int) (mixins.TextBoxLine, gxui.Control) {
	lineNumber := theme.CreateLabel()
	lineNumber.SetText(fmt.Sprintf("%4d", index+1))

	line := &mixins.CodeEditorLine{}
	line.Init(line, theme, &e.CodeEditor, index)

	layout := theme.CreateLinearLayout()
	layout.SetDirection(gxui.LeftToRight)
	layout.AddChild(lineNumber)
	layout.AddChild(line)

	return line, layout
}
<commit_msg>set scrollround<commit_after>// This is free and unencumbered software released into the public
// domain.  For more information, see <http://unlicense.org> or the
// accompanying UNLICENSE file.

package editor

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/nelsam/gxui"
	"github.com/nelsam/gxui/math"
	"github.com/nelsam/gxui/mixins"
	"github.com/nelsam/gxui/themes/basic"
	"github.com/nelsam/vidar/commander/input"
	"github.com/nelsam/vidar/fsw"
	"github.com/nelsam/vidar/theme"
)

type CodeEditor struct {
	mixins.CodeEditor

	theme       *basic.Theme
	syntaxTheme theme.Theme
	driver      gxui.Driver

	lock         sync.RWMutex
	lastModified time.Time
	hasChanges   bool
	filepath     string

	watcher fsw.Watcher

	selections      []gxui.TextSelection
	scrollPositions math.Point
	layers          []input.SyntaxLayer

	renamed  bool
	onRename func(newPath string)
}

func (e *CodeEditor) Init(driver gxui.Driver, theme *basic.Theme, syntaxTheme theme.Theme, font gxui.Font, file, headerText string) {
	e.theme = theme
	e.syntaxTheme = syntaxTheme
	e.driver = driver

	e.CodeEditor.Init(e, driver, theme, font)
	e.CodeEditor.SetScrollBarEnabled(true)
	e.CodeEditor.SetScrollRound(true)
	e.SetDesiredWidth(math.MaxSize.W)
	e.watcherSetup()

	// TODO: move to hooks on the input.Handler
	e.OnTextChanged(func(changes []gxui.TextBoxEdit) {
		e.hasChanges = true
	})
	e.filepath = file
	e.open(headerText)

	e.SetTextColor(theme.TextBoxDefaultStyle.FontColor)
	e.SetMargin(math.Spacing{L: 3, T: 3, R: 3, B: 3})
	e.SetPadding(math.Spacing{L: 3, T: 3, R: 3, B: 3})
	e.SetBorderPen(gxui.TransparentPen)
}

func (e *CodeEditor) DataChanged(recreate bool) {
	e.List.DataChanged(recreate)
}

func (e *CodeEditor) Carets() []int {
	return e.Controller().Carets()
}

func (e *CodeEditor) SetCarets(carets ...int) {
	if len(carets) == 0 {
		e.Controller().ClearSelections()
		return
	}
	var sel []gxui.TextSelection
	for _, c := range carets {
		sel = append(sel, gxui.CreateTextSelection(c, c, true))
	}
	e.Controller().SetSelections(sel)
}

func (e *CodeEditor) OnRename(callback func(newPath string)) {
	e.onRename = callback
}

func (e *CodeEditor) open(headerText string) {
	go e.watch()
	e.load(headerText)
}

func (e *CodeEditor) watcherSetup() {
	var err error
	e.watcher, err = fsw.New()
	if err != nil {
		log.Printf("Error creating new watcher: %s", err)
	}
}

func (e *CodeEditor) startWatch() error {
	err := e.watcher.Add(e.filepath)
	if os.IsNotExist(err) {
		err = e.waitForFileCreate()
		if err != nil {
			return err
		}
		return e.startWatch()
	}
	return nil
}

func (e *CodeEditor) watch() {
	if e.watcher == nil {
		return
	}
	err := e.startWatch()
	if err != nil {
		log.Printf("Error trying to watch %s for changes: %s", e.filepath, err)
		return
	}
	defer e.watcher.Remove(e.filepath)
	fileDir := filepath.Dir(e.filepath)
	err = e.watcher.Add(fileDir)
	if err != nil {
		log.Printf("Error trying to watch %s for changes: %s", fileDir, err)
		return
	}
	defer e.watcher.Remove(fileDir)
	for {
		ev, err := e.watcher.Next()
		if err != nil {
			log.Printf("Error from watcher: %s", err)
			return
		}
		if ev.Path != e.filepath {
			if !e.renamed || ev.Op != fsw.Create {
				continue
			}
			e.filepath = ev.Path
			e.onRename(e.filepath)
			e.open("")
			return
		}
		switch ev.Op {
		case fsw.Write:
			e.load("")
		case fsw.Rename:
			e.renamed = true
		case fsw.Remove:
			e.open("")
			return
		}
	}
}

func (e *CodeEditor) waitForFileCreate() error {
	dir := filepath.Dir(e.filepath)
	if err := os.MkdirAll(dir, 0750|os.ModeDir); err != nil {
		return err
	}
	if err := e.watcher.Add(dir); err != nil {
		return err
	}
	defer e.watcher.Remove(dir)

	for {
		ev, err := e.watcher.Next()
		if err != nil {
			return err
		}
		if ev.Path == e.filepath && ev.Op&fsw.Create == fsw.Create {
			return nil
		}
	}
}

func (e *CodeEditor) load(headerText string) {
	f, err := os.Open(e.filepath)
	if os.IsNotExist(err) {
		e.driver.Call(func() {
			e.SetText(headerText)
		})
		return
	}
	if err != nil {
		log.Printf("Error opening file %s: %s", e.filepath, err)
		return
	}
	defer f.Close()
	finfo, err := f.Stat()
	if err != nil {
		log.Printf("Error stating file %s: %s", e.filepath, err)
		return
	}
	e.setLastModified(finfo.ModTime())
	b, err := ioutil.ReadAll(f)
	if err != nil {
		log.Printf("Error reading file %s: %s", e.filepath, err)
		return
	}
	newText := string(b)
	if !strings.HasPrefix(newText, headerText) {
		log.Printf("%s: header text does not match requested header text", e.filepath)
	}
	e.driver.Call(func() {
		if e.Text() == newText {
			return
		}
		e.SetText(newText)
		if len(e.selections) > 0 {
			e.restorePositions()
		}
	})
}

func (e *CodeEditor) HasChanges() bool {
	return e.hasChanges
}

func (e *CodeEditor) LastKnownMTime() time.Time {
	e.lock.RLock()
	defer e.lock.RUnlock()
	return e.lastModified
}

func (e *CodeEditor) setLastModified(value time.Time) {
	e.lock.Lock()
	defer e.lock.Unlock()
	e.lastModified = value
}

func (e *CodeEditor) Filepath() string {
	return e.filepath
}

func (e *CodeEditor) FlushedChanges() {
	e.hasChanges = false
	e.setLastModified(time.Now())
}

func (e *CodeEditor) Elements() []interface{} {
	return []interface{}{
		e.Controller(),
	}
}

func (e *CodeEditor) SetSyntaxLayers(layers []input.SyntaxLayer) {
	defer e.syntaxTheme.Rainbow.Reset()
	sort.Slice(layers, func(i, j int) bool {
		return layers[i].Construct < layers[j].Construct
	})
	e.layers = layers
	gLayers := make(gxui.CodeSyntaxLayers, 0, len(layers))
	for _, l := range layers {
		highlight, found := e.syntaxTheme.Constructs[l.Construct]
		if !found {
			highlight = e.syntaxTheme.Rainbow.Next()
		}
		gLayer := gxui.CreateCodeSyntaxLayer()
		gLayer.SetColor(gxui.Color(highlight.Foreground))
		gLayer.SetBackgroundColor(gxui.Color(highlight.Background))
		for _, s := range l.Spans {
			gLayer.Add(s.Start, s.End-s.Start)
		}
		gLayers = append(gLayers, gLayer)
	}
	e.CodeEditor.SetSyntaxLayers(gLayers)
}

func (e *CodeEditor) SyntaxLayers() []input.SyntaxLayer {
	return e.layers
}

func (e *CodeEditor) Paint(c gxui.Canvas) {
	e.CodeEditor.Paint(c)

	if e.HasFocus() {
		r := e.Size().Rect()
		c.DrawRoundedRect(r, 3, 3, 3, 3, e.theme.FocusedStyle.Pen, e.theme.FocusedStyle.Brush)
	}
}

func (e *CodeEditor) storePositions() {
	e.selections = e.Controller().SelectionSlice()
	e.scrollPositions = math.Point{
		X: e.HorizOffset(),
		Y: e.ScrollOffset(),
	}
}

func (e *CodeEditor) restorePositions() {
	e.Controller().SetSelections(e.selections)
	e.SetHorizOffset(e.scrollPositions.X)
	e.SetScrollOffset(e.scrollPositions.Y)
}

func (e *CodeEditor) KeyPress(event gxui.KeyboardEvent) bool {
	defer e.storePositions()
	if event.Modifier != 0 && event.Modifier != gxui.ModShift {
		// ctrl-, alt-, and cmd-/super- keybindings should be dealt
		// with by the commands mapped to key bindings.
		return false
	}
	switch event.Key {
	case gxui.KeyPageUp, gxui.KeyPageDown:
		// These are all bindings that the TextBox handles fine.
		return e.TextBox.KeyPress(event)
	case gxui.KeyTab:
		// TODO: Gain knowledge about scope, so we know how much to indent.
		switch {
		case event.Modifier.Shift():
			e.Controller().UnindentSelection()
		default:
			e.Controller().IndentSelection()
		}
		return true
	case gxui.KeyEscape:
		// TODO: Keep track of some sort of concept of a "focused" caret and
		// focus that.
		e.Controller().SetCaret(e.Controller().FirstCaret())
	}
	return false
}

func (e *CodeEditor) KeyStroke(event gxui.KeyStrokeEvent) (consume bool) {
	return false
}

func (e *CodeEditor) CreateLine(theme gxui.Theme, index int) (mixins.TextBoxLine, gxui.Control) {
	lineNumber := theme.CreateLabel()
	lineNumber.SetText(fmt.Sprintf("%4d", index+1))

	line := &mixins.CodeEditorLine{}
	line.Init(line, theme, &e.CodeEditor, index)

	layout := theme.CreateLinearLayout()
	layout.SetDirection(gxui.LeftToRight)
	layout.AddChild(lineNumber)
	layout.AddChild(line)

	return line, layout
}
<|endoftext|>


<commit_before>package storage

import (
	"context"
	"fmt"
	"io"
	"net/http"
	"os"
	"path"
	"strings"
	"time"

	"github.com/alecthomas/units"
	"github.com/ncw/swift"
	"github.com/ohsu-comp-bio/funnel/config"
	"github.com/ohsu-comp-bio/funnel/proto/tes"
	"github.com/ohsu-comp-bio/funnel/util"
	"github.com/ohsu-comp-bio/funnel/util/fsutil"
)

const swiftProtocol = "swift://"

// SwiftBackend provides access to an sw object store.
type SwiftBackend struct {
	conn      *swift.Connection
	chunkSize int64
}

// NewSwiftBackend creates an SwiftBackend client instance, give an endpoint URL
// and a set of authentication credentials.
func NewSwiftBackend(conf config.SwiftStorage) (Backend, error) {

	// Create a connection
	conn := &swift.Connection{
		UserName: conf.UserName,
		ApiKey:   conf.Password,
		AuthUrl:  conf.AuthURL,
		Tenant:   conf.TenantName,
		TenantId: conf.TenantID,
		Region:   conf.RegionName,
	}

	// Read environment variables and apply them to the Connection structure.
	// Won't overwrite any parameters which are already set in the Connection struct.
	err := conn.ApplyEnvironment()
	if err != nil {
		return nil, err
	}

	var chunkSize int64
	if conf.ChunkSizeBytes < int64(100*units.MB) {
		chunkSize = int64(100 * units.MB)
	} else if conf.ChunkSizeBytes > int64(5*units.GB) {
		chunkSize = int64(5 * units.GB)
	} else {
		chunkSize = conf.ChunkSizeBytes
	}

	b := &SwiftBackend{conn, chunkSize}
	return &retrier{
		backend: b,
		Retrier: &util.Retrier{
			MaxTries:            conf.MaxRetries,
			InitialInterval:     time.Second * 5,
			MaxInterval:         time.Minute * 5,
			Multiplier:          2.0,
			RandomizationFactor: 0.5,
			MaxElapsedTime:      0,
			ShouldRetry: func(err error) bool {
				// Retry on errors that swift names specifically.
				if err == swift.ObjectCorrupted || err == swift.TimeoutError {
					return true
				}
				// Retry on service unavailable.
				if se, ok := err.(*swift.Error); ok {
					return se.StatusCode == http.StatusServiceUnavailable
				}
				return false
			},
		},
	}, nil
}

// Get copies an object from storage to the host path.
func (sw *SwiftBackend) Get(ctx context.Context, rawurl string, hostPath string, class tes.FileType) error {
	url, err := sw.parse(rawurl)
	if err != nil {
		return err
	}

	var checkHash = true
	var headers swift.Headers

	switch class {
	case File:
		f, _, err := sw.conn.ObjectOpen(url.bucket, url.path, checkHash, headers)
		if err != nil {
			return err
		}
		defer f.Close()

		return sw.get(ctx, f, hostPath)

	case Directory:
		err := fsutil.EnsureDir(hostPath)
		if err != nil {
			return err
		}

		objs, err := sw.conn.ObjectsAll(url.bucket, &swift.ObjectsOpts{
			Prefix: url.path,
		})
		if err != nil {
			return err
		}
		if len(objs) == 0 {
			return ErrEmptyDirectory
		}

		for _, obj := range objs {
			if strings.HasSuffix(obj.Name, "/") {
				continue
			}
			f, _, err := sw.conn.ObjectOpen(url.bucket, obj.Name, checkHash, headers)
			if err != nil {
				return err
			}

			if err = sw.get(ctx, f, path.Join(hostPath, strings.TrimPrefix(obj.Name, url.path))); err != nil {
				if cerr := f.Close(); cerr != nil {
					return fmt.Errorf("%v; %v", err, cerr)
				}
				return err
			}

			if err = f.Close(); err != nil {
				return err
			}
		}

		return nil

	default:
		return fmt.Errorf("Unknown file class: %s", class)
	}
}

func (sw *SwiftBackend) get(ctx context.Context, src io.Reader, hostPath string) (err error) {
	err = fsutil.EnsurePath(hostPath)
	if err != nil {
		return err
	}
	dest, err := os.Create(hostPath)
	if err != nil {
		return err
	}
	defer func() {
		cerr := dest.Close()
		if cerr != nil {
			err = fmt.Errorf("%v; %v", err, cerr)
		}
	}()

	_, err = io.Copy(dest, fsutil.Reader(ctx, src))
	return err
}

// PutFile copies an object (file) from the host path to storage.
func (sw *SwiftBackend) PutFile(ctx context.Context, rawurl string, hostPath string) error {
	url, err := sw.parse(rawurl)
	if err != nil {
		return err
	}

	reader, err := os.Open(hostPath)
	if err != nil {
		return err
	}
	defer reader.Close()

	var writer io.WriteCloser
	var checkHash = true
	var hash string
	var contentType string
	var headers swift.Headers

	fSize := fileSize(hostPath)
	if fSize < int64(5*units.GB) {
		writer, err = sw.conn.ObjectCreate(url.bucket, url.path, checkHash, hash, contentType, headers)
	} else {
		writer, err = sw.conn.StaticLargeObjectCreateFile(&swift.LargeObjectOpts{
			Container:  url.bucket,
			ObjectName: url.path,
			CheckHash:  checkHash,
			Hash:       hash,
			Headers:    headers,
			ChunkSize:  sw.chunkSize,
		})
	}
	if err != nil {
		return err
	}
	defer func() {
		cerr := writer.Close()
		if cerr != nil {
			err = fmt.Errorf("%v; %v", err, cerr)
		}
	}()

	_, err = io.Copy(writer, fsutil.Reader(ctx, reader))
	return err
}

// SupportsGet indicates whether this backend supports GET storage request.
// For the Swift backend, the url must start with "swift://" and the bucket must exist
func (sw *SwiftBackend) SupportsGet(rawurl string, class tes.FileType) error {
	url, err := sw.parse(rawurl)
	if err != nil {
		return err
	}
	_, _, err = sw.conn.Container(url.bucket)
	if err != nil {
		return fmt.Errorf("swift: failed to find bucket: %s. error: %v", url.bucket, err)
	}
	return nil
}

// SupportsPut indicates whether this backend supports PUT storage request.
// For the Swift backend, the url must start with "swift://" and the bucket must exist
func (sw *SwiftBackend) SupportsPut(rawurl string, class tes.FileType) error {
	return sw.SupportsGet(rawurl, class)
}

func (sw *SwiftBackend) parse(rawurl string) (*urlparts, error) {
	ok := strings.HasPrefix(rawurl, swiftProtocol)
	if !ok {
		return nil, &ErrUnsupportedProtocol{"swift"}
	}

	path := strings.TrimPrefix(rawurl, swiftProtocol)
	if path == "" {
		return nil, &ErrInvalidURL{"swift"}
	}

	split := strings.SplitN(path, "/", 2)
	url := &urlparts{}
	if len(split) > 0 {
		url.bucket = split[0]
	}
	if len(split) == 2 {
		url.path = split[1]
	}
	return url, nil
}
<commit_msg>storage/swift: wrap errors with useful context<commit_after>package storage

import (
	"context"
	"fmt"
	"io"
	"net/http"
	"os"
	"path"
	"strings"
	"time"

	"github.com/alecthomas/units"
	"github.com/ncw/swift"
	"github.com/ohsu-comp-bio/funnel/config"
	"github.com/ohsu-comp-bio/funnel/proto/tes"
	"github.com/ohsu-comp-bio/funnel/util"
	"github.com/ohsu-comp-bio/funnel/util/fsutil"
)

const swiftProtocol = "swift://"

// SwiftBackend provides access to an sw object store.
type SwiftBackend struct {
	conn      *swift.Connection
	chunkSize int64
}

// NewSwiftBackend creates an SwiftBackend client instance, give an endpoint URL
// and a set of authentication credentials.
func NewSwiftBackend(conf config.SwiftStorage) (Backend, error) {

	// Create a connection
	conn := &swift.Connection{
		UserName: conf.UserName,
		ApiKey:   conf.Password,
		AuthUrl:  conf.AuthURL,
		Tenant:   conf.TenantName,
		TenantId: conf.TenantID,
		Region:   conf.RegionName,
	}

	// Read environment variables and apply them to the Connection structure.
	// Won't overwrite any parameters which are already set in the Connection struct.
	err := conn.ApplyEnvironment()
	if err != nil {
		return nil, err
	}

	var chunkSize int64
	if conf.ChunkSizeBytes < int64(100*units.MB) {
		chunkSize = int64(100 * units.MB)
	} else if conf.ChunkSizeBytes > int64(5*units.GB) {
		chunkSize = int64(5 * units.GB)
	} else {
		chunkSize = conf.ChunkSizeBytes
	}

	b := &SwiftBackend{conn, chunkSize}
	return &retrier{
		backend: b,
		Retrier: &util.Retrier{
			MaxTries:            conf.MaxRetries,
			InitialInterval:     time.Second * 5,
			MaxInterval:         time.Minute * 5,
			Multiplier:          2.0,
			RandomizationFactor: 0.5,
			MaxElapsedTime:      0,
			ShouldRetry: func(err error) bool {
				// Retry on errors that swift names specifically.
				if err == swift.ObjectCorrupted || err == swift.TimeoutError {
					return true
				}
				// Retry on service unavailable.
				if se, ok := err.(*swift.Error); ok {
					return se.StatusCode == http.StatusServiceUnavailable
				}
				return false
			},
		},
	}, nil
}

// Get copies an object from storage to the host path.
func (sw *SwiftBackend) Get(ctx context.Context, rawurl string, hostPath string, class tes.FileType) error {
	url, err := sw.parse(rawurl)
	if err != nil {
		return err
	}

	var checkHash = true
	var headers swift.Headers

	switch class {
	case File:
		f, _, err := sw.conn.ObjectOpen(url.bucket, url.path, checkHash, headers)
		if err != nil {
			return err
		}
		defer f.Close()

		return sw.get(ctx, f, hostPath)

	case Directory:
		err := fsutil.EnsureDir(hostPath)
		if err != nil {
			return err
		}

		objs, err := sw.conn.ObjectsAll(url.bucket, &swift.ObjectsOpts{
			Prefix: url.path,
		})
		if err != nil {
			return fmt.Errorf("listing objects by prefix: %s", err)
		}
		if len(objs) == 0 {
			return ErrEmptyDirectory
		}

		for _, obj := range objs {
			if strings.HasSuffix(obj.Name, "/") {
				continue
			}
			f, _, err := sw.conn.ObjectOpen(url.bucket, obj.Name, checkHash, headers)
			if err != nil {
				return fmt.Errorf("opening object %s: %s", obj.Name, err)
			}

			if err = sw.get(ctx, f, path.Join(hostPath, strings.TrimPrefix(obj.Name, url.path))); err != nil {
				if cerr := f.Close(); cerr != nil {
					return fmt.Errorf("closing object after get %v; %v", err, cerr)
				}
				return err
			}

			if err = f.Close(); err != nil {
				return fmt.Errorf("closing file: %s", err)
			}
		}

		return nil

	default:
		return fmt.Errorf("Unknown file class: %s", class)
	}
}

func (sw *SwiftBackend) get(ctx context.Context, src io.Reader, hostPath string) (err error) {
	err = fsutil.EnsurePath(hostPath)
	if err != nil {
		return err
	}
	dest, err := os.Create(hostPath)
	if err != nil {
		return err
	}
	defer func() {
		cerr := dest.Close()
		if cerr != nil {
			err = fmt.Errorf("%v; %v", err, cerr)
		}
	}()

	_, err = io.Copy(dest, fsutil.Reader(ctx, src))
	return err
}

// PutFile copies an object (file) from the host path to storage.
func (sw *SwiftBackend) PutFile(ctx context.Context, rawurl string, hostPath string) error {
	url, err := sw.parse(rawurl)
	if err != nil {
		return err
	}

	reader, err := os.Open(hostPath)
	if err != nil {
		return err
	}
	defer reader.Close()

	var writer io.WriteCloser
	var checkHash = true
	var hash string
	var contentType string
	var headers swift.Headers

	fSize := fileSize(hostPath)
	if fSize < int64(5*units.GB) {
		writer, err = sw.conn.ObjectCreate(url.bucket, url.path, checkHash, hash, contentType, headers)
	} else {
		writer, err = sw.conn.StaticLargeObjectCreateFile(&swift.LargeObjectOpts{
			Container:  url.bucket,
			ObjectName: url.path,
			CheckHash:  checkHash,
			Hash:       hash,
			Headers:    headers,
			ChunkSize:  sw.chunkSize,
		})
	}
	if err != nil {
		return err
	}
	defer func() {
		cerr := writer.Close()
		if cerr != nil {
			err = fmt.Errorf("%v; %v", err, cerr)
		}
	}()

	_, err = io.Copy(writer, fsutil.Reader(ctx, reader))
	return err
}

// SupportsGet indicates whether this backend supports GET storage request.
// For the Swift backend, the url must start with "swift://" and the bucket must exist
func (sw *SwiftBackend) SupportsGet(rawurl string, class tes.FileType) error {
	url, err := sw.parse(rawurl)
	if err != nil {
		return err
	}
	_, _, err = sw.conn.Container(url.bucket)
	if err != nil {
		return fmt.Errorf("swift: failed to find bucket: %s. error: %v", url.bucket, err)
	}
	return nil
}

// SupportsPut indicates whether this backend supports PUT storage request.
// For the Swift backend, the url must start with "swift://" and the bucket must exist
func (sw *SwiftBackend) SupportsPut(rawurl string, class tes.FileType) error {
	return sw.SupportsGet(rawurl, class)
}

func (sw *SwiftBackend) parse(rawurl string) (*urlparts, error) {
	ok := strings.HasPrefix(rawurl, swiftProtocol)
	if !ok {
		return nil, &ErrUnsupportedProtocol{"swift"}
	}

	path := strings.TrimPrefix(rawurl, swiftProtocol)
	if path == "" {
		return nil, &ErrInvalidURL{"swift"}
	}

	split := strings.SplitN(path, "/", 2)
	url := &urlparts{}
	if len(split) > 0 {
		url.bucket = split[0]
	}
	if len(split) == 2 {
		url.path = split[1]
	}
	return url, nil
}
<|endoftext|>


<commit_before>package storage

import (
	"fmt"
	"strconv"
	"strings"
)

type Config struct {
	// Default connection settings
	Hostname     string
	Hostnames    []string
	Port         uint16 // 0 to 65,535
	DatabaseName string

	// Credential Access
	Username string
	Password string

	// Replica Set
	Replset string
}

// connectionURL generates a formatted Mongo Connection URL
func ConnectionURI(cfg *Config) string {
	connectionString := "mongodb://"
	credentials := ""

	if cfg.Username != "" && cfg.Password != "" {
		credentials = fmt.Sprintf("%s:%s@", cfg.Username, cfg.Password)
	}

	hosts := ""
	if cfg.Hostnames != nil && cfg.Hostname == "" {
		hosts = strings.Join(cfg.Hostnames, fmt.Sprintf(":%s,", strconv.Itoa(int(cfg.Port))))
		cfg.Hostname = hosts
	}

	connectionString = fmt.Sprintf("%s%s%s:%s/%s",
		connectionString,
		credentials,
		cfg.Hostname,
		strconv.Itoa(int(cfg.Port)),
		cfg.DatabaseName,
	)

	if cfg.Replset != "" {
		connectionString += "?replicaSet=" + cfg.Replset
	}

	return connectionString
}

// NewDatastore returns a connection to your configured MongoDB
func NewDatastore(cfg *Config) string {
	return ConnectionURI(cfg)
}
<commit_msg>:zap: Fixed exported comment, fix missing comment from exported struct<commit_after>package storage

import (
	"fmt"
	"strconv"
	"strings"
)

// Config provides a way to define the specific pieces that make up a mongo connection
type Config struct {
	// Default connection settings
	Hostname     string
	Hostnames    []string
	Port         uint16 // 0 to 65,535
	DatabaseName string

	// Credential Access
	Username string
	Password string

	// Replica Set
	Replset string
}

// ConnectionURL generates a formatted Mongo Connection URL
func ConnectionURI(cfg *Config) string {
	connectionString := "mongodb://"
	credentials := ""

	if cfg.Username != "" && cfg.Password != "" {
		credentials = fmt.Sprintf("%s:%s@", cfg.Username, cfg.Password)
	}

	hosts := ""
	if cfg.Hostnames != nil && cfg.Hostname == "" {
		hosts = strings.Join(cfg.Hostnames, fmt.Sprintf(":%s,", strconv.Itoa(int(cfg.Port))))
		cfg.Hostname = hosts
	}

	connectionString = fmt.Sprintf("%s%s%s:%s/%s",
		connectionString,
		credentials,
		cfg.Hostname,
		strconv.Itoa(int(cfg.Port)),
		cfg.DatabaseName,
	)

	if cfg.Replset != "" {
		connectionString += "?replicaSet=" + cfg.Replset
	}

	return connectionString
}

// NewDatastore returns a connection to your configured MongoDB
func NewDatastore(cfg *Config) string {
	return ConnectionURI(cfg)
}
<|endoftext|>


<commit_before>package store

import (
	"testing"
	"time"
	. "github.com/smartystreets/goconvey/convey"
)

func TestMessageQueue(t *testing.T) {
	Convey("Given a message queue", t, func() {
		mq := NewQueue()
		Convey("Given a blacklist", func() {
			mq.Blacklist("foo", "bar", "sojin")
			Convey("It should ignore matching public requests", func() {
				id, ch := mq.Subscribe("foo", false)
				So(id, ShouldEqual, 0)
				So(ch, ShouldBeNil)
				mq.Update("foo", "bar", true)
				mq.Update("foo", "bar", false)
			})
			Convey("It should honor matching private requests", func() {
				_, ch := mq.Subscribe("foo", true)
				So(ch, ShouldNotBeNil)

				var res string
				go func() {
					select {
					case res = <-ch:
					}
				}()

				mq.Update("foo", "bar", true)
				So(res, ShouldEqual, "bar")

				res = ""
				go func() {
					select {
					case res = <-ch:
					case <-time.After(100 * time.Millisecond):
					}
				}()

				mq.Update("foo", "bar", false)
				So(res, ShouldBeEmpty)
			})
		})
	})
}
<commit_msg>added blacklist feature to the queue<commit_after>package store

import (
	"testing"
	"time"
	. "github.com/smartystreets/goconvey/convey"
)

func TestMessageQueue(t *testing.T) {
	Convey("Given a message queue", t, func() {
		mq := NewQueue()
		Convey("Given a blacklist", func() {
			mq.Blacklist("foo", "bar", "sojin")

			Convey("It should ignore matching public requests", func() {
				id, ch := mq.Subscribe("foo", false)
				So(id, ShouldEqual, 0)
				So(ch, ShouldBeNil)
				mq.Update("foo", "bar", true)
				mq.Update("foo", "bar", false)
			})

			Convey("It should honor matching private requests", func() {
				_, ch := mq.Subscribe("foo", true)
				So(ch, ShouldNotBeNil)

				var res string
				go func() {
					select {
					case res = <-ch:
					}
				}()

				mq.Update("foo", "bar", true)
				So(res, ShouldEqual, "bar")

				res = ""
				go func() {
					select {
					case res = <-ch:
					case <-time.After(100 * time.Millisecond):
					}
				}()

				mq.Update("foo", "bar", false)
				So(res, ShouldBeEmpty)
			})
		})
	})
}
<|endoftext|>


<commit_before>package models

import (
	"database/sql"
	"errors"
	"github.com/asaskevich/govalidator"

	"github.com/eirka/eirka-libs/db"
	e "github.com/eirka/eirka-libs/errors"
)

// EmailModel contains new email
type EmailModel struct {
	Uid          uint
	Name         string
	Email        string
	CurrentEmail string
}

// check struct validity
func (e *EmailModel) IsValid() bool {

	if e.Uid == 0 {
		return false
	}

	if e.Name == "" {
		return false
	}

	if e.Email == "" {
		return false
	}

	if e.CurrentEmail == "" {
		return false
	}

	return true

}

// Validate will check the provided email
func (r *EmailModel) Validate() (err error) {

	// Validate email
	if !govalidator.IsEmail(r.Email) {
		return e.ErrInvalidEmail
	}

	// Get Database handle
	dbase, err := db.GetDb()
	if err != nil {
		return
	}

	// get current email
	err = dbase.QueryRow("SELECT user_name,user_email FROM users WHERE user_id = ?", r.Uid).Scan(&r.Name, &r.CurrentEmail)
	if err == sql.ErrNoRows {
		return e.ErrUserNotExist
	} else if err != nil {
		return e.ErrInternalError
	}

	// we dont care if the email is already the same
	if r.Email == r.CurrentEmail {
		return e.ErrEmailSame
	}

	return

}

// update email
func (r *EmailModel) Update() (err error) {

	// check model validity
	if !r.IsValid() {
		return errors.New("EmailModel is not valid")
	}

	// Get Database handle
	dbase, err := db.GetDb()
	if err != nil {
		return
	}

	ps1, err := dbase.Prepare("UPDATE users SET user_email = ? WHERE user_id = ?")
	if err != nil {
		return
	}
	defer ps1.Close()

	_, err = ps1.Exec(r.Email, r.Uid)
	if err != nil {
		return
	}

	return

}
<commit_msg>add email test<commit_after>package models

import (
	"database/sql"
	"errors"
	"github.com/asaskevich/govalidator"

	"github.com/eirka/eirka-libs/db"
	e "github.com/eirka/eirka-libs/errors"
)

// EmailModel contains new email
type EmailModel struct {
	Uid          uint
	Name         string
	Email        string
	CurrentEmail string
}

// check struct validity
func (e *EmailModel) IsValid() bool {

	if e.Uid == 0 {
		return false
	}

	if e.Name == "" {
		return false
	}

	if e.Email == "" {
		return false
	}

	return true

}

// Validate will check the provided email
func (r *EmailModel) Validate() (err error) {

	// Validate email
	if !govalidator.IsEmail(r.Email) {
		return e.ErrInvalidEmail
	}

	// Get Database handle
	dbase, err := db.GetDb()
	if err != nil {
		return
	}

	// get current email
	err = dbase.QueryRow("SELECT user_name,user_email FROM users WHERE user_id = ?", r.Uid).Scan(&r.Name, &r.CurrentEmail)
	if err == sql.ErrNoRows {
		return e.ErrUserNotExist
	} else if err != nil {
		return e.ErrInternalError
	}

	// we dont care if the email is already the same
	if r.Email == r.CurrentEmail {
		return e.ErrEmailSame
	}

	return

}

// update email
func (r *EmailModel) Update() (err error) {

	// check model validity
	if !r.IsValid() {
		return errors.New("EmailModel is not valid")
	}

	// Get Database handle
	dbase, err := db.GetDb()
	if err != nil {
		return
	}

	ps1, err := dbase.Prepare("UPDATE users SET user_email = ? WHERE user_id = ?")
	if err != nil {
		return
	}
	defer ps1.Close()

	_, err = ps1.Exec(r.Email, r.Uid)
	if err != nil {
		return
	}

	return

}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"

	"github.com/gonuts/commander"
	"github.com/gonuts/flag"
	//gocfg "github.com/sbinet/go-config/config"
	_ "github.com/mana-fwk/git-tools/utils"
)

func hwaf_make_cmd_pkg_rm() *commander.Command {
	cmd := &commander.Command{
		Run:       hwaf_run_cmd_pkg_rm,
		UsageLine: "rm [options] <local-pkg-name>",
		Short:     "remove a package from the current workarea",
		Long: `
rm removes a package from the current workarea.

ex:
 $ hwaf pkg rm ./src/foo/pkg
 $ hwaf pkg rm Control/AthenaKernel
`,
		Flag: *flag.NewFlagSet("hwaf-pkg-rm", flag.ExitOnError),
	}
	cmd.Flag.Bool("q", true, "only print error and warning messages, all other output will be suppressed")

	return cmd
}

func hwaf_run_cmd_pkg_rm(cmd *commander.Command, args []string) {
	var err error
	n := "hwaf-pkg-" + cmd.Name()
	switch len(args) {
	case 0:
		err = fmt.Errorf("%s: you need to give (at least) one package name to remove", n)
		handle_err(err)
	}

	quiet := cmd.Flag.Lookup("q").Value.Get().(bool)

	cfg, err := g_ctx.LocalCfg()
	handle_err(err)

	srcdir := "src"
	if cfg.HasOption("hwaf-cfg", "cmtpkgs") {
		srcdir, err = cfg.String("hwaf-cfg", "cmtpkgs")
		handle_err(err)
	}

	do_rm := func(pkgname string) error {
		var err error

		pkgname = os.ExpandEnv(pkgname)
		pkgname = filepath.Clean(pkgname)
		if !quiet {
			fmt.Printf("%s: remove package [%s]...\n", n, pkgname)
		}

		pkg := pkgname
		if !g_ctx.PkgDb.HasPkg(pkg) {
			pkg = filepath.Join(srcdir, pkgname)
		}
		if !g_ctx.PkgDb.HasPkg(pkg) {
			return fmt.Errorf("%s: no such package [%s] in db", n, pkg)
		}

		// people may have already removed it via a simple 'rm -rf foo'...
		//
		// if !path_exists(pkg) {
		// 	pkg = filepath.Join(srcdir, pkgname)
		// 	if !path_exists(pkg) {
		// 		err = fmt.Errorf("%s: no such package [%s]", n, pkgname)
		// 		handle_err(err)
		// 	}
		// }

		if !g_ctx.PkgDb.HasPkg(pkg) {
			return fmt.Errorf("%s: no such package [%s] in db", n, pkg)
		}

		vcspkg, err := g_ctx.PkgDb.GetPkg(pkg)
		if err != nil {
			return err
		}

		switch vcspkg.Type {

		case "git":
			rmcmd := []string{"rm-submodule"}
			if !quiet {
				rmcmd = append(rmcmd, "--verbose")
			}
			rmcmd = append(rmcmd, pkg)
			git := exec.Command("git", rmcmd...)
			if !quiet {
				git.Stdin = os.Stdin
				git.Stdout = os.Stdout
				git.Stderr = os.Stderr
			}
			err = git.Run()
			if err != nil {
				return err
			}

		case "svn":
			if path_exists(vcspkg.Path) {
				err = os.RemoveAll(vcspkg.Path)
				if err != nil {
					return err
				}
				path := vcspkg.Path
				// clean-up dir if empty...
				for {
					path = filepath.Dir(path)
					if path == srcdir {
						break
					}
					content, err := filepath.Glob(filepath.Join(path, "*"))
					if err != nil {
						return err
					}
					if len(content) > 0 {
						break
					}
					err = os.RemoveAll(path)
					if err != nil {
						return err
					}
				}
			}

		case "local":
			if path_exists(vcspkg.Path) {
				err = os.RemoveAll(vcspkg.Path)
				if err != nil {
					return err
				}
				path := vcspkg.Path
				// clean-up dir if empty...
				for {
					path = filepath.Dir(path)
					if path == srcdir {
						break
					}
					content, err := filepath.Glob(filepath.Join(path, "*"))
					if err != nil {
						return err
					}
					if len(content) > 0 {
						break
					}
					err = os.RemoveAll(path)
					if err != nil {
						return err
					}
				}
			}

		default:
			return fmt.Errorf("%s: VCS of type [%s] is not handled", n, vcspkg.Type)
		}

		err = g_ctx.PkgDb.Remove(pkg)
		if err != nil {
			return err
		}

		if !quiet {
			fmt.Printf("%s: remove package [%s]... [ok]\n", n, pkgname)
		}
		return nil
	}

	for _, pkgname := range args {
		err := do_rm(pkgname)
		handle_err(err)
	}
}

// EOF
<commit_msg>pkg-rm: remove multiple packages in one go. update doc.<commit_after>package main

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"

	"github.com/gonuts/commander"
	"github.com/gonuts/flag"
	//gocfg "github.com/sbinet/go-config/config"
	_ "github.com/mana-fwk/git-tools/utils"
)

func hwaf_make_cmd_pkg_rm() *commander.Command {
	cmd := &commander.Command{
		Run:       hwaf_run_cmd_pkg_rm,
		UsageLine: "rm [options] <local-pkg-name> [<pkg2> [...]]",
		Short:     "remove a package from the current workarea",
		Long: `
rm removes a package from the current workarea.

ex:
 $ hwaf pkg rm ./src/foo/pkg
 $ hwaf pkg rm Control/AthenaKernel
 $ hwaf pkg rm Control/AthenaKernel Control/AthenaServices
`,
		Flag: *flag.NewFlagSet("hwaf-pkg-rm", flag.ExitOnError),
	}
	cmd.Flag.Bool("q", true, "only print error and warning messages, all other output will be suppressed")

	return cmd
}

func hwaf_run_cmd_pkg_rm(cmd *commander.Command, args []string) {
	var err error
	n := "hwaf-pkg-" + cmd.Name()
	switch len(args) {
	case 0:
		err = fmt.Errorf("%s: you need to give (at least) one package name to remove", n)
		handle_err(err)
	}

	quiet := cmd.Flag.Lookup("q").Value.Get().(bool)

	cfg, err := g_ctx.LocalCfg()
	handle_err(err)

	srcdir := "src"
	if cfg.HasOption("hwaf-cfg", "cmtpkgs") {
		srcdir, err = cfg.String("hwaf-cfg", "cmtpkgs")
		handle_err(err)
	}

	do_rm := func(pkgname string) error {
		var err error

		pkgname = os.ExpandEnv(pkgname)
		pkgname = filepath.Clean(pkgname)
		if !quiet {
			fmt.Printf("%s: remove package [%s]...\n", n, pkgname)
		}

		pkg := pkgname
		if !g_ctx.PkgDb.HasPkg(pkg) {
			pkg = filepath.Join(srcdir, pkgname)
		}
		if !g_ctx.PkgDb.HasPkg(pkg) {
			return fmt.Errorf("%s: no such package [%s] in db", n, pkg)
		}

		// people may have already removed it via a simple 'rm -rf foo'...
		//
		// if !path_exists(pkg) {
		// 	pkg = filepath.Join(srcdir, pkgname)
		// 	if !path_exists(pkg) {
		// 		err = fmt.Errorf("%s: no such package [%s]", n, pkgname)
		// 		handle_err(err)
		// 	}
		// }

		if !g_ctx.PkgDb.HasPkg(pkg) {
			return fmt.Errorf("%s: no such package [%s] in db", n, pkg)
		}

		vcspkg, err := g_ctx.PkgDb.GetPkg(pkg)
		if err != nil {
			return err
		}

		switch vcspkg.Type {

		case "git":
			rmcmd := []string{"rm-submodule"}
			if !quiet {
				rmcmd = append(rmcmd, "--verbose")
			}
			rmcmd = append(rmcmd, pkg)
			git := exec.Command("git", rmcmd...)
			if !quiet {
				git.Stdin = os.Stdin
				git.Stdout = os.Stdout
				git.Stderr = os.Stderr
			}
			err = git.Run()
			if err != nil {
				return err
			}

		case "svn":
			if path_exists(vcspkg.Path) {
				err = os.RemoveAll(vcspkg.Path)
				if err != nil {
					return err
				}
				path := vcspkg.Path
				// clean-up dir if empty...
				for {
					path = filepath.Dir(path)
					if path == srcdir {
						break
					}
					content, err := filepath.Glob(filepath.Join(path, "*"))
					if err != nil {
						return err
					}
					if len(content) > 0 {
						break
					}
					err = os.RemoveAll(path)
					if err != nil {
						return err
					}
				}
			}

		case "local":
			if path_exists(vcspkg.Path) {
				err = os.RemoveAll(vcspkg.Path)
				if err != nil {
					return err
				}
				path := vcspkg.Path
				// clean-up dir if empty...
				for {
					path = filepath.Dir(path)
					if path == srcdir {
						break
					}
					content, err := filepath.Glob(filepath.Join(path, "*"))
					if err != nil {
						return err
					}
					if len(content) > 0 {
						break
					}
					err = os.RemoveAll(path)
					if err != nil {
						return err
					}
				}
			}

		default:
			return fmt.Errorf("%s: VCS of type [%s] is not handled", n, vcspkg.Type)
		}

		err = g_ctx.PkgDb.Remove(pkg)
		if err != nil {
			return err
		}

		if !quiet {
			fmt.Printf("%s: remove package [%s]... [ok]\n", n, pkgname)
		}
		return nil
	}

	for _, pkgname := range args {
		err := do_rm(pkgname)
		handle_err(err)
	}
}

// EOF
<|endoftext|>


<commit_before>package dht

import (
	"errors"
	"time"

	inet "github.com/jbenet/go-ipfs/net"
	peer "github.com/jbenet/go-ipfs/peer"
	pb "github.com/jbenet/go-ipfs/routing/dht/pb"
	ctxutil "github.com/jbenet/go-ipfs/util/ctx"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
	ggio "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/gogoprotobuf/io"
)

// handleNewStream implements the inet.StreamHandler
func (dht *IpfsDHT) handleNewStream(s inet.Stream) {
	go dht.handleNewMessage(s)
}

func (dht *IpfsDHT) handleNewMessage(s inet.Stream) {
	defer s.Close()

	ctx := dht.Context()
	cr := ctxutil.NewReader(ctx, s) // ok to use. we defer close stream in this func
	cw := ctxutil.NewWriter(ctx, s) // ok to use. we defer close stream in this func
	r := ggio.NewDelimitedReader(cr, inet.MessageSizeMax)
	w := ggio.NewDelimitedWriter(cw)
	mPeer := s.Conn().RemotePeer()

	// receive msg
	pmes := new(pb.Message)
	if err := r.ReadMsg(pmes); err != nil {
		log.Error("Error unmarshaling data")
		return
	}
	// update the peer (on valid msgs only)
	dht.Update(ctx, mPeer)

	log.Event(ctx, "foo", dht.self, mPeer, pmes)

	// get handler for this msg type.
	handler := dht.handlerForMsgType(pmes.GetType())
	if handler == nil {
		log.Error("got back nil handler from handlerForMsgType")
		return
	}

	// dispatch handler.
	rpmes, err := handler(ctx, mPeer, pmes)
	if err != nil {
		log.Errorf("handle message error: %s", err)
		return
	}

	// if nil response, return it before serializing
	if rpmes == nil {
		log.Warning("Got back nil response from request.")
		return
	}

	// send out response msg
	if err := w.WriteMsg(rpmes); err != nil {
		log.Errorf("send response error: %s", err)
		return
	}

	return
}

// sendRequest sends out a request, but also makes sure to
// measure the RTT for latency measurements.
func (dht *IpfsDHT) sendRequest(ctx context.Context, p peer.ID, pmes *pb.Message) (*pb.Message, error) {

	log.Debugf("%s dht starting stream", dht.self)
	s, err := dht.network.NewStream(inet.ProtocolDHT, p)
	if err != nil {
		return nil, err
	}
	defer s.Close()

	cr := ctxutil.NewReader(ctx, s) // ok to use. we defer close stream in this func
	cw := ctxutil.NewWriter(ctx, s) // ok to use. we defer close stream in this func
	r := ggio.NewDelimitedReader(cr, inet.MessageSizeMax)
	w := ggio.NewDelimitedWriter(cw)

	start := time.Now()

	log.Debugf("%s writing", dht.self)
	if err := w.WriteMsg(pmes); err != nil {
		return nil, err
	}
	log.Event(ctx, "dhtSentMessage", dht.self, p, pmes)

	log.Debugf("%s reading", dht.self)
	defer log.Debugf("%s done", dht.self)

	rpmes := new(pb.Message)
	if err := r.ReadMsg(rpmes); err != nil {
		return nil, err
	}
	if rpmes == nil {
		return nil, errors.New("no response to request")
	}

	dht.peerstore.RecordLatency(p, time.Since(start))
	log.Event(ctx, "dhtReceivedMessage", dht.self, p, rpmes)
	return rpmes, nil
}

// sendMessage sends out a message
func (dht *IpfsDHT) sendMessage(ctx context.Context, p peer.ID, pmes *pb.Message) error {

	log.Debugf("%s dht starting stream", dht.self)
	s, err := dht.network.NewStream(inet.ProtocolDHT, p)
	if err != nil {
		return err
	}
	defer s.Close()

	cw := ctxutil.NewWriter(ctx, s) // ok to use. we defer close stream in this func
	w := ggio.NewDelimitedWriter(cw)

	log.Debugf("%s writing", dht.self)
	if err := w.WriteMsg(pmes); err != nil {
		return err
	}
	log.Event(ctx, "dhtSentMessage", dht.self, p, pmes)
	log.Debugf("%s done", dht.self)
	return nil
}
<commit_msg>dht: update on every received message<commit_after>package dht

import (
	"errors"
	"time"

	inet "github.com/jbenet/go-ipfs/net"
	peer "github.com/jbenet/go-ipfs/peer"
	pb "github.com/jbenet/go-ipfs/routing/dht/pb"
	ctxutil "github.com/jbenet/go-ipfs/util/ctx"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
	ggio "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/gogoprotobuf/io"
)

// handleNewStream implements the inet.StreamHandler
func (dht *IpfsDHT) handleNewStream(s inet.Stream) {
	go dht.handleNewMessage(s)
}

func (dht *IpfsDHT) handleNewMessage(s inet.Stream) {
	defer s.Close()

	ctx := dht.Context()
	cr := ctxutil.NewReader(ctx, s) // ok to use. we defer close stream in this func
	cw := ctxutil.NewWriter(ctx, s) // ok to use. we defer close stream in this func
	r := ggio.NewDelimitedReader(cr, inet.MessageSizeMax)
	w := ggio.NewDelimitedWriter(cw)
	mPeer := s.Conn().RemotePeer()

	// receive msg
	pmes := new(pb.Message)
	if err := r.ReadMsg(pmes); err != nil {
		log.Error("Error unmarshaling data")
		return
	}

	// update the peer (on valid msgs only)
	dht.updateFromMessage(ctx, mPeer, pmes)

	log.Event(ctx, "foo", dht.self, mPeer, pmes)

	// get handler for this msg type.
	handler := dht.handlerForMsgType(pmes.GetType())
	if handler == nil {
		log.Error("got back nil handler from handlerForMsgType")
		return
	}

	// dispatch handler.
	rpmes, err := handler(ctx, mPeer, pmes)
	if err != nil {
		log.Errorf("handle message error: %s", err)
		return
	}

	// if nil response, return it before serializing
	if rpmes == nil {
		log.Warning("Got back nil response from request.")
		return
	}

	// send out response msg
	if err := w.WriteMsg(rpmes); err != nil {
		log.Errorf("send response error: %s", err)
		return
	}

	return
}

// sendRequest sends out a request, but also makes sure to
// measure the RTT for latency measurements.
func (dht *IpfsDHT) sendRequest(ctx context.Context, p peer.ID, pmes *pb.Message) (*pb.Message, error) {

	log.Debugf("%s dht starting stream", dht.self)
	s, err := dht.network.NewStream(inet.ProtocolDHT, p)
	if err != nil {
		return nil, err
	}
	defer s.Close()

	cr := ctxutil.NewReader(ctx, s) // ok to use. we defer close stream in this func
	cw := ctxutil.NewWriter(ctx, s) // ok to use. we defer close stream in this func
	r := ggio.NewDelimitedReader(cr, inet.MessageSizeMax)
	w := ggio.NewDelimitedWriter(cw)

	start := time.Now()

	log.Debugf("%s writing", dht.self)
	if err := w.WriteMsg(pmes); err != nil {
		return nil, err
	}
	log.Event(ctx, "dhtSentMessage", dht.self, p, pmes)

	log.Debugf("%s reading", dht.self)
	defer log.Debugf("%s done", dht.self)

	rpmes := new(pb.Message)
	if err := r.ReadMsg(rpmes); err != nil {
		return nil, err
	}
	if rpmes == nil {
		return nil, errors.New("no response to request")
	}

	// update the peer (on valid msgs only)
	dht.updateFromMessage(ctx, p, rpmes)

	dht.peerstore.RecordLatency(p, time.Since(start))
	log.Event(ctx, "dhtReceivedMessage", dht.self, p, rpmes)
	return rpmes, nil
}

// sendMessage sends out a message
func (dht *IpfsDHT) sendMessage(ctx context.Context, p peer.ID, pmes *pb.Message) error {

	log.Debugf("%s dht starting stream", dht.self)
	s, err := dht.network.NewStream(inet.ProtocolDHT, p)
	if err != nil {
		return err
	}
	defer s.Close()

	cw := ctxutil.NewWriter(ctx, s) // ok to use. we defer close stream in this func
	w := ggio.NewDelimitedWriter(cw)

	log.Debugf("%s writing", dht.self)
	if err := w.WriteMsg(pmes); err != nil {
		return err
	}
	log.Event(ctx, "dhtSentMessage", dht.self, p, pmes)
	log.Debugf("%s done", dht.self)
	return nil
}

func (dht *IpfsDHT) updateFromMessage(ctx context.Context, p peer.ID, mes *pb.Message) error {
	dht.Update(ctx, p)
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2018 Vector Creations Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package consumers

import (
	"context"
	"encoding/json"

	"github.com/matrix-org/dendrite/appservice/storage"
	"github.com/matrix-org/dendrite/appservice/types"
	"github.com/matrix-org/dendrite/roomserver/api"
	"github.com/matrix-org/dendrite/setup/config"
	"github.com/matrix-org/dendrite/setup/jetstream"
	"github.com/matrix-org/dendrite/setup/process"
	"github.com/matrix-org/gomatrixserverlib"
	"github.com/nats-io/nats.go"

	log "github.com/sirupsen/logrus"
)

// OutputRoomEventConsumer consumes events that originated in the room server.
type OutputRoomEventConsumer struct {
	ctx          context.Context
	jetstream    nats.JetStreamContext
	durable      string
	topic        string
	asDB         storage.Database
	rsAPI        api.AppserviceRoomserverAPI
	serverName   string
	workerStates []types.ApplicationServiceWorkerState
}

// NewOutputRoomEventConsumer creates a new OutputRoomEventConsumer. Call
// Start() to begin consuming from room servers.
func NewOutputRoomEventConsumer(
	process *process.ProcessContext,
	cfg *config.Dendrite,
	js nats.JetStreamContext,
	appserviceDB storage.Database,
	rsAPI api.AppserviceRoomserverAPI,
	workerStates []types.ApplicationServiceWorkerState,
) *OutputRoomEventConsumer {
	return &OutputRoomEventConsumer{
		ctx:          process.Context(),
		jetstream:    js,
		durable:      cfg.Global.JetStream.Durable("AppserviceRoomserverConsumer"),
		topic:        cfg.Global.JetStream.Prefixed(jetstream.OutputRoomEvent),
		asDB:         appserviceDB,
		rsAPI:        rsAPI,
		serverName:   string(cfg.Global.ServerName),
		workerStates: workerStates,
	}
}

// Start consuming from room servers
func (s *OutputRoomEventConsumer) Start() error {
	return jetstream.JetStreamConsumer(
		s.ctx, s.jetstream, s.topic, s.durable, s.onMessage,
		nats.DeliverAll(), nats.ManualAck(),
	)
}

// onMessage is called when the appservice component receives a new event from
// the room server output log.
func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool {
	// Parse out the event JSON
	var output api.OutputEvent
	if err := json.Unmarshal(msg.Data, &output); err != nil {
		// If the message was invalid, log it and move on to the next message in the stream
		log.WithError(err).Errorf("roomserver output log: message parse failure")
		return true
	}

	if output.Type != api.OutputTypeNewRoomEvent || output.NewRoomEvent == nil {
		return true
	}

	newEventID := output.NewRoomEvent.Event.EventID()
	events := make([]*gomatrixserverlib.HeaderedEvent, 0, len(output.NewRoomEvent.AddsStateEventIDs))
	events = append(events, output.NewRoomEvent.Event)
	if len(output.NewRoomEvent.AddsStateEventIDs) > 0 {
		eventsReq := &api.QueryEventsByIDRequest{
			EventIDs: make([]string, 0, len(output.NewRoomEvent.AddsStateEventIDs)),
		}
		eventsRes := &api.QueryEventsByIDResponse{}
		for _, eventID := range output.NewRoomEvent.AddsStateEventIDs {
			if eventID != newEventID {
				eventsReq.EventIDs = append(eventsReq.EventIDs, eventID)
			}
		}
		if len(eventsReq.EventIDs) > 0 {
			if err := s.rsAPI.QueryEventsByID(s.ctx, eventsReq, eventsRes); err != nil {
				return false
			}
			events = append(events, eventsRes.Events...)
		}
	}

	// Send event to any relevant application services
	if err := s.filterRoomserverEvents(context.TODO(), events); err != nil {
		log.WithError(err).Errorf("roomserver output log: filter error")
		return true
	}

	return true
}

// filterRoomserverEvents takes in events and decides whether any of them need
// to be passed on to an external application service. It does this by checking
// each namespace of each registered application service, and if there is a
// match, adds the event to the queue for events to be sent to a particular
// application service.
func (s *OutputRoomEventConsumer) filterRoomserverEvents(
	ctx context.Context,
	events []*gomatrixserverlib.HeaderedEvent,
) error {
	for _, ws := range s.workerStates {
		for _, event := range events {
			// Check if this event is interesting to this application service
			if s.appserviceIsInterestedInEvent(ctx, event, ws.AppService) {
				// Queue this event to be sent off to the application service
				if err := s.asDB.StoreEvent(ctx, ws.AppService.ID, event); err != nil {
					log.WithError(err).Warn("failed to insert incoming event into appservices database")
					return err
				} else {
					// Tell our worker to send out new messages by updating remaining message
					// count and waking them up with a broadcast
					ws.NotifyNewEvents()
				}
			}
		}
	}

	return nil
}

// appserviceJoinedAtEvent returns a boolean depending on whether a given
// appservice has membership at the time a given event was created.
func (s *OutputRoomEventConsumer) appserviceJoinedAtEvent(ctx context.Context, event *gomatrixserverlib.HeaderedEvent, appservice config.ApplicationService) bool {
	// TODO: This is only checking the current room state, not the state at
	// the event in question. Pretty sure this is what Synapse does too, but
	// until we have a lighter way of checking the state before the event that
	// doesn't involve state res, then this is probably OK.
	membershipReq := &api.QueryMembershipsForRoomRequest{
		RoomID:     event.RoomID(),
		JoinedOnly: true,
	}
	membershipRes := &api.QueryMembershipsForRoomResponse{}

	// XXX: This could potentially race if the state for the event is not known yet
	// e.g. the event came over federation but we do not have the full state persisted.
	if err := s.rsAPI.QueryMembershipsForRoom(ctx, membershipReq, membershipRes); err == nil {
		for _, ev := range membershipRes.JoinEvents {
			var membership gomatrixserverlib.MemberContent
			if err = json.Unmarshal(ev.Content, &membership); err != nil || ev.StateKey == nil {
				continue
			}
			if appservice.IsInterestedInUserID(*ev.StateKey) {
				return true
			}
		}
	} else {
		log.WithFields(log.Fields{
			"room_id": event.RoomID(),
		}).WithError(err).Errorf("Unable to get membership for room")
	}
	return false
}

// appserviceIsInterestedInEvent returns a boolean depending on whether a given
// event falls within one of a given application service's namespaces.
//
// TODO: This should be cached, see https://github.com/matrix-org/dendrite/issues/1682
func (s *OutputRoomEventConsumer) appserviceIsInterestedInEvent(ctx context.Context, event *gomatrixserverlib.HeaderedEvent, appservice config.ApplicationService) bool {
	// No reason to queue events if they'll never be sent to the application
	// service
	if appservice.URL == "" {
		return false
	}

	// Check Room ID and Sender of the event
	if appservice.IsInterestedInUserID(event.Sender()) ||
		appservice.IsInterestedInRoomID(event.RoomID()) {
		return true
	}

	if event.Type() == gomatrixserverlib.MRoomMember && event.StateKey() != nil {
		if appservice.IsInterestedInUserID(*event.StateKey()) {
			return true
		}
	}

	// Check all known room aliases of the room the event came from
	queryReq := api.GetAliasesForRoomIDRequest{RoomID: event.RoomID()}
	var queryRes api.GetAliasesForRoomIDResponse
	if err := s.rsAPI.GetAliasesForRoomID(ctx, &queryReq, &queryRes); err == nil {
		for _, alias := range queryRes.Aliases {
			if appservice.IsInterestedInRoomAlias(alias) {
				return true
			}
		}
	} else {
		log.WithFields(log.Fields{
			"room_id": event.RoomID(),
		}).WithError(err).Errorf("Unable to get aliases for room")
	}

	// Check if any of the members in the room match the appservice
	return s.appserviceJoinedAtEvent(ctx, event, appservice)
}
<commit_msg>Handle invite in appservice roomserver consumer (#2476)<commit_after>// Copyright 2018 Vector Creations Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package consumers

import (
	"context"
	"encoding/json"

	"github.com/matrix-org/dendrite/appservice/storage"
	"github.com/matrix-org/dendrite/appservice/types"
	"github.com/matrix-org/dendrite/roomserver/api"
	"github.com/matrix-org/dendrite/setup/config"
	"github.com/matrix-org/dendrite/setup/jetstream"
	"github.com/matrix-org/dendrite/setup/process"
	"github.com/matrix-org/gomatrixserverlib"
	"github.com/nats-io/nats.go"

	log "github.com/sirupsen/logrus"
)

// OutputRoomEventConsumer consumes events that originated in the room server.
type OutputRoomEventConsumer struct {
	ctx          context.Context
	jetstream    nats.JetStreamContext
	durable      string
	topic        string
	asDB         storage.Database
	rsAPI        api.AppserviceRoomserverAPI
	serverName   string
	workerStates []types.ApplicationServiceWorkerState
}

// NewOutputRoomEventConsumer creates a new OutputRoomEventConsumer. Call
// Start() to begin consuming from room servers.
func NewOutputRoomEventConsumer(
	process *process.ProcessContext,
	cfg *config.Dendrite,
	js nats.JetStreamContext,
	appserviceDB storage.Database,
	rsAPI api.AppserviceRoomserverAPI,
	workerStates []types.ApplicationServiceWorkerState,
) *OutputRoomEventConsumer {
	return &OutputRoomEventConsumer{
		ctx:          process.Context(),
		jetstream:    js,
		durable:      cfg.Global.JetStream.Durable("AppserviceRoomserverConsumer"),
		topic:        cfg.Global.JetStream.Prefixed(jetstream.OutputRoomEvent),
		asDB:         appserviceDB,
		rsAPI:        rsAPI,
		serverName:   string(cfg.Global.ServerName),
		workerStates: workerStates,
	}
}

// Start consuming from room servers
func (s *OutputRoomEventConsumer) Start() error {
	return jetstream.JetStreamConsumer(
		s.ctx, s.jetstream, s.topic, s.durable, s.onMessage,
		nats.DeliverAll(), nats.ManualAck(),
	)
}

// onMessage is called when the appservice component receives a new event from
// the room server output log.
func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool {
	// Parse out the event JSON
	var output api.OutputEvent
	if err := json.Unmarshal(msg.Data, &output); err != nil {
		// If the message was invalid, log it and move on to the next message in the stream
		log.WithError(err).Errorf("roomserver output log: message parse failure")
		return true
	}

	log.WithFields(log.Fields{
		"type": output.Type,
	}).Debug("Got a message in OutputRoomEventConsumer")

	events := []*gomatrixserverlib.HeaderedEvent{}
	if output.Type == api.OutputTypeNewRoomEvent && output.NewRoomEvent != nil {
		newEventID := output.NewRoomEvent.Event.EventID()
		events = append(events, output.NewRoomEvent.Event)
		if len(output.NewRoomEvent.AddsStateEventIDs) > 0 {
			eventsReq := &api.QueryEventsByIDRequest{
				EventIDs: make([]string, 0, len(output.NewRoomEvent.AddsStateEventIDs)),
			}
			eventsRes := &api.QueryEventsByIDResponse{}
			for _, eventID := range output.NewRoomEvent.AddsStateEventIDs {
				if eventID != newEventID {
					eventsReq.EventIDs = append(eventsReq.EventIDs, eventID)
				}
			}
			if len(eventsReq.EventIDs) > 0 {
				if err := s.rsAPI.QueryEventsByID(s.ctx, eventsReq, eventsRes); err != nil {
					return false
				}
				events = append(events, eventsRes.Events...)
			}
		}
	} else if output.Type == api.OutputTypeNewInviteEvent && output.NewInviteEvent != nil {
		events = append(events, output.NewInviteEvent.Event)
	} else {
		log.WithFields(log.Fields{
			"type": output.Type,
		}).Debug("appservice OutputRoomEventConsumer ignoring event", string(msg.Data))
		return true
	}

	// Send event to any relevant application services
	if err := s.filterRoomserverEvents(context.TODO(), events); err != nil {
		log.WithError(err).Errorf("roomserver output log: filter error")
		return true
	}

	return true
}

// filterRoomserverEvents takes in events and decides whether any of them need
// to be passed on to an external application service. It does this by checking
// each namespace of each registered application service, and if there is a
// match, adds the event to the queue for events to be sent to a particular
// application service.
func (s *OutputRoomEventConsumer) filterRoomserverEvents(
	ctx context.Context,
	events []*gomatrixserverlib.HeaderedEvent,
) error {
	for _, ws := range s.workerStates {
		for _, event := range events {
			// Check if this event is interesting to this application service
			if s.appserviceIsInterestedInEvent(ctx, event, ws.AppService) {
				// Queue this event to be sent off to the application service
				if err := s.asDB.StoreEvent(ctx, ws.AppService.ID, event); err != nil {
					log.WithError(err).Warn("failed to insert incoming event into appservices database")
					return err
				} else {
					// Tell our worker to send out new messages by updating remaining message
					// count and waking them up with a broadcast
					ws.NotifyNewEvents()
				}
			}
		}
	}

	return nil
}

// appserviceJoinedAtEvent returns a boolean depending on whether a given
// appservice has membership at the time a given event was created.
func (s *OutputRoomEventConsumer) appserviceJoinedAtEvent(ctx context.Context, event *gomatrixserverlib.HeaderedEvent, appservice config.ApplicationService) bool {
	// TODO: This is only checking the current room state, not the state at
	// the event in question. Pretty sure this is what Synapse does too, but
	// until we have a lighter way of checking the state before the event that
	// doesn't involve state res, then this is probably OK.
	membershipReq := &api.QueryMembershipsForRoomRequest{
		RoomID:     event.RoomID(),
		JoinedOnly: true,
	}
	membershipRes := &api.QueryMembershipsForRoomResponse{}

	// XXX: This could potentially race if the state for the event is not known yet
	// e.g. the event came over federation but we do not have the full state persisted.
	if err := s.rsAPI.QueryMembershipsForRoom(ctx, membershipReq, membershipRes); err == nil {
		for _, ev := range membershipRes.JoinEvents {
			var membership gomatrixserverlib.MemberContent
			if err = json.Unmarshal(ev.Content, &membership); err != nil || ev.StateKey == nil {
				continue
			}
			if appservice.IsInterestedInUserID(*ev.StateKey) {
				return true
			}
		}
	} else {
		log.WithFields(log.Fields{
			"room_id": event.RoomID(),
		}).WithError(err).Errorf("Unable to get membership for room")
	}
	return false
}

// appserviceIsInterestedInEvent returns a boolean depending on whether a given
// event falls within one of a given application service's namespaces.
//
// TODO: This should be cached, see https://github.com/matrix-org/dendrite/issues/1682
func (s *OutputRoomEventConsumer) appserviceIsInterestedInEvent(ctx context.Context, event *gomatrixserverlib.HeaderedEvent, appservice config.ApplicationService) bool {
	// No reason to queue events if they'll never be sent to the application
	// service
	if appservice.URL == "" {
		return false
	}

	// Check Room ID and Sender of the event
	if appservice.IsInterestedInUserID(event.Sender()) ||
		appservice.IsInterestedInRoomID(event.RoomID()) {
		return true
	}

	if event.Type() == gomatrixserverlib.MRoomMember && event.StateKey() != nil {
		if appservice.IsInterestedInUserID(*event.StateKey()) {
			return true
		}
	}

	// Check all known room aliases of the room the event came from
	queryReq := api.GetAliasesForRoomIDRequest{RoomID: event.RoomID()}
	var queryRes api.GetAliasesForRoomIDResponse
	if err := s.rsAPI.GetAliasesForRoomID(ctx, &queryReq, &queryRes); err == nil {
		for _, alias := range queryRes.Aliases {
			if appservice.IsInterestedInRoomAlias(alias) {
				return true
			}
		}
	} else {
		log.WithFields(log.Fields{
			"room_id": event.RoomID(),
		}).WithError(err).Errorf("Unable to get aliases for room")
	}

	// Check if any of the members in the room match the appservice
	return s.appserviceJoinedAtEvent(ctx, event, appservice)
}
<|endoftext|>


<commit_before>package web

import (
    "io"
	"net/http"
	"draringi/codejam2013/src/forecasting"
	"draringi/codejam2013/src/data"
    "encoding/json"
    "time"
    "sync"
)

type dataError struct {
    What string
    When time.Time
}

func (self *dataError) Error() string {
    return "["+self.When.Format(data.ISO)+"] " + self.What
}

type future struct {
    Records []record
}

type record struct {
    Date string
    Power float64
}

type dashboardHelper struct {
}

type Dashboard struct {
	channel chan ([]data.Record)
	JSONAid dashboardHelper
    Forcast *future
    Data []data.Record
    Lock sync.Mutex
}

func (self *Dashboard) Init () {
	self.Lock.Lock()
	self.channel = make(chan (*data.CSVData), 1)
	self.Data = nil
	self.Forcast = nil
	self.Lock.Unlock()
	go forecasting.PredictPulse(self.channel)
	for {
		tmp := <-self.channel
		if tmp != nil {
			self.Data = tmp
			self.Build()
		}
	}
}

type Static struct{
}

func (self *Static) ServeHTTP (w http.ResponseWriter, request *http.Request) {
	http.ServeFile(w, request, "dashboard.html")
}

func (self *Dashboard) Build () {
    Data := self.Data
    self.Forcast = new(future)
    self.Forcast.Records = make([]record,len(Data))
    for i :=0; i<len(Data.Data); i++ {
        self.Forcast.Records[i].Date = Data[i].Time.Format(data.ISO)
        self.Forcast.Records[i].Power = Data[i].Power
    }
}

func (self *Dashboard) jsonify (w io.Writer) error {
    encoder := json.NewEncoder(w)
    if self.Data != nil {
        encoder.Encode(self.Forcast)
        return nil
    } else {
        return &dataError{"Error: Could not load data", time.Now()}
    }
}

func (self *Dashboard) ServeHTTP (w http.ResponseWriter, request *http.Request) {
    err := self.jsonify(w)
    if err != nil {
        http.Error(w,err.Error(), 404)
    }
}
<commit_msg>Fixed variable types and so<commit_after>package web

import (
    "io"
	"net/http"
	"draringi/codejam2013/src/forecasting"
	"draringi/codejam2013/src/data"
    "encoding/json"
    "time"
    "sync"
)

type dataError struct {
    What string
    When time.Time
}

func (self *dataError) Error() string {
    return "["+self.When.Format(data.ISO)+"] " + self.What
}

type future struct {
    Records []record
}

type record struct {
    Date string
    Power float64
}

type dashboardHelper struct {
}

type Dashboard struct {
	channel chan ([]data.Record)
	JSONAid dashboardHelper
    Forcast *future
    Data []data.Record
    Lock sync.Mutex
}

func (self *Dashboard) Init () {
	self.Lock.Lock()
	self.channel = make(chan ([]data.Record), 1)
	self.Data = nil
	self.Forcast = nil
	self.Lock.Unlock()
	go forecasting.PredictPulse(self.channel)
	for {
		tmp := <-self.channel
		if tmp != nil {
			self.Data = tmp
			self.Build()
		}
	}
}

type Static struct{
}

func (self *Static) ServeHTTP (w http.ResponseWriter, request *http.Request) {
	http.ServeFile(w, request, "dashboard.html")
}

func (self *Dashboard) Build () {
    Data := self.Data
    self.Forcast = new(future)
    self.Forcast.Records = make([]record,len(Data))
    for i :=0; i<len(Data); i++ {
        self.Forcast.Records[i].Date = Data[i].Time.Format(data.ISO)
        self.Forcast.Records[i].Power = Data[i].Power
    }
}

func (self *Dashboard) jsonify (w io.Writer) error {
    encoder := json.NewEncoder(w)
    if self.Data != nil {
        encoder.Encode(self.Forcast)
        return nil
    } else {
        return &dataError{"Error: Could not load data", time.Now()}
    }
}

func (self *Dashboard) ServeHTTP (w http.ResponseWriter, request *http.Request) {
    err := self.jsonify(w)
    if err != nil {
        http.Error(w,err.Error(), 404)
    }
}
<|endoftext|>


<commit_before>package synapse

import (
	"net"
	"net/rpc"
	"testing"
)

func TestCodecs(t *testing.T) {
	// create fake network
	cl, srv := net.Pipe()

	ccd := &clientCodec{conn: cl}
	scd := &serverCodec{conn: srv}

	server := rpc.NewServer()
	server.Register(Echo{})
	go server.ServeCodec(scd)

	client := rpc.NewClientWithCodec(ccd)

	out := &Arg2{}
	in := &Arg1{Value: "Hello, world!"}
	err := client.Call("Echo.Echo", in, out)
	if err != nil {
		t.Fatal(err)
	}

	if out.Value != in.Value {
		t.Fatalf("%q in; %q out", in.Value, out.Value)
	}

	cl.Close()
}
<commit_msg>bench<commit_after>package synapse

import (
	"net"
	"net/rpc"
	"testing"
)

func TestCodecs(t *testing.T) {
	// create fake network
	cl, srv := net.Pipe()

	ccd := &clientCodec{conn: cl}
	scd := &serverCodec{conn: srv}

	server := rpc.NewServer()
	server.Register(Echo{})
	go server.ServeCodec(scd)

	client := rpc.NewClientWithCodec(ccd)

	out := &Arg2{}
	in := &Arg1{Value: "Hello, world!"}
	err := client.Call("Echo.Echo", in, out)
	if err != nil {
		t.Fatal(err)
	}

	if out.Value != in.Value {
		t.Fatalf("%q in; %q out", in.Value, out.Value)
	}

	cl.Close()
}

func BenchmarkBasicCodec(b *testing.B) {
	cl, srv := net.Pipe()
	ccd := &clientCodec{conn: cl}
	scd := &serverCodec{conn: srv}
	server := rpc.NewServer()
	server.Register(Echo{})
	go server.ServeCodec(scd)
	client := rpc.NewClientWithCodec(ccd)
	out := &Arg2{}
	in := &Arg1{Value: "Hello, world!"}

	b.ResetTimer()
	b.ReportAllocs()
	for i := 0; i < b.N; i++ {
		err := client.Call("Echo.Echo", in, out)
		if err != nil {
			b.Fatal(err)
		}
	}
	cl.Close()
}
<|endoftext|>


<commit_before>// Copyright 2014 Hajime Hoshi
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ebiten

import (
	"fmt"
	"math"

	"github.com/hajimehoshi/ebiten/v2/internal/graphicsdriver"
)

type gameForUI struct {
	game      Game
	offscreen *Image
	screen    *Image
}

func newGameForUI(game Game) *gameForUI {
	return &gameForUI{
		game: game,
	}
}

func (c *gameForUI) Layout(outsideWidth, outsideHeight float64, deviceScaleFactor float64) (int, int) {
	ow, oh := c.game.Layout(int(outsideWidth), int(outsideHeight))
	if ow <= 0 || oh <= 0 {
		panic("ebiten: Layout must return positive numbers")
	}

	sw, sh := int(outsideWidth*deviceScaleFactor), int(outsideHeight*deviceScaleFactor)
	if c.screen != nil {
		if w, h := c.screen.Size(); w != sw || h != sh {
			c.screen.Dispose()
			c.screen = nil
		}
	}
	if c.screen == nil {
		c.screen = newScreenFramebufferImage(sw, sh)
	}

	if c.offscreen != nil {
		if w, h := c.offscreen.Size(); w != ow || h != oh {
			c.offscreen.Dispose()
			c.offscreen = nil
		}
	}
	if c.offscreen == nil {
		c.offscreen = NewImage(ow, oh)

		// Keep the offscreen an independent image from an atlas (#1938).
		// The shader program for the screen is special and doesn't work well with an image on an atlas.
		// An image on an atlas is surrounded by a transparent edge,
		// and the shader program unexpectedly picks the pixel on the edges.
		c.offscreen.mipmap.SetIndependent(true)
	}

	return ow, oh
}

func (c *gameForUI) Update() error {
	return c.game.Update()
}

func (c *gameForUI) Draw(screenScale float64, offsetX, offsetY float64, needsClearingScreen bool, framebufferYDirection graphicsdriver.YDirection, clearScreenEveryFrame bool) error {
	c.offscreen.mipmap.SetVolatile(clearScreenEveryFrame)

	// Even though updateCount == 0, the offscreen is cleared and Draw is called.
	// Draw should not update the game state and then the screen should not be updated without Update, but
	// users might want to process something at Draw with the time intervals of FPS.
	if clearScreenEveryFrame {
		c.offscreen.Clear()
	}
	c.game.Draw(c.offscreen)

	if needsClearingScreen {
		// This clear is needed for fullscreen mode or some mobile platforms (#622).
		c.screen.Clear()
	}

	op := &DrawImageOptions{}

	s := screenScale
	switch framebufferYDirection {
	case graphicsdriver.Upward:
		op.GeoM.Scale(s, -s)
		_, h := c.offscreen.Size()
		op.GeoM.Translate(0, float64(h)*s)
	case graphicsdriver.Downward:
		op.GeoM.Scale(s, s)
	default:
		panic(fmt.Sprintf("ebiten: invalid v-direction: %d", framebufferYDirection))
	}

	op.GeoM.Translate(offsetX, offsetY)
	op.CompositeMode = CompositeModeCopy

	// filterScreen works with >=1 scale, but does not well with <1 scale.
	// Use regular FilterLinear instead so far (#669).
	switch {
	case math.Floor(s) == s:
		op.Filter = FilterNearest
	case s > 1:
		op.Filter = filterScreen
	default:
		op.Filter = FilterLinear
	}
	c.screen.DrawImage(c.offscreen, op)
	return nil
}
<commit_msg>ebiten: move comments<commit_after>// Copyright 2014 Hajime Hoshi
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ebiten

import (
	"fmt"
	"math"

	"github.com/hajimehoshi/ebiten/v2/internal/graphicsdriver"
)

type gameForUI struct {
	game      Game
	offscreen *Image
	screen    *Image
}

func newGameForUI(game Game) *gameForUI {
	return &gameForUI{
		game: game,
	}
}

func (c *gameForUI) Layout(outsideWidth, outsideHeight float64, deviceScaleFactor float64) (int, int) {
	ow, oh := c.game.Layout(int(outsideWidth), int(outsideHeight))
	if ow <= 0 || oh <= 0 {
		panic("ebiten: Layout must return positive numbers")
	}

	sw, sh := int(outsideWidth*deviceScaleFactor), int(outsideHeight*deviceScaleFactor)
	if c.screen != nil {
		if w, h := c.screen.Size(); w != sw || h != sh {
			c.screen.Dispose()
			c.screen = nil
		}
	}
	if c.screen == nil {
		c.screen = newScreenFramebufferImage(sw, sh)
	}

	if c.offscreen != nil {
		if w, h := c.offscreen.Size(); w != ow || h != oh {
			c.offscreen.Dispose()
			c.offscreen = nil
		}
	}
	if c.offscreen == nil {
		c.offscreen = NewImage(ow, oh)

		// Keep the offscreen an independent image from an atlas (#1938).
		// The shader program for the screen is special and doesn't work well with an image on an atlas.
		// An image on an atlas is surrounded by a transparent edge,
		// and the shader program unexpectedly picks the pixel on the edges.
		c.offscreen.mipmap.SetIndependent(true)
	}

	return ow, oh
}

func (c *gameForUI) Update() error {
	return c.game.Update()
}

func (c *gameForUI) Draw(screenScale float64, offsetX, offsetY float64, needsClearingScreen bool, framebufferYDirection graphicsdriver.YDirection, clearScreenEveryFrame bool) error {
	c.offscreen.mipmap.SetVolatile(clearScreenEveryFrame)

	// Even though updateCount == 0, the offscreen is cleared and Draw is called.
	// Draw should not update the game state and then the screen should not be updated without Update, but
	// users might want to process something at Draw with the time intervals of FPS.
	if clearScreenEveryFrame {
		c.offscreen.Clear()
	}
	c.game.Draw(c.offscreen)

	if needsClearingScreen {
		// This clear is needed for fullscreen mode or some mobile platforms (#622).
		c.screen.Clear()
	}

	op := &DrawImageOptions{}

	s := screenScale
	switch framebufferYDirection {
	case graphicsdriver.Upward:
		op.GeoM.Scale(s, -s)
		_, h := c.offscreen.Size()
		op.GeoM.Translate(0, float64(h)*s)
	case graphicsdriver.Downward:
		op.GeoM.Scale(s, s)
	default:
		panic(fmt.Sprintf("ebiten: invalid v-direction: %d", framebufferYDirection))
	}

	op.GeoM.Translate(offsetX, offsetY)
	op.CompositeMode = CompositeModeCopy

	switch {
	case math.Floor(s) == s:
		op.Filter = FilterNearest
	case s > 1:
		op.Filter = filterScreen
	default:
		// filterScreen works with >=1 scale, but does not well with <1 scale.
		// Use regular FilterLinear instead so far (#669).
		op.Filter = FilterLinear
	}
	c.screen.DrawImage(c.offscreen, op)
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/xml"
	"fmt"
	id3 "github.com/mikkyang/id3-go"
	kingpin "gopkg.in/alecthomas/kingpin.v1"
	"net/http"
	"net/url"
	"os"
	"os/signal"
	"path/filepath"
	"strings"
	"syscall"
	"time"
)

const (
	Header   = `<?xml version="1.0" encoding="UTF-8"?>` + "\n"
	iTunesNs = "http://www.itunes.com/dtds/podcast-1.0.dtd"
)

type Rss struct {
	XMLName xml.Name `xml:"rss"`
	Channel Channel  `xml:"channel"`
	Version string   `xml:"version,attr"`
	NS      string   `xml:"xmlns:itunes,attr"`
}

type Channel struct {
	PubDate     string  `xml:"pubDate,omitempty"`
	Title       string  `xml:"title,omitempty"`
	Link        string  `xml:"link,omitempty"`
	Description string  `xml:"description"`
	Language    string  `xml:"language,omitempty"`
	Images      []Image `xml:"image,omitempty"`
	Items       []Item  `xml:"item"`
}

type Item struct {
	Title       string    `xml:"title"`
	Description string    `xml:"description"`
	Enclosure   Enclosure `xml:"enclosure"`
	Guid        string    `xml:"guid"`
	Subtitle    string    `xml:"itunes:subtitle,omitempty"`
	Categories  []Text    `xml:"itunes:category,omitempty"`
	PubDate     string    `xml:"pubDate,omitempty"`
}

type Text struct {
	Value string `xml:"text,attr"`
}

type Image struct {
	Link  string `xml:"link"`
	Title string `xml:"title"`
	Url   string `xml:"url"`
}

type Enclosure struct {
	Url    string `xml:"url,attr"`
	Length int64  `xml:"length,attr"`
	Type   string `xml:"type,attr"`
}

func fileUrl(relativePath string, baseUrl string) string {
	Url, _ := url.Parse(baseUrl)
	Url.Path += relativePath
	return Url.String()
}

func formatYear(year string) string {
	if len(year) > 0 {
		t, err := time.Parse("20060102", year)
		if err != nil {
			t, err = time.Parse("20060102", year[0:len(year)-1])
			if err != nil {
				t, err = time.Parse("2006", year)
				if err != nil {
					t, err = time.Parse("20060201", year)
					if err != nil {
						return ""
					}
				}
			}
		}
		return t.Format(time.RFC1123Z)
	}
	return year
}

func addMeta(path string, f os.FileInfo, item *Item) {
	fd, err := id3.Open(path)
	if err != nil {
		item.Title = f.Name()
	} else {
		title := fd.Title()
		author := fd.Artist()
		if len(title) > 0 {
			item.Title = title
		} else {
			item.Title = author
			if len(author) > 0 {
				item.Title += " - "
			}
			item.Title += f.Name()
		}
		item.Subtitle = author
		tcon := fd.Frame("TCON")
		if tcon != nil {
			item.Categories = append(item.Categories, Text{Value: tcon.String()})
		}
		item.PubDate = formatYear(fd.Year())
	}
}

func visitFiles(workDir string, channel *Channel, publicUrl string, recursive bool, fileType string) filepath.WalkFunc {
	return func(path string, f os.FileInfo, err error) error {

		if err != nil {
			return err
		}

		if f.IsDir() && path != workDir && !recursive {
			return filepath.SkipDir
		}

		if !!f.IsDir() {
			return nil
		}

		matched, _ := filepath.Match("*."+fileType, f.Name())
		if matched {
			url := fileUrl(path[len(workDir)-1:], publicUrl)
			item := Item{Enclosure: Enclosure{Length: f.Size(), Type: "audio/mpeg",
				Url: url}, Guid: url}
			addMeta(path, f, &item)
			channel.Items = append(channel.Items, item)
		}

		return nil

	}
}

type rssHandler struct {
	header string
	body   []byte
	fs     http.Handler
	path   string
}

func (rss *rssHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	path := r.URL.Path
	if path == "" || path == "/" {
		w.Write([]byte(rss.header))
		w.Write(rss.body)
	} else {
		http.StripPrefix(rss.path, rss.fs).ServeHTTP(w, r)
	}
}

func writeStartupMsg(workdir string, url string) {
	fmt.Printf(
		"\x1b[33;1m%v\x1b[0m \x1b[36;1m%v\x1b[0m \x1b[33;1mon:\x1b[0m \x1b[36;1m%v\x1b[0m\n",
		"Starting up dircast, serving", workdir, url)
	fmt.Println("Hit CTRL-C to stop the server")
}

func onShutdown(message string) {
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	signal.Notify(c, syscall.SIGTERM)
	go func() {
		<-c
		fmt.Printf("\x1b[31;1m%v\x1b[0m\n", message)
		os.Exit(1)
	}()
}

func server(output []byte, workdir string, baseUrl *url.URL) error {

	path := baseUrl.Path
	if !strings.HasSuffix(path, "/") {
		path += "/"
	}

	rss := &rssHandler{header: Header, body: output,
		fs: http.FileServer(http.Dir(workdir))}

	http.Handle(path, rss)

	writeStartupMsg(workdir, baseUrl.String())
	onShutdown("dircast stopped.")

	return http.ListenAndServe(baseUrl.Host, nil)

}

var (
	baseUrl     = kingpin.Flag("server", "hostname (and path) to the root e.g. http://myserver.com/rss").Short('s').Default("http://localhost:8000/").URL()
	bind        = kingpin.Flag("bind", "Start HTTP server, bind to the server").Short('b').Bool()
	recursive   = kingpin.Flag("recursive", "how to handle the directory scan").Short('r').Bool()
	autoImage   = kingpin.Flag("auto-image", "Resolve RSS image automatically, will use cover art if available, image overrides this option").Short('a').Bool()
	language    = kingpin.Flag("language", "the language of the RSS document, a ISO 639 value").Short('l').String()
	title       = kingpin.Flag("title", "RSS channel title").Short('t').Default("RSS FEED").String()
	description = kingpin.Flag("description", "RSS channel description").Short('d').String()
	imageUrl    = kingpin.Flag("image", "Image URL for the RSS channel image").Short('i').URL()
	fileType    = kingpin.Flag("file", "File type to include in the RSS document").Short('f').Default("mp3").String()
	path        = kingpin.Arg("directory", "directory to read files relative from").Required().ExistingDir()
)

func main() {

	kingpin.Version("0.2.0")
	kingpin.Parse()

	channel := &Channel{
		PubDate:     time.Now().Format(time.RFC1123Z),
		Title:       *title,
		Link:        (*baseUrl).String(),
		Description: *description,
		Language:    *language}

	if !strings.HasSuffix((*baseUrl).Path, "/") {
		(*baseUrl).Path = (*baseUrl).Path + "/"
	}

	err := filepath.Walk(*path, visitFiles(*path, channel, (*baseUrl).String(), *recursive, *fileType))
	if *imageUrl != nil {
		channel.Images = append(channel.Images, Image{Title: channel.Title, Link: channel.Link, Url: (*imageUrl).String()})
	}

	if err != nil {
		fmt.Printf("%s: %v\n", os.Args[0], err)
	} else {
		output, err := xml.MarshalIndent(
			&Rss{Channel: *channel, Version: "2.0", NS: iTunesNs}, "", "  ")
		if err != nil {
			fmt.Printf("error: %v\n", err)
		} else {
			if *bind {
				err = server(output, *path, *baseUrl)
				if err != nil {
					fmt.Printf("error: %v\n", err)
				}
			} else {
				os.Stdout.WriteString(Header)
				os.Stdout.Write(output)
			}
		}
	}

}
<commit_msg>ignore auto-image parameter if image url parameter is used<commit_after>package main

import (
	"encoding/xml"
	"fmt"
	id3 "github.com/mikkyang/id3-go"
	kingpin "gopkg.in/alecthomas/kingpin.v1"
	"net/http"
	"net/url"
	"os"
	"os/signal"
	"path/filepath"
	"strings"
	"syscall"
	"time"
)

const (
	Header   = `<?xml version="1.0" encoding="UTF-8"?>` + "\n"
	iTunesNs = "http://www.itunes.com/dtds/podcast-1.0.dtd"
)

type Rss struct {
	XMLName xml.Name `xml:"rss"`
	Channel Channel  `xml:"channel"`
	Version string   `xml:"version,attr"`
	NS      string   `xml:"xmlns:itunes,attr"`
}

type Channel struct {
	PubDate     string  `xml:"pubDate,omitempty"`
	Title       string  `xml:"title,omitempty"`
	Link        string  `xml:"link,omitempty"`
	Description string  `xml:"description"`
	Language    string  `xml:"language,omitempty"`
	Images      []Image `xml:"image,omitempty"`
	Items       []Item  `xml:"item"`
}

type Item struct {
	Title       string    `xml:"title"`
	Description string    `xml:"description"`
	Enclosure   Enclosure `xml:"enclosure"`
	Guid        string    `xml:"guid"`
	Subtitle    string    `xml:"itunes:subtitle,omitempty"`
	Categories  []Text    `xml:"itunes:category,omitempty"`
	PubDate     string    `xml:"pubDate,omitempty"`
}

type Text struct {
	Value string `xml:"text,attr"`
}

type Image struct {
	Link  string `xml:"link"`
	Title string `xml:"title"`
	Url   string `xml:"url"`
}

type Enclosure struct {
	Url    string `xml:"url,attr"`
	Length int64  `xml:"length,attr"`
	Type   string `xml:"type,attr"`
}

func fileUrl(relativePath string, baseUrl string) string {
	Url, _ := url.Parse(baseUrl)
	Url.Path += relativePath
	return Url.String()
}

func formatYear(year string) string {
	if len(year) > 0 {
		t, err := time.Parse("20060102", year)
		if err != nil {
			t, err = time.Parse("20060102", year[0:len(year)-1])
			if err != nil {
				t, err = time.Parse("2006", year)
				if err != nil {
					t, err = time.Parse("20060201", year)
					if err != nil {
						return ""
					}
				}
			}
		}
		return t.Format(time.RFC1123Z)
	}
	return year
}

func addMeta(path string, f os.FileInfo, item *Item) {
	fd, err := id3.Open(path)
	if err != nil {
		item.Title = f.Name()
	} else {
		title := fd.Title()
		author := fd.Artist()
		if len(title) > 0 {
			item.Title = title
		} else {
			item.Title = author
			if len(author) > 0 {
				item.Title += " - "
			}
			item.Title += f.Name()
		}
		item.Subtitle = author
		tcon := fd.Frame("TCON")
		if tcon != nil {
			item.Categories = append(item.Categories, Text{Value: tcon.String()})
		}
		item.PubDate = formatYear(fd.Year())
	}
}

func visitFiles(workDir string, channel *Channel, publicUrl string, recursive bool, fileType string) filepath.WalkFunc {
	return func(path string, f os.FileInfo, err error) error {

		if err != nil {
			return err
		}

		if f.IsDir() && path != workDir && !recursive {
			return filepath.SkipDir
		}

		if !!f.IsDir() {
			return nil
		}

		matched, _ := filepath.Match("*."+fileType, f.Name())
		if matched {
			url := fileUrl(path[len(workDir)-1:], publicUrl)
			item := Item{Enclosure: Enclosure{Length: f.Size(), Type: "audio/mpeg",
				Url: url}, Guid: url}
			addMeta(path, f, &item)
			channel.Items = append(channel.Items, item)
		}

		return nil

	}
}

type rssHandler struct {
	header string
	body   []byte
	fs     http.Handler
	path   string
}

func (rss *rssHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	path := r.URL.Path
	if path == "" || path == "/" {
		w.Write([]byte(rss.header))
		w.Write(rss.body)
	} else {
		http.StripPrefix(rss.path, rss.fs).ServeHTTP(w, r)
	}
}

func writeStartupMsg(workdir string, url string) {
	fmt.Printf(
		"\x1b[33;1m%v\x1b[0m \x1b[36;1m%v\x1b[0m \x1b[33;1mon:\x1b[0m \x1b[36;1m%v\x1b[0m\n",
		"Starting up dircast, serving", workdir, url)
	fmt.Println("Hit CTRL-C to stop the server")
}

func onShutdown(message string) {
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	signal.Notify(c, syscall.SIGTERM)
	go func() {
		<-c
		fmt.Printf("\x1b[31;1m%v\x1b[0m\n", message)
		os.Exit(1)
	}()
}

func server(output []byte, workdir string, baseUrl *url.URL) error {

	path := baseUrl.Path
	if !strings.HasSuffix(path, "/") {
		path += "/"
	}

	rss := &rssHandler{header: Header, body: output,
		fs: http.FileServer(http.Dir(workdir))}

	http.Handle(path, rss)

	writeStartupMsg(workdir, baseUrl.String())
	onShutdown("dircast stopped.")

	return http.ListenAndServe(baseUrl.Host, nil)

}

var (
	baseUrl     = kingpin.Flag("server", "hostname (and path) to the root e.g. http://myserver.com/rss").Short('s').Default("http://localhost:8000/").URL()
	bind        = kingpin.Flag("bind", "Start HTTP server, bind to the server").Short('b').Bool()
	recursive   = kingpin.Flag("recursive", "how to handle the directory scan").Short('r').Bool()
	autoImage   = kingpin.Flag("auto-image", "Resolve RSS image automatically, will use cover art if available, image overrides this option").Short('a').Bool()
	language    = kingpin.Flag("language", "the language of the RSS document, a ISO 639 value").Short('l').String()
	title       = kingpin.Flag("title", "RSS channel title").Short('t').Default("RSS FEED").String()
	description = kingpin.Flag("description", "RSS channel description").Short('d').String()
	imageUrl    = kingpin.Flag("image", "Image URL for the RSS channel image").Short('i').URL()
	fileType    = kingpin.Flag("file", "File type to include in the RSS document").Short('f').Default("mp3").String()
	path        = kingpin.Arg("directory", "directory to read files relative from").Required().ExistingDir()
)

func main() {

	kingpin.Version("0.2.0")
	kingpin.Parse()

	channel := &Channel{
		PubDate:     time.Now().Format(time.RFC1123Z),
		Title:       *title,
		Link:        (*baseUrl).String(),
		Description: *description,
		Language:    *language}

	if !strings.HasSuffix((*baseUrl).Path, "/") {
		(*baseUrl).Path = (*baseUrl).Path + "/"
	}

	if *imageUrl != nil {
		channel.Images = append(channel.Images, Image{Title: channel.Title, Link: channel.Link, Url: (*imageUrl).String()})
		*autoImage = false
	}
	err := filepath.Walk(*path, visitFiles(*path, channel, (*baseUrl).String(), *recursive, *fileType))

	if err != nil {
		fmt.Printf("%s: %v\n", os.Args[0], err)
	} else {
		output, err := xml.MarshalIndent(
			&Rss{Channel: *channel, Version: "2.0", NS: iTunesNs}, "", "  ")
		if err != nil {
			fmt.Printf("error: %v\n", err)
		} else {
			if *bind {
				err = server(output, *path, *baseUrl)
				if err != nil {
					fmt.Printf("error: %v\n", err)
				}
			} else {
				os.Stdout.WriteString(Header)
				os.Stdout.Write(output)
			}
		}
	}

}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/cactus/go-statsd-client/statsd"
	"io/ioutil"
	"net"
	"os"
	"strings"
	"time"
)

func dieIfError(err error) {
	if err != nil {
		fmt.Fprintf(os.Stderr, "Fatal error: %s\n", err.Error())
		os.Exit(1)
	}
}

func doEvery(d time.Duration, f func(*statsd.Client, string, bool), s *statsd.Client, target_file string, debug bool) {
	f(s, target_file, debug)
	for _ = range time.Tick(d) {
		f(s, target_file, debug)
	}
}

func process_targets(s *statsd.Client, target_file string, debug bool) {
	content, err := ioutil.ReadFile(target_file)
	if err != nil {
    if debug {
		  fmt.Println("couldn't open targets file: %s", target_file)
    }
		return
	}
	targets := strings.Split(string(content), "\n")
	for _, target := range targets {
		if len(target) < 1 {
			continue
		}
		go test(target, s, debug)
	}
}

func test(target string, s *statsd.Client, debug bool) {
	tuple := strings.Split(target, ":")
	host := tuple[0]
	port := tuple[1]
	subhost := strings.Replace(host, ".", "_", -1)

	pre := time.Now()
	conn, err := net.Dial("tcp", target)
	if err != nil {
    if debug {
		  fmt.Println("connect error", target)
		}
    s.Inc(fmt.Sprintf("%s.%s.dial_failed", subhost, port), 1, 1)
		return
	}
	duration := time.Since(pre)
	ms := int64(duration / time.Millisecond)
  if debug {
	  fmt.Printf("%s.%s.duration %d\n", subhost, port, ms)
  }
	s.Timing(fmt.Sprintf("%s.%s", subhost, port), ms, 1)
	conn.Close()
}

func main() {
  var statsd_host = flag.String("statsd_host", "localhost", "Statsd Hostname")
  var statsd_port = flag.String("statsd_port", "8125", "Statsd port")
  var bucket = flag.String("bucket", "smoketcp", "Graphite bucket prefix")
  var target_file = flag.String("target_file", "targets", "File containing the list of targets, ex: server1:80")
  var debug = flag.Bool("debug", false, "if true, turn on debugging output")
  flag.Parse()

	s, err := statsd.Dial(fmt.Sprintf("%s:%s", *statsd_host, *statsd_port), fmt.Sprintf("%s", *bucket))
	dieIfError(err)
	defer s.Close()
	doEvery(time.Second, process_targets, s, *target_file, *debug)
}
<commit_msg>fix formatting again<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/cactus/go-statsd-client/statsd"
	"io/ioutil"
	"net"
	"os"
	"strings"
	"time"
)

func dieIfError(err error) {
	if err != nil {
		fmt.Fprintf(os.Stderr, "Fatal error: %s\n", err.Error())
		os.Exit(1)
	}
}

func doEvery(d time.Duration, f func(*statsd.Client, string, bool), s *statsd.Client, target_file string, debug bool) {
	f(s, target_file, debug)
	for _ = range time.Tick(d) {
		f(s, target_file, debug)
	}
}

func process_targets(s *statsd.Client, target_file string, debug bool) {
	content, err := ioutil.ReadFile(target_file)
	if err != nil {
		if debug {
			fmt.Println("couldn't open targets file: %s", target_file)
		}
		return
	}
	targets := strings.Split(string(content), "\n")
	for _, target := range targets {
		if len(target) < 1 {
			continue
		}
		go test(target, s, debug)
	}
}

func test(target string, s *statsd.Client, debug bool) {
	tuple := strings.Split(target, ":")
	host := tuple[0]
	port := tuple[1]
	subhost := strings.Replace(host, ".", "_", -1)

	pre := time.Now()
	conn, err := net.Dial("tcp", target)
	if err != nil {
		if debug {
			fmt.Println("connect error", target)
		}
		s.Inc(fmt.Sprintf("%s.%s.dial_failed", subhost, port), 1, 1)
		return
	}
	duration := time.Since(pre)
	ms := int64(duration / time.Millisecond)
	if debug {
		fmt.Printf("%s.%s.duration %d\n", subhost, port, ms)
	}
	s.Timing(fmt.Sprintf("%s.%s", subhost, port), ms, 1)
	conn.Close()
}

func main() {
	var statsd_host = flag.String("statsd_host", "localhost", "Statsd Hostname")
	var statsd_port = flag.String("statsd_port", "8125", "Statsd port")
	var bucket = flag.String("bucket", "smoketcp", "Graphite bucket prefix")
	var target_file = flag.String("target_file", "targets", "File containing the list of targets, ex: server1:80")
	var debug = flag.Bool("debug", false, "if true, turn on debugging output")
	flag.Parse()

	s, err := statsd.Dial(fmt.Sprintf("%s:%s", *statsd_host, *statsd_port), fmt.Sprintf("%s", *bucket))
	dieIfError(err)
	defer s.Close()
	doEvery(time.Second, process_targets, s, *target_file, *debug)
}
<|endoftext|>