<commit_before>package docker

import (
	"bufio"
	"context"
	"encoding/json"
	"github.com/ViBiOh/dashboard/auth"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/filters"
	"github.com/gorilla/websocket"
	"log"
	"net/http"
	"regexp"
)

const ignoredByteLogSize = 8
const tailSize = `100`
const start = `start`
const stop = `stop`

var eventsDemand = regexp.MustCompile(`^events (\S+)`)
var logsDemand = regexp.MustCompile(`^logs (\S+)(?: (.+))?`)
var statsDemand = regexp.MustCompile(`^stats (\S+)(?: (.+))?`)
var busWebsocketRequest = regexp.MustCompile(`bus`)
var eventsPrefix = []byte(`events `)
var logsPrefix = []byte(`logs `)
var statsPrefix = []byte(`stats `)

var upgrader = websocket.Upgrader{
	ReadBufferSize:  1024,
	WriteBufferSize: 1024,
	CheckOrigin: func(r *http.Request) bool {
		return hostCheck.MatchString(r.Host)
	},
}

func upgradeAndAuth(w http.ResponseWriter, r *http.Request) (*websocket.Conn, *auth.User, error) {
	ws, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		defer ws.Close()
		return nil, nil, err
	}

	_, basicAuth, err := ws.ReadMessage()
	if err != nil {
		defer ws.Close()
		return nil, nil, err
	}

	user, err := auth.IsAuthenticatedByAuth(string(basicAuth))
	if err != nil {
		ws.WriteMessage(websocket.TextMessage, []byte(err.Error()))

		defer ws.Close()
		return nil, nil, err
	}

	return ws, user, nil
}

func readContent(user *auth.User, ws *websocket.Conn, name string, done chan<- int, content chan<- []byte) {
	for {
		messageType, message, err := ws.ReadMessage()

		if messageType == websocket.CloseMessage {
			close(done)
			return
		}

		if err != nil {
			if websocket.IsUnexpectedCloseError(err, websocket.CloseNormalClosure, websocket.CloseGoingAway, websocket.CloseNoStatusReceived, websocket.CloseAbnormalClosure) {
				log.Printf(`[%s] Error while reading from %s socket: %v`, user.Username, name, err)
			}

			close(done)
			return
		}

		content <- message
	}
}

func streamEvents(ctx context.Context, cancel context.CancelFunc, user *auth.User, _ string, output chan<- []byte) {
	defer cancel()

	filtersArgs := filters.NewArgs()
	if err := labelFilters(&filtersArgs, user, nil); err != nil {
		log.Printf(`[%s] Events opening in error: %v`, user.Username, err)
		return
	}
	if err := eventFilters(&filtersArgs); err != nil {
		log.Printf(`[%s] Events opening in error: %v`, user.Username, err)
		return
	}

	messages, errors := docker.Events(ctx, types.EventsOptions{Filters: filtersArgs})

	log.Printf(`[%s] Events streaming started`, user.Username)
	for {
		select {
		case <-ctx.Done():
			log.Printf(`[%s] Events streaming ended`, user.Username)
			return

		case message := <-messages:
			messageJSON, err := json.Marshal(message)
			if err != nil {
				log.Printf(`[%s] Events marshalling in error: %v`, user.Username, err)
				cancel()
			} else {
				output <- append(eventsPrefix, messageJSON...)
			}

		case err := <-errors:
			log.Printf(`[%s] Events reading in error: %v`, user.Username, err)
			cancel()
		}
	}
}

func streamLogs(ctx context.Context, cancel context.CancelFunc, user *auth.User, containerID string, output chan<- []byte) {
	logs, err := docker.ContainerLogs(ctx, containerID, types.ContainerLogsOptions{ShowStdout: true, ShowStderr: true, Follow: true, Tail: tailSize})
	defer cancel()

	if err != nil {
		log.Printf(`[%s] Logs opening in error: %v`, user.Username, err)
		return
	}
	defer logs.Close()

	scanner := bufio.NewScanner(logs)
	log.Printf(`[%s] Logs streaming started for %s`, user.Username, containerID)

	for scanner.Scan() {
		logLine := scanner.Bytes()
		if len(logLine) > ignoredByteLogSize {
			output <- append(logsPrefix, logLine[ignoredByteLogSize:]...)
		}
	}

	log.Printf(`[%s] Logs streaming ended for %s`, user.Username, containerID)
}

func streamStats(ctx context.Context, cancel context.CancelFunc, user *auth.User, containerID string, output chan<- []byte) {
	stats, err := docker.ContainerStats(ctx, containerID, true)
	defer cancel()

	if err != nil {
		log.Printf(`[%s] Stats opening in error for %s: %v`, user.Username, containerID, err)
		return
	}
	defer stats.Body.Close()

	scanner := bufio.NewScanner(stats.Body)
	log.Printf(`[%s] Stats streaming started for %s`, user.Username, containerID)

	for scanner.Scan() {
		output <- append(statsPrefix, scanner.Bytes()...)
	}

	log.Printf(`[%s] Stats streaming ended for %s`, user.Username, containerID)
}

func handleBusDemand(user *auth.User, name string, input []byte, demand *regexp.Regexp, cancel context.CancelFunc, output chan<- []byte, streamFn func(context.Context, context.CancelFunc, *auth.User, string, chan<- []byte)) context.CancelFunc {
	demandGroups := demand.FindSubmatch(input)
	if len(demandGroups) < 2 {
		log.Printf(`[%s] Unable to parse bus demand %s for %s`, user.Username, input, name)
	}

	action := string(demandGroups[1])

	containerID := ``
	if len(demandGroups) > 2 {
		containerID = string(demandGroups[2])
	}

	if action == stop && cancel != nil {
		log.Printf(`[%s] Stopping %s stream`, user.Username, name)
		cancel()
	} else if action == start {
		log.Printf(`[%s] Starting %s stream`, user.Username, name)

		if cancel != nil {
			log.Printf(`[%s] Cancelling previous %s stream`, user.Username, name)
			cancel()
		}

		ctx, newCancel := context.WithCancel(context.Background())
		go streamFn(ctx, newCancel, user, string(containerID), output)

		return newCancel
	}

	return nil
}

func busWebsocketHandler(w http.ResponseWriter, r *http.Request) {
	ws, user, err := upgradeAndAuth(w, r)
	if err != nil {
		log.Print(err)
		return
	}
	defer ws.Close()

	done := make(chan int)

	output := make(chan []byte)
	defer close(output)

	input := make(chan []byte)
	defer close(input)

	go readContent(user, ws, `streaming`, done, input)
	log.Printf(`[%s] Streaming started`, user.Username)

	var eventsCancelFunc context.CancelFunc
	var logsCancelFunc context.CancelFunc
	var statsCancelFunc context.CancelFunc

	for {
		select {
		case <-done:
			log.Printf(`[%s] Streaming ended`, user.Username)
			return

		case inputBytes := <-input:
			if eventsDemand.Match(inputBytes) {
				eventsCancelFunc = handleBusDemand(user, `events`, inputBytes, eventsDemand, eventsCancelFunc, output, streamEvents)
				if eventsCancelFunc != nil {
					defer eventsCancelFunc()
				}
			} else if logsDemand.Match(inputBytes) {
				logsCancelFunc = handleBusDemand(user, `logs`, inputBytes, logsDemand, logsCancelFunc, output, streamLogs)
				if logsCancelFunc != nil {
					defer logsCancelFunc()
				}
			} else if statsDemand.Match(inputBytes) {
				statsCancelFunc = handleBusDemand(user, `stats`, inputBytes, statsDemand, statsCancelFunc, output, streamStats)
				if statsCancelFunc != nil {
					defer statsCancelFunc()
				}
			}

		case outputBytes := <-output:
			if err = ws.WriteMessage(websocket.TextMessage, outputBytes); err != nil {
				log.Printf(`[%s] Error while writing to streaming: %v`, user.Username, err)
				close(done)
			}
		}
	}
}

// WebsocketHandler for Docker Websocket request. Should be use with net/http
type WebsocketHandler struct {
}

func (handler WebsocketHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	urlPath := []byte(r.URL.Path)

	if busWebsocketRequest.Match(urlPath) {
		busWebsocketHandler(w, r)
	}
}
<commit_msg>Update websocket.go<commit_after>package docker

import (
	"bufio"
	"context"
	"encoding/json"
	"github.com/ViBiOh/dashboard/auth"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/filters"
	"github.com/gorilla/websocket"
	"log"
	"net/http"
	"regexp"
)

const ignoredByteLogSize = 8
const tailSize = `100`
const start = `start`
const stop = `stop`

var eventsDemand = regexp.MustCompile(`^events (\S+)`)
var logsDemand = regexp.MustCompile(`^logs (\S+)(?: (.+))?`)
var statsDemand = regexp.MustCompile(`^stats (\S+)(?: (.+))?`)
var busWebsocketRequest = regexp.MustCompile(`bus`)
var eventsPrefix = []byte(`events `)
var logsPrefix = []byte(`logs `)
var statsPrefix = []byte(`stats `)

var upgrader = websocket.Upgrader{
	ReadBufferSize:  1024,
	WriteBufferSize: 1024,
	CheckOrigin: func(r *http.Request) bool {
		return hostCheck.MatchString(r.Host)
	},
}

func upgradeAndAuth(w http.ResponseWriter, r *http.Request) (*websocket.Conn, *auth.User, error) {
	ws, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		defer ws.Close()
		return nil, nil, err
	}

	_, basicAuth, err := ws.ReadMessage()
	if err != nil {
		defer ws.Close()
		return nil, nil, err
	}

	user, err := auth.IsAuthenticatedByAuth(string(basicAuth))
	if err != nil {
		ws.WriteMessage(websocket.TextMessage, []byte(err.Error()))

		defer ws.Close()
		return nil, nil, err
	}

	return ws, user, nil
}

func readContent(user *auth.User, ws *websocket.Conn, name string, done chan<- int, content chan<- []byte) {
	for {
		messageType, message, err := ws.ReadMessage()

		if messageType == websocket.CloseMessage {
			close(done)
			return
		}

		if err != nil {
			if websocket.IsUnexpectedCloseError(err, websocket.CloseNormalClosure, websocket.CloseGoingAway, websocket.CloseNoStatusReceived, websocket.CloseAbnormalClosure) {
				log.Printf(`[%s] Error while reading from %s socket: %v`, user.Username, name, err)
			}

			close(done)
			return
		}

		content <- message
	}
}

func streamEvents(ctx context.Context, cancel context.CancelFunc, user *auth.User, _ string, output chan<- []byte) {
	defer cancel()

	filtersArgs := filters.NewArgs()
	if err := labelFilters(&filtersArgs, user, nil); err != nil {
		log.Printf(`[%s] Events opening in error: %v`, user.Username, err)
		return
	}
	if err := eventFilters(&filtersArgs); err != nil {
		log.Printf(`[%s] Events opening in error: %v`, user.Username, err)
		return
	}

	messages, errors := docker.Events(ctx, types.EventsOptions{Filters: filtersArgs})

	log.Printf(`[%s] Events streaming started`, user.Username)
	for {
		select {
		case <-ctx.Done():
			log.Printf(`[%s] Events streaming ended`, user.Username)
			return

		case message := <-messages:
			messageJSON, err := json.Marshal(message)
			if err != nil {
				log.Printf(`[%s] Events marshalling in error: %v`, user.Username, err)
				cancel()
			} else {
				output <- append(eventsPrefix, messageJSON...)
			}

		case err := <-errors:
			log.Printf(`[%s] Events reading in error: %v`, user.Username, err)
			cancel()
		}
	}
}

func streamLogs(ctx context.Context, cancel context.CancelFunc, user *auth.User, containerID string, output chan<- []byte) {
	logs, err := docker.ContainerLogs(ctx, containerID, types.ContainerLogsOptions{ShowStdout: true, ShowStderr: true, Follow: true, Tail: tailSize})
	defer cancel()

	if err != nil {
		log.Printf(`[%s] Logs opening in error: %v`, user.Username, err)
		return
	}
	defer logs.Close()

	log.Printf(`[%s] Logs streaming started for %s`, user.Username, containerID)
	scanner := bufio.NewScanner(logs)

	for scanner.Scan() {
		logLine := scanner.Bytes()
		if len(logLine) > ignoredByteLogSize {
			output <- append(logsPrefix, logLine[ignoredByteLogSize:]...)
		}
	}

	log.Printf(`[%s] Logs streaming ended for %s`, user.Username, containerID)
}

func streamStats(ctx context.Context, cancel context.CancelFunc, user *auth.User, containerID string, output chan<- []byte) {
	stats, err := docker.ContainerStats(ctx, containerID, true)
	defer cancel()

	if err != nil {
		log.Printf(`[%s] Stats opening in error for %s: %v`, user.Username, containerID, err)
		return
	}
	defer stats.Body.Close()

	scanner := bufio.NewScanner(stats.Body)
	log.Printf(`[%s] Stats streaming started for %s`, user.Username, containerID)

	for scanner.Scan() {
		output <- append(statsPrefix, scanner.Bytes()...)
	}

	log.Printf(`[%s] Stats streaming ended for %s`, user.Username, containerID)
}

func handleBusDemand(user *auth.User, name string, input []byte, demand *regexp.Regexp, cancel context.CancelFunc, output chan<- []byte, streamFn func(context.Context, context.CancelFunc, *auth.User, string, chan<- []byte)) context.CancelFunc {
	demandGroups := demand.FindSubmatch(input)
	if len(demandGroups) < 2 {
		log.Printf(`[%s] Unable to parse bus demand %s for %s`, user.Username, input, name)
	}

	action := string(demandGroups[1])

	containerID := ``
	if len(demandGroups) > 2 {
		containerID = string(demandGroups[2])
	}

	if action == stop && cancel != nil {
		log.Printf(`[%s] Stopping %s stream`, user.Username, name)
		cancel()
	} else if action == start {
		log.Printf(`[%s] Starting %s stream`, user.Username, name)

		if cancel != nil {
			log.Printf(`[%s] Cancelling previous %s stream`, user.Username, name)
			cancel()
		}

		ctx, newCancel := context.WithCancel(context.Background())
		go streamFn(ctx, newCancel, user, string(containerID), output)

		return newCancel
	}

	return nil
}

func busWebsocketHandler(w http.ResponseWriter, r *http.Request) {
	ws, user, err := upgradeAndAuth(w, r)
	if err != nil {
		log.Print(err)
		return
	}
	defer ws.Close()

	done := make(chan int)

	output := make(chan []byte)
	defer close(output)

	input := make(chan []byte)
	defer close(input)

	go readContent(user, ws, `streaming`, done, input)
	log.Printf(`[%s] Streaming started`, user.Username)

	var eventsCancelFunc context.CancelFunc
	var logsCancelFunc context.CancelFunc
	var statsCancelFunc context.CancelFunc

	for {
		select {
		case <-done:
			log.Printf(`[%s] Streaming ended`, user.Username)
			return

		case inputBytes := <-input:
			if eventsDemand.Match(inputBytes) {
				eventsCancelFunc = handleBusDemand(user, `events`, inputBytes, eventsDemand, eventsCancelFunc, output, streamEvents)
				if eventsCancelFunc != nil {
					defer eventsCancelFunc()
				}
			} else if logsDemand.Match(inputBytes) {
				logsCancelFunc = handleBusDemand(user, `logs`, inputBytes, logsDemand, logsCancelFunc, output, streamLogs)
				if logsCancelFunc != nil {
					defer logsCancelFunc()
				}
			} else if statsDemand.Match(inputBytes) {
				statsCancelFunc = handleBusDemand(user, `stats`, inputBytes, statsDemand, statsCancelFunc, output, streamStats)
				if statsCancelFunc != nil {
					defer statsCancelFunc()
				}
			}

		case outputBytes := <-output:
			if err = ws.WriteMessage(websocket.TextMessage, outputBytes); err != nil {
				log.Printf(`[%s] Error while writing to streaming: %v`, user.Username, err)
				close(done)
			}
		}
	}
}

// WebsocketHandler for Docker Websocket request. Should be use with net/http
type WebsocketHandler struct {
}

func (handler WebsocketHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	urlPath := []byte(r.URL.Path)

	if busWebsocketRequest.Match(urlPath) {
		busWebsocketHandler(w, r)
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 Andreas Koch. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package repository

import (
	"fmt"
	"github.com/andreaskoch/allmark/markdown"
	"github.com/andreaskoch/allmark/path"
	"github.com/andreaskoch/allmark/util"
	"github.com/andreaskoch/allmark/view"
	"github.com/andreaskoch/allmark/watcher"
	"io/ioutil"
	"path/filepath"
	"sort"
	"time"
)

const (
	FilesDirectoryName = "files"
)

type Item struct {
	*view.Model

	Modified chan bool
	Moved    chan bool

	Level  int
	Files  *FileIndex
	Parent *Item
	Childs Items

	Title            string
	Description      string
	RawContent       []string
	ConvertedContent string
	MetaData         MetaData

	AbsolutePath string
	RelativePath string

	ChildsReady  chan bool
	newChild     chan *Item
	deletedChild chan *Item

	directory string
	path      string
	isVirtual bool

	rootPathProvider     *path.Provider
	relativePathProvider *path.Provider
	filePathProvider     *path.Provider
}

func newItem(rootPathProvider *path.Provider, parent *Item, itemPath string, level int, newItem chan *Item, deletedItem chan *Item) (*Item, error) {

	// abort if path does not exist
	if !util.PathExists(itemPath) {
		return nil, fmt.Errorf("The path %q does not exist.", itemPath)
	}

	// abort if path is reserved
	if isReservedDirectory(itemPath) {
		return nil, fmt.Errorf("The path %q is using a reserved name and cannot be an item.", itemPath)
	}

	// check if its a virtual item or a markdown item
	isVirtualItem := false
	itemDirectory := filepath.Dir(itemPath)

	if isDirectory, _ := util.IsDirectory(itemPath); isDirectory {

		if found, filepath := findMarkdownFileInDirectory(itemPath); found {

			itemDirectory = itemPath
			itemPath = filepath

		} else {

			isVirtualItem = true
			itemDirectory = itemPath

		}

	} else if !markdown.IsMarkdownFile(itemPath) {

		// the supplied item path does not point to a markdown file
		return nil, fmt.Errorf("%q is not a markdown file.", itemPath)
	}

	// create a relative path provider
	pathProviderDirectory := itemDirectory
	if level > 0 {
		pathProviderDirectory = filepath.Dir(itemDirectory)
	}

	relativePathProvider := rootPathProvider.New(pathProviderDirectory)

	// create a file path provider
	filePathProvider := rootPathProvider.New(itemDirectory)

	// create the file index
	filesDirectory := filepath.Join(itemDirectory, FilesDirectoryName)
	fileIndex, err := newFileIndex(rootPathProvider, filesDirectory)
	if err != nil {
		return nil, fmt.Errorf("Could not create a file index for folder %q.\nError: %s\n", filesDirectory, err)
	}

	// create the item
	item := &Item{

		ChildsReady: make(chan bool),
		Modified:    make(chan bool),
		Moved:       make(chan bool),

		Parent: parent,
		Level:  level,
		Files:  fileIndex,

		newChild:     newItem,
		deletedChild: deletedItem,

		rootPathProvider:     rootPathProvider,
		relativePathProvider: relativePathProvider,
		filePathProvider:     filePathProvider,

		directory: itemDirectory,
		path:      itemPath,
		isVirtual: isVirtualItem,
	}

	// assign paths
	item.RelativePath = item.RelativePathProvider().GetWebRoute(item)
	item.AbsolutePath = item.RootPathProvider().GetWebRoute(item)

	// find childs
	item.updateChilds()

	// look for changes to the markdown file (if the item is not virtual)
	if !item.isVirtual {
		item.startFileWatcher()
	}

	// look for changes in the item directory
	go func() {
		var skipFunc = func(path string) bool {
			isItem := path == item.path
			isReserved := isReservedDirectory(path)
			return isItem || isReserved
		}

		folderWatcher := watcher.NewFolderWatcher(itemDirectory, false, skipFunc).Start()

		for folderWatcher.IsRunning() {

			select {
			case <-folderWatcher.Change:

				if item.isVirtual {
					if found, filepath := findMarkdownFileInDirectory(itemDirectory); found {

						fmt.Printf("Converting the virtual item %q into a physical item\n", item)

						// make the item physical
						item.path = filepath
						item.isVirtual = false
						item.startFileWatcher()
					}
				}

				fmt.Printf("Updating the childs of item %q\n", item)
				item.updateChilds()

				go func() {
					time.Sleep(time.Second * 3)
					item.Modified <- true
				}()
			}

		}
	}()

	// look for file changes
	go func() {
		for {
			select {
			case <-item.Files.Changed:
				fmt.Printf("Files of item %q changed\n", item)
				item.Modified <- true
			case <-item.Files.Stopped:
				fmt.Printf("File index watcher for item %q was stopped\n", item)
				break
			}
		}
	}()

	return item, nil
}

func (item *Item) startFileWatcher() (started bool, itemWatcher *watcher.FileWatcher) {
	if isFile, _ := util.IsFile(item.path); !isFile {
		return false, nil
	}

	fileWatcher := watcher.NewFileWatcher(item.path).Start()

	go func() {

		for fileWatcher.IsRunning() {

			select {
			case <-fileWatcher.Modified:

				fmt.Printf("Item %q has been modified\n", item)
				item.Modified <- true

			case <-fileWatcher.Moved:

				fmt.Printf("Item %q has been moved\n", item)
				item.Moved <- true
			}

		}
	}()

	return true, fileWatcher
}

func (item *Item) String() string {
	return fmt.Sprintf("%s", item.RootPathProvider().GetWebRoute(item))
}

func (item *Item) Sort() {

	for _, child := range item.Childs {
		child.Sort()
	}

	sort.Sort(item.Childs)
}

func (item *Item) Less(otherItem *Item) bool {
	return item.MetaData.Date.After(otherItem.MetaData.Date)
}

func (item *Item) Path() string {
	return item.path
}

func (item *Item) Directory() string {
	return item.directory
}

func (item *Item) PathType() string {
	return path.PatherTypeItem
}

func (item *Item) IsVirtual() bool {
	return item.isVirtual
}

func (item *Item) RootPathProvider() *path.Provider {
	return item.rootPathProvider
}

func (item *Item) RelativePathProvider() *path.Provider {
	return item.relativePathProvider
}

func (item *Item) FilePathProvider() *path.Provider {
	return item.filePathProvider
}

func (item *Item) HasChild(itemDirectory string) bool {
	if item.Childs == nil {
		return false
	}

	for _, childItem := range item.Childs {
		if childItem.Directory() == itemDirectory {
			return true
		}
	}

	return false
}

func (item *Item) updateChilds() {

	childItemDirectories := item.getChildItemDirectories()

	// check if the child list needs initialization
	if item.Childs == nil {
		item.Childs = Items{}
	}

	// add new childs
	for _, childItemDirectory := range childItemDirectories {

		if item.HasChild(childItemDirectory) {
			continue // Child already present
		}

		if child, err := newItem(item.RootPathProvider(), item, childItemDirectory, item.Level+1, item.newChild, item.deletedChild); err == nil {

			// inform others about the new child
			go func() {
				item.newChild <- child
			}()

			item.Childs = append(item.Childs, child) // append new child

		} else {
			fmt.Printf("Could not create a item for folder %q. Error: %s\n", childItemDirectory, err)
		}
	}

	// remove deleted childs
	newChildList := Items{}
	for _, child := range item.Childs {

		if util.SliceContainsElement(childItemDirectories, child.Directory()) {

			newChildList = append(newChildList, child)

		} else {

			// Todo: stop all go routines on this child

			// inform others about the removed child
			go func() {
				item.deletedChild <- child
			}()

		}
	}

	// assign new child list
	go func() {
		item.ChildsReady <- true
	}()

	item.Childs = newChildList
}

func (item *Item) getChildItemDirectories() []string {

	directory := item.Directory()
	directories := make([]string, 0)

	directoryEntries, _ := ioutil.ReadDir(directory)
	for _, entry := range directoryEntries {

		if !entry.IsDir() {
			continue // skip files
		}

		childDirectory := filepath.Join(directory, entry.Name())
		if isReservedDirectory(childDirectory) {
			continue // skip reserved directories
		}

		// append directory
		directories = append(directories, childDirectory)
	}

	return directories
}
<commit_msg>Sort items by title if the dates are equal<commit_after>// Copyright 2013 Andreas Koch. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package repository

import (
	"fmt"
	"github.com/andreaskoch/allmark/markdown"
	"github.com/andreaskoch/allmark/path"
	"github.com/andreaskoch/allmark/util"
	"github.com/andreaskoch/allmark/view"
	"github.com/andreaskoch/allmark/watcher"
	"io/ioutil"
	"path/filepath"
	"sort"
	"time"
)

const (
	FilesDirectoryName = "files"
)

type Item struct {
	*view.Model

	Modified chan bool
	Moved    chan bool

	Level  int
	Files  *FileIndex
	Parent *Item
	Childs Items

	Title            string
	Description      string
	RawContent       []string
	ConvertedContent string
	MetaData         MetaData

	AbsolutePath string
	RelativePath string

	ChildsReady  chan bool
	newChild     chan *Item
	deletedChild chan *Item

	directory string
	path      string
	isVirtual bool

	rootPathProvider     *path.Provider
	relativePathProvider *path.Provider
	filePathProvider     *path.Provider
}

func newItem(rootPathProvider *path.Provider, parent *Item, itemPath string, level int, newItem chan *Item, deletedItem chan *Item) (*Item, error) {

	// abort if path does not exist
	if !util.PathExists(itemPath) {
		return nil, fmt.Errorf("The path %q does not exist.", itemPath)
	}

	// abort if path is reserved
	if isReservedDirectory(itemPath) {
		return nil, fmt.Errorf("The path %q is using a reserved name and cannot be an item.", itemPath)
	}

	// check if its a virtual item or a markdown item
	isVirtualItem := false
	itemDirectory := filepath.Dir(itemPath)

	if isDirectory, _ := util.IsDirectory(itemPath); isDirectory {

		if found, filepath := findMarkdownFileInDirectory(itemPath); found {

			itemDirectory = itemPath
			itemPath = filepath

		} else {

			isVirtualItem = true
			itemDirectory = itemPath

		}

	} else if !markdown.IsMarkdownFile(itemPath) {

		// the supplied item path does not point to a markdown file
		return nil, fmt.Errorf("%q is not a markdown file.", itemPath)
	}

	// create a relative path provider
	pathProviderDirectory := itemDirectory
	if level > 0 {
		pathProviderDirectory = filepath.Dir(itemDirectory)
	}

	relativePathProvider := rootPathProvider.New(pathProviderDirectory)

	// create a file path provider
	filePathProvider := rootPathProvider.New(itemDirectory)

	// create the file index
	filesDirectory := filepath.Join(itemDirectory, FilesDirectoryName)
	fileIndex, err := newFileIndex(rootPathProvider, filesDirectory)
	if err != nil {
		return nil, fmt.Errorf("Could not create a file index for folder %q.\nError: %s\n", filesDirectory, err)
	}

	// create the item
	item := &Item{

		ChildsReady: make(chan bool),
		Modified:    make(chan bool),
		Moved:       make(chan bool),

		Parent: parent,
		Level:  level,
		Files:  fileIndex,

		newChild:     newItem,
		deletedChild: deletedItem,

		rootPathProvider:     rootPathProvider,
		relativePathProvider: relativePathProvider,
		filePathProvider:     filePathProvider,

		directory: itemDirectory,
		path:      itemPath,
		isVirtual: isVirtualItem,
	}

	// assign paths
	item.RelativePath = item.RelativePathProvider().GetWebRoute(item)
	item.AbsolutePath = item.RootPathProvider().GetWebRoute(item)

	// find childs
	item.updateChilds()

	// look for changes to the markdown file (if the item is not virtual)
	if !item.isVirtual {
		item.startFileWatcher()
	}

	// look for changes in the item directory
	go func() {
		var skipFunc = func(path string) bool {
			isItem := path == item.path
			isReserved := isReservedDirectory(path)
			return isItem || isReserved
		}

		folderWatcher := watcher.NewFolderWatcher(itemDirectory, false, skipFunc).Start()

		for folderWatcher.IsRunning() {

			select {
			case <-folderWatcher.Change:

				if item.isVirtual {
					if found, filepath := findMarkdownFileInDirectory(itemDirectory); found {

						fmt.Printf("Converting the virtual item %q into a physical item\n", item)

						// make the item physical
						item.path = filepath
						item.isVirtual = false
						item.startFileWatcher()
					}
				}

				fmt.Printf("Updating the childs of item %q\n", item)
				item.updateChilds()

				go func() {
					time.Sleep(time.Second * 3)
					item.Modified <- true
				}()
			}

		}
	}()

	// look for file changes
	go func() {
		for {
			select {
			case <-item.Files.Changed:
				fmt.Printf("Files of item %q changed\n", item)
				item.Modified <- true
			case <-item.Files.Stopped:
				fmt.Printf("File index watcher for item %q was stopped\n", item)
				break
			}
		}
	}()

	return item, nil
}

func (item *Item) startFileWatcher() (started bool, itemWatcher *watcher.FileWatcher) {
	if isFile, _ := util.IsFile(item.path); !isFile {
		return false, nil
	}

	fileWatcher := watcher.NewFileWatcher(item.path).Start()

	go func() {

		for fileWatcher.IsRunning() {

			select {
			case <-fileWatcher.Modified:

				fmt.Printf("Item %q has been modified\n", item)
				item.Modified <- true

			case <-fileWatcher.Moved:

				fmt.Printf("Item %q has been moved\n", item)
				item.Moved <- true
			}

		}
	}()

	return true, fileWatcher
}

func (item *Item) String() string {
	return fmt.Sprintf("%s", item.RootPathProvider().GetWebRoute(item))
}

func (item *Item) Sort() {

	for _, child := range item.Childs {
		child.Sort()
	}

	sort.Sort(item.Childs)
}

func (item *Item) Less(otherItem *Item) bool {

	// compare title if dates are equal
	if item.MetaData.Date.Equal(otherItem.MetaData.Date) {
		return item.Title < otherItem.Title
	}

	return item.MetaData.Date.After(otherItem.MetaData.Date)
}

func (item *Item) Path() string {
	return item.path
}

func (item *Item) Directory() string {
	return item.directory
}

func (item *Item) PathType() string {
	return path.PatherTypeItem
}

func (item *Item) IsVirtual() bool {
	return item.isVirtual
}

func (item *Item) RootPathProvider() *path.Provider {
	return item.rootPathProvider
}

func (item *Item) RelativePathProvider() *path.Provider {
	return item.relativePathProvider
}

func (item *Item) FilePathProvider() *path.Provider {
	return item.filePathProvider
}

func (item *Item) HasChild(itemDirectory string) bool {
	if item.Childs == nil {
		return false
	}

	for _, childItem := range item.Childs {
		if childItem.Directory() == itemDirectory {
			return true
		}
	}

	return false
}

func (item *Item) updateChilds() {

	childItemDirectories := item.getChildItemDirectories()

	// check if the child list needs initialization
	if item.Childs == nil {
		item.Childs = Items{}
	}

	// add new childs
	for _, childItemDirectory := range childItemDirectories {

		if item.HasChild(childItemDirectory) {
			continue // Child already present
		}

		if child, err := newItem(item.RootPathProvider(), item, childItemDirectory, item.Level+1, item.newChild, item.deletedChild); err == nil {

			// inform others about the new child
			go func() {
				item.newChild <- child
			}()

			item.Childs = append(item.Childs, child) // append new child

		} else {
			fmt.Printf("Could not create a item for folder %q. Error: %s\n", childItemDirectory, err)
		}
	}

	// remove deleted childs
	newChildList := Items{}
	for _, child := range item.Childs {

		if util.SliceContainsElement(childItemDirectories, child.Directory()) {

			newChildList = append(newChildList, child)

		} else {

			// Todo: stop all go routines on this child

			// inform others about the removed child
			go func() {
				item.deletedChild <- child
			}()

		}
	}

	// assign new child list
	go func() {
		item.ChildsReady <- true
	}()

	item.Childs = newChildList
}

func (item *Item) getChildItemDirectories() []string {

	directory := item.Directory()
	directories := make([]string, 0)

	directoryEntries, _ := ioutil.ReadDir(directory)
	for _, entry := range directoryEntries {

		if !entry.IsDir() {
			continue // skip files
		}

		childDirectory := filepath.Join(directory, entry.Name())
		if isReservedDirectory(childDirectory) {
			continue // skip reserved directories
		}

		// append directory
		directories = append(directories, childDirectory)
	}

	return directories
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"crypto/tls"
	"errors"
	"fmt"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/fclairamb/ftpserver/server"
	"log"
	"os"
	"path/filepath"
	"strings"
	"time"
)

// SampleDriver defines a very basic serverftp driver
type S3Driver struct {
	baseDir   string
	s3Session *session.Session
	s3Service *s3.S3
}

func (d *S3Driver) WelcomeUser(cc server.ClientContext) (string, error) {
	cc.SetDebug(true)
	return "Welcome to the FTP server for S3", nil
}

func (d *S3Driver) AuthUser(cc server.ClientContext, user, pass string) (server.ClientHandlingDriver, error) {

	if user != FTP_USER {
		log.Println("username does not match expected user", user)
		return nil, fmt.Errorf("incorrect username: %s", user)
	}

	if pass != FTP_PASSWD {
		log.Println("incorrect password")
		return nil, errors.New("incorrect password")
	}

	_, err := session.NewSession()
	if err != nil {
		log.Println("error creating S3 session (check AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY env vars on server):", err)
		return nil, err
	}

	return d, nil
}

func (d *S3Driver) GetTLSConfig() (*tls.Config, error) {
	return nil, errors.New("TLS not implemented")
}

func (d *S3Driver) ChangeDirectory(cc server.ClientContext, directory string) error {
	// Query S3 to see if the directory exists.  If it does update the baseDir variable to emulate cd-ing to that
	// directory

	var err error
	var dirname string

	// join paths and keep relative to root dir.
	if dirname, err = filepath.Rel("/", directory); err != nil {
		return err
	}

	if dirname == "." {
		dirname = ""
	}

	// directories in S3 must always have a trailing slash (filepath.Rel removes this)
	if len(dirname) > 0 && !strings.HasSuffix(dirname, "/") {
		dirname += "/"
	}

	delimiter := "/"
	params := &s3.ListObjectsV2Input{
		Bucket:    &S3_BUCKET_NAME,
		Prefix:    &dirname,
		Delimiter: &delimiter, // limits the search to directories (they have a trailing slash)
	}

	var resp *s3.ListObjectsV2Output
	if resp, err = d.s3Service.ListObjectsV2(params); err != nil {
		return err
	}

	if *resp.KeyCount == 0 {
		return errors.New("No such directory: " + directory)
	}

	d.baseDir = dirname

	return nil
}

func (d *S3Driver) MakeDirectory(cc server.ClientContext, directory string) error {

	dirname := directory
	if !strings.HasSuffix(dirname, "/") {
		dirname += "/"
	}

	params := &s3.PutObjectInput{
		Bucket: &S3_BUCKET_NAME,
		Key:    &dirname,
		Body:   bytes.NewReader([]byte("")),
	}

	var err error
	if _, err = d.s3Service.PutObject(params); err != nil {
		return err
	}

	return nil
}

func (d *S3Driver) ListFiles(cc server.ClientContext) ([]os.FileInfo, error) {

	delimiter := "/" // delimiter keeps the listing from being recursive
	params := &s3.ListObjectsV2Input{
		Bucket:    &S3_BUCKET_NAME,
		Prefix:    &d.baseDir,
		Delimiter: &delimiter,
	}

	var err error
	var resp *s3.ListObjectsV2Output
	if resp, err = d.s3Service.ListObjectsV2(params); err != nil {
		return nil, err
	}

	files := []os.FileInfo{}

	// directories other than CWD
	for _, dir := range resp.CommonPrefixes {
		var dirInfo os.FileInfo

		if dirInfo, err = d.GetFileInfo(cc, filepath.Join("/", *dir.Prefix)+"/"); err != nil {
			return nil, err
		}

		files = append(files, dirInfo)
	}

	// files and CWD
	for _, f := range resp.Contents {

		// don't list CWD in the list of files
		if *f.Key == d.baseDir {
			continue
		}

		var fi os.FileInfo
		if fi, err = d.fakeFileInfo(*f.Key, *f.Size, *f.LastModified); err != nil {
			return nil, err
		}
		files = append(files, fi)
	}

	return files, nil
}

func (d *S3Driver) UserLeft(cc server.ClientContext) {

}

func (d *S3Driver) OpenFile(cc server.ClientContext, path string, flag int) (server.FileStream, error) {
	// our implementation uses an interface that mimics a file but uploads/downloads to S3
	var err error
	var s3file *S3VirtualFile

	if s3file, err = NewS3VirtualFile(path, d.s3Session, d.s3Service); err != nil {
		return nil, err
	}

	return s3file, nil
}

func (d *S3Driver) GetFileInfo(cc server.ClientContext, path string) (os.FileInfo, error) {

	var err error
	var relPath string
	if relPath, err = filepath.Rel("/", path); err != nil {
		return nil, err
	}

	// dirs need to have a trailing slash
	if strings.HasSuffix(path, "/") {
		relPath += "/"
	}

	params := &s3.GetObjectInput{
		Bucket: &S3_BUCKET_NAME,
		Key:    &relPath,
	}

	req, resp := d.s3Service.GetObjectRequest(params)

	if err := req.Send(); err != nil {
		return nil, err
	}

	// resp.ContentLength and LastModified are sometimes nil (!) so check for this state
	var objectSize int64
	if resp.ContentLength != nil {
		objectSize = *resp.ContentLength
	}

	var modTime time.Time
	if resp.LastModified != nil {
		modTime = *resp.LastModified
	}

	var f os.FileInfo
	if f, err = d.fakeFileInfo(relPath, objectSize, modTime); err != nil {
		return nil, err
	}

	return f, nil
}

// Return a fakeInfo struct that satisfies the os.FileInfo interface, emulating a file from an S3 object
func (d *S3Driver) fakeFileInfo(name string, size int64, modTime time.Time) (os.FileInfo, error) {
	var err error

	displayPath := name
	// make the path we display relative to the current working directory
	if strings.HasPrefix(displayPath, d.baseDir) {
		if displayPath, err = filepath.Rel(d.baseDir, displayPath); err != nil {
			return nil, err
		}
	}

	isDir := strings.HasSuffix(name, "/")

	mode := os.FileMode(0666)
	if isDir {
		mode = mode | os.ModeDir
	}

	f := fakeInfo{
		name:    displayPath,
		size:    size,
		mode:    mode,
		modTime: modTime,
		isDir:   isDir,
	}

	return f, nil
}

func (d *S3Driver) CanAllocate(cc server.ClientContext, size int) (bool, error) {
	return true, nil
}

func (d *S3Driver) ChmodFile(cc server.ClientContext, path string, mode os.FileMode) error {
	return errors.New("ChmodFile not implemented")
}

func (d *S3Driver) DeleteFile(cc server.ClientContext, path string) error {
	params := &s3.DeleteObjectInput{
		Bucket: &S3_BUCKET_NAME,
		Key:    &path,
	}

	_, err := d.s3Service.DeleteObject(params)
	return err
}

func (d *S3Driver) RenameFile(cc server.ClientContext, from, to string) error {
	// S3 doesn't have rename (or move) so we need to copy (CopyObject) and delete the old one (DeleteObject).
	var err error

	var relFrom, relTo string
	if relFrom, err = filepath.Rel("/", from); err != nil {
		return err
	}

	if relTo, err = filepath.Rel("/", to); err != nil {
		return err
	}

	copySrc := S3_BUCKET_NAME + "/" + relFrom
	copyParams := &s3.CopyObjectInput{
		Bucket:     &S3_BUCKET_NAME,
		Key:        &relTo,
		CopySource: &copySrc,
	}

	if _, err = d.s3Service.CopyObject(copyParams); err != nil {
		return err
	}

	if err = d.DeleteFile(cc, from); err != nil {
		return err
	}

	return nil
}

func (d *S3Driver) GetSettings() *server.Settings {
	config := server.Settings{
		Host:           "0.0.0.0",
		Port:           FTP_PORT,
		MaxConnections: 300,
		MonitorOn:      true,
		MonitorPort:    3379,
	}
	return &config
}

func NewS3Driver() (*S3Driver, error) {

	var err error

	driver := &S3Driver{}

	if driver.s3Session, err = session.NewSession(); err != nil {
		log.Println("error creating S3 session:", err)
		return nil, err
	}

	driver.s3Service = s3.New(driver.s3Session)

	// an empty string corresponds to the root of the S3 bucket
	driver.baseDir = ""

	return driver, nil
}
<commit_msg>refactor all methods to use getS3Key to use common logic.  Also implemented a recursive delete.<commit_after>package main

import (
	"bytes"
	"crypto/tls"
	"errors"
	"fmt"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/fclairamb/ftpserver/server"
	"log"
	"os"
	"path/filepath"
	"strings"
	"time"
)

type S3Driver struct {
	baseDir   string
	s3Session *session.Session
	s3Service *s3.S3
}

func (d *S3Driver) WelcomeUser(cc server.ClientContext) (string, error) {
	cc.SetDebug(true)
	return "Welcome to the FTP server for S3", nil
}

func (d *S3Driver) AuthUser(cc server.ClientContext, user, pass string) (server.ClientHandlingDriver, error) {

	if user != FTP_USER {
		log.Println("username does not match expected user", user)
		return nil, fmt.Errorf("incorrect username: %s", user)
	}

	if pass != FTP_PASSWD {
		log.Println("incorrect password")
		return nil, errors.New("incorrect password")
	}

	_, err := session.NewSession()
	if err != nil {
		log.Println("error creating S3 session (check AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY env vars on server):", err)
		return nil, err
	}

	return d, nil
}

func (d *S3Driver) GetTLSConfig() (*tls.Config, error) {
	return nil, errors.New("TLS not implemented")
}

func (d *S3Driver) ChangeDirectory(cc server.ClientContext, directory string) error {
	// Query S3 to see if the directory exists.  If it does update the baseDir variable to emulate cd-ing to that
	// directory

	var err error
	var dirname string
	if dirname, err = getS3Key(directory + "/"); err != nil {
		return err
	}

	delimiter := "/"
	params := &s3.ListObjectsV2Input{
		Bucket:    &S3_BUCKET_NAME,
		Prefix:    &dirname,
		Delimiter: &delimiter, // limits the search to directories (they have a trailing slash)
	}

	var resp *s3.ListObjectsV2Output
	if resp, err = d.s3Service.ListObjectsV2(params); err != nil {
		return err
	}

	if *resp.KeyCount == 0 {
		return errors.New("No such directory: " + directory)
	}

	d.baseDir = dirname

	return nil
}

func (d *S3Driver) MakeDirectory(cc server.ClientContext, directory string) error {

	var err error
	var dirname string
	if dirname, err = getS3Key(directory + "/"); err != nil {
		return err
	}

	params := &s3.PutObjectInput{
		Bucket: &S3_BUCKET_NAME,
		Key:    &dirname,
		Body:   bytes.NewReader([]byte("")),
	}

	if _, err = d.s3Service.PutObject(params); err != nil {
		return err
	}

	return nil
}

func (d *S3Driver) ListFiles(cc server.ClientContext) ([]os.FileInfo, error) {

	delimiter := "/" // delimiter keeps the listing from being recursive
	params := &s3.ListObjectsV2Input{
		Bucket:    &S3_BUCKET_NAME,
		Prefix:    &d.baseDir,
		Delimiter: &delimiter,
	}

	var err error
	var resp *s3.ListObjectsV2Output
	if resp, err = d.s3Service.ListObjectsV2(params); err != nil {
		return nil, err
	}

	files := []os.FileInfo{}

	// directories other than CWD
	for _, dir := range resp.CommonPrefixes {
		var dirInfo os.FileInfo

		if dirInfo, err = d.GetFileInfo(cc, filepath.Join("/", *dir.Prefix)+"/"); err != nil {
			return nil, err
		}

		files = append(files, dirInfo)
	}

	// files and CWD
	for _, f := range resp.Contents {

		// don't list CWD in the list of files
		if *f.Key == d.baseDir {
			continue
		}

		var fi os.FileInfo
		if fi, err = d.getFakeFileInfo(*f.Key, *f.Size, *f.LastModified); err != nil {
			return nil, err
		}
		files = append(files, fi)
	}

	return files, nil
}

func (d *S3Driver) UserLeft(cc server.ClientContext) {

}

func (d *S3Driver) OpenFile(cc server.ClientContext, path string, flag int) (server.FileStream, error) {
	// our implementation uses an interface that mimics a file but uploads/downloads to S3
	var err error
	var s3file *S3VirtualFile

	if s3file, err = NewS3VirtualFile(path, d.s3Session, d.s3Service); err != nil {
		return nil, err
	}

	return s3file, nil
}

func (d *S3Driver) GetFileInfo(cc server.ClientContext, path string) (os.FileInfo, error) {

	var err error
	var relPath string
	if relPath, err = getS3Key(path); err != nil {
		return nil, err
	}

	params := &s3.GetObjectInput{
		Bucket: &S3_BUCKET_NAME,
		Key:    &relPath,
	}

	req, resp := d.s3Service.GetObjectRequest(params)

	if err := req.Send(); err != nil {
		return nil, err
	}

	// resp.ContentLength and LastModified are sometimes nil (!) so check for this state
	var objectSize int64
	if resp.ContentLength != nil {
		objectSize = *resp.ContentLength
	}

	var modTime time.Time
	if resp.LastModified != nil {
		modTime = *resp.LastModified
	}

	var f os.FileInfo
	if f, err = d.getFakeFileInfo(relPath, objectSize, modTime); err != nil {
		return nil, err
	}

	return f, nil
}

func (d *S3Driver) CanAllocate(cc server.ClientContext, size int) (bool, error) {
	return true, nil
}

func (d *S3Driver) ChmodFile(cc server.ClientContext, path string, mode os.FileMode) error {
	return errors.New("ChmodFile not implemented")
}

func (d *S3Driver) DeleteFile(cc server.ClientContext, path string) error {
	// list objects matching the path, then use DeleteObjects on all of them.  Needed because you must delete all
	// child key/objects belonging to a directory key before deleting that key

	var err error
	var relPath string
	if relPath, err = getS3Key(path); err != nil {
		return err
	}

	var isDir bool
	if isDir, err = d.isS3Dir(relPath); err != nil {
		return err
	}

	if isDir {
		relPath += "/"
	}

	listParams := &s3.ListObjectsV2Input{
		Bucket:    &S3_BUCKET_NAME,
		Prefix:    &relPath,
		//Delimiter: &delimiter, // empty delim makes this recursive
	}

	var resp *s3.ListObjectsV2Output
	if resp, err = d.s3Service.ListObjectsV2(listParams); err != nil {
		return err
	}

	var delObjects []*s3.ObjectIdentifier
	for _, f := range resp.Contents {
		delObjects = append(delObjects, &s3.ObjectIdentifier{Key:f.Key})
	}

	delParams := &s3.DeleteObjectsInput{
		Bucket: &S3_BUCKET_NAME,
		Delete: &s3.Delete{Objects:delObjects},
	}

	if _, err = d.s3Service.DeleteObjects(delParams); err != nil {
		return err
	}

	return nil
}

// returns true or false depending on if the path exists as a file (no trailing slash) or a directory (trailing slash)
// in S3.
func (d *S3Driver) isS3Dir(path string) (bool, error) {
	var err, errFile, errDir error
	var s3Path string

	log.Println("DEBUG", path)
	if s3Path, err = getS3Key(path); err != nil {
		return false, err
	}

	// see if it's file
	_, errFile = d.GetFileInfo(nil, s3Path)
	if errFile == nil {
		return false, nil
	}

	// see if it's a directory
	_, errDir = d.GetFileInfo(nil, s3Path + "/")
	if errDir == nil {
		return true, nil
	}

	return false, errors.New("No such file or directory")
}

func (d *S3Driver) RenameFile(cc server.ClientContext, from, to string) error {
	// S3 doesn't have rename (or move) so we need to copy (CopyObject) and delete the old one (DeleteObject).
	var err error
	var relFrom string
	if relFrom, err = getS3Key(from); err != nil {
		return err
	}

	var relTo string
	if relTo, err = getS3Key(to); err != nil {
		return err
	}

	copySrc := S3_BUCKET_NAME + "/" + relFrom
	copyParams := &s3.CopyObjectInput{
		Bucket:     &S3_BUCKET_NAME,
		Key:        &relTo,
		CopySource: &copySrc,
	}

	if _, err = d.s3Service.CopyObject(copyParams); err != nil {
		return err
	}

	if err = d.DeleteFile(cc, from); err != nil {
		return err
	}

	return nil
}

func (d *S3Driver) GetSettings() *server.Settings {
	config := server.Settings{
		Host:           "0.0.0.0",
		Port:           FTP_PORT,
		MaxConnections: 300,
		MonitorOn:      true,
		MonitorPort:    3379,
	}
	return &config
}

// Return a fakeInfo struct that satisfies the os.FileInfo interface, emulating a file from an S3 object
func (d *S3Driver) getFakeFileInfo(name string, size int64, modTime time.Time) (os.FileInfo, error) {
	var err error

	displayPath := name
	// make the path we display relative to the current working directory
	if strings.HasPrefix(displayPath, d.baseDir) {
		if displayPath, err = filepath.Rel(d.baseDir, displayPath); err != nil {
			return nil, err
		}
	}

	isDir := strings.HasSuffix(name, "/")

	mode := os.FileMode(0666)
	if isDir {
		mode = mode | os.ModeDir
	}

	f := fakeInfo{
		name:    displayPath,
		size:    size,
		mode:    mode,
		modTime: modTime,
		isDir:   isDir,
	}

	return f, nil
}

func NewS3Driver() (*S3Driver, error) {

	var err error

	driver := &S3Driver{}

	if driver.s3Session, err = session.NewSession(); err != nil {
		log.Println("error creating S3 session:", err)
		return nil, err
	}

	driver.s3Service = s3.New(driver.s3Session)

	driver.baseDir = ""

	return driver, nil
}

// cleans the input path and queries S3 to see if it's a directory or file key.  Will return an error if it cannot find
// either a file or directory key (with a trailing slash)
func getS3Key(path string) (string, error) {
	var err error
	s3Key := path

	// join paths and keep relative to root dir.
	if strings.HasPrefix(s3Key, "/") {
		if s3Key, err = filepath.Rel("/", path); err != nil {
			return "", err
		}
	}

	// "/" relative to "/" is "." but in S3 this should be ""
	if s3Key == "." {
		s3Key = ""
	}

	// directories in S3 must always have a trailing slash (filepath.Rel removes this)
	if strings.HasSuffix(path, "/") && len(s3Key) > 0 && !strings.HasSuffix(s3Key, "/") {
		s3Key += "/"
	}

	return s3Key, nil
}
<|endoftext|>


<commit_before>package main

import (
	"io/ioutil"
	"fmt"
	"os"
	"path"
)

type StateMachine struct {
	Name string
	Path string
	Usage string
	States []string
}

var globalStateMachines []StateMachine

func initMachines() {
	machines, err := ioutil.ReadDir(globalConfig.StateMachinePath)
	if err != nil {
		fmt.Printf("error listing StateMachinePath %s directory: %s\n", globalConfig.StateMachinePath, err)
		os.Exit(1)
	}

	for _, machine := range machines {

		machinePath := path.Join(globalConfig.StateMachinePath, machine.Name())

		machineInfo, err := os.Stat(machinePath)
		if err != nil {
			fmt.Printf("error stat'ing %s: %s\n", machinePath, err)
		}

		if machineInfo.IsDir() {
			machineStruct := StateMachine{Name: machine.Name(), Path: machinePath}

			states, err := ioutil.ReadDir(machinePath)
			if err != nil {
				fmt.Printf("error listing %s directory: %s\n", machine.Name(), err)
				os.Exit(1)
			}

			hasStart := false
			for _, stateInfo := range states {
				if stateInfo.Mode().Perm() & 0111 > 0 {
					machineStruct.States = append(machineStruct.States, stateInfo.Name())

					if stateInfo.Name() == "start" {
						hasStart = true
					}
				}
			}

			if !hasStart {
				fmt.Printf("state machine directory for %s has no start state\n", machineStruct.Name)
				os.Exit(1)
			}

			machineStruct.Usage = "TODO"

			globalStateMachines = append(globalStateMachines, machineStruct)
		}
	}
}
<commit_msg>Add usage retrieving from start executable<commit_after>package main

import (
	"io/ioutil"
	"fmt"
	"os"
	"os/exec"
	"path"
)

type StateMachine struct {
	Name string
	Path string
	Usage string
	States []string
}

var globalStateMachines []StateMachine

func initMachines() {
	machines, err := ioutil.ReadDir(globalConfig.StateMachinePath)
	if err != nil {
		fmt.Printf("error listing StateMachinePath %s directory: %s\n", globalConfig.StateMachinePath, err)
		os.Exit(1)
	}

	for _, machine := range machines {

		machinePath := path.Join(globalConfig.StateMachinePath, machine.Name())

		machineInfo, err := os.Stat(machinePath)
		if err != nil {
			fmt.Printf("error stat'ing %s: %s\n", machinePath, err)
		}

		if machineInfo.IsDir() {
			machineStruct := StateMachine{Name: machine.Name(), Path: machinePath}

			states, err := ioutil.ReadDir(machinePath)
			if err != nil {
				fmt.Printf("error listing %s directory: %s\n", machine.Name(), err)
				os.Exit(1)
			}

			hasStart := false
			for _, stateInfo := range states {
				if stateInfo.Mode().Perm() & 0111 > 0 {
					machineStruct.States = append(machineStruct.States, stateInfo.Name())

					if stateInfo.Name() == "start" {
						hasStart = true
					}
				}
			}

			if !hasStart {
				fmt.Printf("state machine directory for %s has no start state\n", machineStruct.Name)
				os.Exit(1)
			}

			cmd := exec.Command(machinePath + "/start", "--help")
			output, err := cmd.CombinedOutput()
			if err != nil {
				fmt.Printf("error executing '%s/start --help' to get usage for %s: %s\n", machinePath, machineStruct.Name, err)
				os.Exit(1)
			} else {
				machineStruct.Usage = string(output)
			}

			globalStateMachines = append(globalStateMachines, machineStruct)
		}
	}
}
<|endoftext|>


<commit_before>package obj

import (
	"context"
	"io"
	"sync"
	"time"

	lru "github.com/hashicorp/golang-lru"
	log "github.com/sirupsen/logrus"
)

const numEvictors = 10

var _ Client = &cacheClient{}

type cacheClient struct {
	slow, fast Client

	cache         *lru.Cache
	pathLocks     sync.Map
	evictionQueue chan string
	populateOnce  sync.Once
}

// NewCacheClient returns slow wrapped in an LRU write-through cache using fast for storing
// cached data.
func NewCacheClient(slow, fast Client, size int) Client {
	if size == 0 {
		return slow
	}
	client := &cacheClient{
		slow:          slow,
		fast:          fast,
		evictionQueue: make(chan string, evictionQueueSize(size)),
	}
	cache, err := lru.NewWithEvict(size, client.onEvicted)
	if err != nil {
		// lru.NewWithEvict only errors for size < 1
		panic(err)
	}
	client.cache = cache
	for i := 0; i < numEvictors; i++ {
		go client.evictor()
	}
	return client
}

func (c *cacheClient) Reader(ctx context.Context, p string, offset, size uint64) (io.ReadCloser, error) {
	c.doPopulateOnce(ctx)
	if offset > 0 || size > 0 {
		return c.slow.Reader(ctx, p, offset, size)
	}
	c.lockPath(p)
	if _, exists := c.cache.Get(p); !exists {
		if err := Copy(ctx, c.slow, c.fast, p, p); err != nil {
			c.unlockPath(p)
			return nil, err
		}
		c.cache.Add(p, struct{}{})
	}
	rc, err := c.fast.Reader(ctx, p, 0, 0)
	if err != nil {
		c.unlockPath(p)
		return nil, err
	}
	// cacheReader.Close() will unlock the path
	return &cacheReader{path: p, ReadCloser: rc, client: c}, nil
}

func (c *cacheClient) Writer(ctx context.Context, p string) (io.WriteCloser, error) {
	return c.slow.Writer(ctx, p)
}

func (c *cacheClient) Delete(ctx context.Context, p string) error {
	c.lockPath(p)
	defer c.unlockPath(p)
	if err := c.fast.Delete(ctx, p); err != nil {
		return err
	}
	if err := c.slow.Delete(ctx, p); err != nil {
		return err
	}
	return nil
}

func (c *cacheClient) Exists(ctx context.Context, p string) bool {
	c.doPopulateOnce(ctx)
	return c.fast.Exists(ctx, p) || c.slow.Exists(ctx, p)
}

func (c *cacheClient) Walk(ctx context.Context, p string, cb func(p string) error) error {
	return c.slow.Walk(ctx, p, cb)
}

func (c *cacheClient) IsIgnorable(err error) bool {
	return c.fast.IsIgnorable(err) || c.slow.IsIgnorable(err)
}

func (c *cacheClient) IsNotExist(err error) bool {
	return c.fast.IsNotExist(err) || c.slow.IsNotExist(err)
}

func (c *cacheClient) IsRetryable(err error) bool {
	return c.fast.IsRetryable(err) || c.slow.IsRetryable(err)
}

func (c *cacheClient) Close() error {
	close(c.evictionQueue)
	return nil
}

func (c *cacheClient) lockPath(p string) {
	v, _ := c.pathLocks.LoadOrStore(p, &sync.Mutex{})
	v.(*sync.Mutex).Lock()
}

func (c *cacheClient) unlockPath(p string) {
	v, exists := c.pathLocks.LoadAndDelete(p)
	if !exists {
		panic("unlock path that was never locked: " + p)
	}
	v.(*sync.Mutex).Unlock()
}

func (c *cacheClient) onEvicted(key, value interface{}) {
	p := key.(string)
	select {
	case c.evictionQueue <- p:
	default:
		// this is preferable to blocking all operations, but after this we are no longer healthy.
		log.Errorf("cache eviction queue is full.  Not deleting %s", p)
	}
}

func (c *cacheClient) evictor() {
	ctx := context.Background()
	for p := range c.evictionQueue {
		ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
		c.lockPath(p)
		if err := c.fast.Delete(ctx, p); err != nil {
			log.Errorf("object cache could not delete from fast layer: %v", err)
		}
		c.unlockPath(p)
		cancel()
	}
}

func (c *cacheClient) populate(ctx context.Context) error {
	return c.fast.Walk(ctx, "", func(p string) error {
		c.cache.Add(p, struct{}{})
		return nil
	})
}

func (c *cacheClient) doPopulateOnce(ctx context.Context) {
	c.populateOnce.Do(func() {
		err := c.populate(ctx)
		log.Warnf("could not populate cache: %v", err)
	})
}

type cacheReader struct {
	io.ReadCloser
	path   string
	client *cacheClient
}

func (cr *cacheReader) Close() error {
	cr.client.unlockPath(cr.path)
	return cr.ReadCloser.Close()
}

func evictionQueueSize(cacheSize int) int {
	y := cacheSize * 1 / 10
	if y == 0 {
		y = 1
	}
	return y
}
<commit_msg>remove eviction queue<commit_after>package obj

import (
	"context"
	"io"
	"sync"
	"time"

	"github.com/hashicorp/golang-lru/simplelru"
	log "github.com/sirupsen/logrus"
)

var _ Client = &cacheClient{}

type cacheClient struct {
	slow, fast Client

	mu           sync.Mutex
	cache        *simplelru.LRU
	pathLocks    sync.Map
	populateOnce sync.Once
}

// NewCacheClient returns slow wrapped in an LRU write-through cache using fast for storing
// cached data.
func NewCacheClient(slow, fast Client, size int) Client {
	if size == 0 {
		return slow
	}
	client := &cacheClient{
		slow: slow,
		fast: fast,
	}
	cache, err := simplelru.NewLRU(size, client.onEvicted)
	if err != nil {
		// lru.NewWithEvict only errors for size < 1
		panic(err)
	}
	client.cache = cache
	return client
}

func (c *cacheClient) Reader(ctx context.Context, p string, offset, size uint64) (io.ReadCloser, error) {
	c.doPopulateOnce(ctx)
	if offset > 0 || size > 0 {
		return c.slow.Reader(ctx, p, offset, size)
	}
	c.mu.Lock()
	if _, exists := c.cache.Get(p); !exists {
		c.lockPath(p)
		c.mu.Unlock()
		if err := Copy(ctx, c.slow, c.fast, p, p); err != nil {
			c.unlockPath(p)
			log.Errorf("untracked object (%s) in cache fast layer. error: %v", p, err)
			return nil, err
		}
		c.mu.Lock()
		c.cache.Add(p, struct{}{})
		c.unlockPath(p)
		c.mu.Unlock()
	} else {
		c.mu.Unlock()
	}

	c.rlockPath(p)
	rc, err := c.fast.Reader(ctx, p, 0, 0)
	if err != nil {
		c.runlockPath(p)
		// at this point it should be in the fast cache. If it is not, return a transient error
		if c.fast.IsNotExist(err) {
			return nil, ErrCacheRace{}
		}
		return nil, err
	}
	// cacheReader.Close() will unlock the path
	return &cacheReader{path: p, ReadCloser: rc, client: c}, nil
}

func (c *cacheClient) Writer(ctx context.Context, p string) (io.WriteCloser, error) {
	return c.slow.Writer(ctx, p)
}

func (c *cacheClient) Delete(ctx context.Context, p string) error {
	c.mu.Lock()
	defer c.mu.Unlock()
	c.lockPath(p)
	defer c.unlockPath(p)
	c.cache.Remove(p)
	if err := c.fast.Delete(ctx, p); err != nil {
		log.Errorf("error deleting object, object is no longer indexed by cache")
		return err
	}
	if err := c.slow.Delete(ctx, p); err != nil {
		return err
	}
	return nil
}

func (c *cacheClient) Exists(ctx context.Context, p string) bool {
	c.doPopulateOnce(ctx)
	return c.fast.Exists(ctx, p) || c.slow.Exists(ctx, p)
}

func (c *cacheClient) Walk(ctx context.Context, p string, cb func(p string) error) error {
	return c.slow.Walk(ctx, p, cb)
}

func (c *cacheClient) IsIgnorable(err error) bool {
	return c.fast.IsIgnorable(err) || c.slow.IsIgnorable(err)
}

func (c *cacheClient) IsNotExist(err error) bool {
	return c.fast.IsNotExist(err) || c.slow.IsNotExist(err)
}

func (c *cacheClient) IsRetryable(err error) bool {
	switch err := err.(type) {
	case ErrCacheRace:
		return true
	default:
		return c.fast.IsRetryable(err) || c.slow.IsRetryable(err)
	}
}

func (c *cacheClient) lockPath(p string) {
	v, _ := c.pathLocks.LoadOrStore(p, &sync.Mutex{})
	v.(*sync.RWMutex).Lock()
}

func (c *cacheClient) rlockPath(p string) {
	v, _ := c.pathLocks.LoadOrStore(p, &sync.Mutex{})
	v.(*sync.RWMutex).RLock()
}

func (c *cacheClient) unlockPath(p string) {
	v, exists := c.pathLocks.Load(p)
	if !exists {
		panic("unlock path that was never locked: " + p)
	}
	v.(*sync.RWMutex).Unlock()
}

func (c *cacheClient) runlockPath(p string) {
	v, exists := c.pathLocks.Load(p)
	if !exists {
		panic("unlock path that was never locked: " + p)
	}
	v.(*sync.RWMutex).RUnlock()
}

func (c *cacheClient) onEvicted(key, value interface{}) {
	p := key.(string)
	c.lockPath(p)
	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
	if err := c.fast.Delete(ctx, p); err != nil {
		log.Error("could not delete from cache's fast store")
	}
	cancel()
	c.unlockPath(p)
}

func (c *cacheClient) populate(ctx context.Context) error {
	return c.fast.Walk(ctx, "", func(p string) error {
		c.cache.Add(p, struct{}{})
		return nil
	})
}

func (c *cacheClient) doPopulateOnce(ctx context.Context) {
	c.populateOnce.Do(func() {
		err := c.populate(ctx)
		log.Warnf("could not populate cache: %v", err)
	})
}

type cacheReader struct {
	io.ReadCloser
	path   string
	client *cacheClient
}

func (cr *cacheReader) Close() error {
	cr.client.unlockPath(cr.path)
	return cr.ReadCloser.Close()
}

type ErrCacheRace struct {
}

func (e ErrCacheRace) Error() string {
	return "cache race. retry"
}
<|endoftext|>


<commit_before>//go:build mage
// +build mage

package main

import (
	"github.com/gilcrest/go-api-basic/commands"
	"github.com/magefile/mage/sh"
)

// DBUp uses the psql command line interface to execute DDL scripts
// in the ./scripts/ddl/deploy/up directory and create all required
// DB objects. All files will be executed, regardless of errors within
// an individual file. Check output to determine if any errors occurred.
// Eventually, I will write this to stop on errors, but for now it is
// what it is.
func DBUp() error {
	args, err := commands.PSQLArgs(true)
	if err != nil {
		return err
	}

	err = sh.Run("psql", args...)
	if err != nil {
		return err
	}
	return nil
}

// DBDown uses the psql command line interface to execute DDL scripts
// in the ./scripts/ddl/deploy/down directory and drops all project-specific
// DB objects. All files will be executed, regardless of errors within
// an individual file. Check output to determine if any errors occurred.
// Eventually, I will write this to stop on errors, but for now it is
// what it is.
func DBDown() error {
	args, err := commands.PSQLArgs(false)
	if err != nil {
		return err
	}

	err = sh.Run("psql", args...)
	if err != nil {
		return err
	}
	return nil
}
<commit_msg>add environment override before db deployment<commit_after>//go:build mage
// +build mage

package main

import (
	"github.com/gilcrest/go-api-basic/commands"
	"github.com/magefile/mage/sh"
)

// DBUp uses the psql command line interface to execute DDL scripts
// in the ./scripts/ddl/db-deploy/up directory and create all required
// DB objects. All files will be executed, regardless of errors within
// an individual file. Check output to determine if any errors occurred.
// Eventually, I will write this to stop on errors, but for now it is
// what it is.
func DBUp() error {
	var (
		err  error
		args []string
	)

	err = commands.OverrideEnv()
	if err != nil {
		return err
	}

	args, err = commands.PSQLArgs(true)
	if err != nil {
		return err
	}

	err = sh.Run("psql", args...)
	if err != nil {
		return err
	}

	return nil
}

// DBDown uses the psql command line interface to execute DDL scripts
// in the ./scripts/ddl/db-deploy/down directory and drops all project-specific
// DB objects. All files will be executed, regardless of errors within
// an individual file. Check output to determine if any errors occurred.
// Eventually, I will write this to stop on errors, but for now it is
// what it is.
func DBDown() error {
	var (
		err  error
		args []string
	)

	err = commands.OverrideEnv()
	if err != nil {
		return err
	}

	args, err = commands.PSQLArgs(false)
	if err != nil {
		return err
	}

	err = sh.Run("psql", args...)
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package dokugen

type RankedObject interface {
	Rank() int
}

type FiniteQueue struct {
	min     int
	max     int
	objects [][]RankedObject
}

//Returns a new queue that will work for items with a rank as low as min or as high as max (inclusive)
func NewFiniteQueue(min int, max int) *FiniteQueue {
	return &FiniteQueue{min, max, make([][]RankedObject, max-min+1)}
}
<commit_msg>Implemented Insert on FiniteList. No new tests yet.<commit_after>package dokugen

type RankedObject interface {
	Rank() int
}

type FiniteQueue struct {
	min     int
	max     int
	objects [][]RankedObject
}

//Returns a new queue that will work for items with a rank as low as min or as high as max (inclusive)
func NewFiniteQueue(min int, max int) *FiniteQueue {
	return &FiniteQueue{min, max, make([][]RankedObject, max-min+1)}
}

func (self *FiniteQueue) Insert(obj RankedObject) {
	rank := obj.Rank()
	list := self.getList(rank)
	if list == nil {
		//Apparently rank wasn't legal.
		return
	}
	//Scrub the list for this item.
	for _, item := range list {
		if item == obj {
			//It's already there, just return.
			return
		}
	}
	//If we get to here we need to add the item.
	self.setList(rank, append(list, obj))
}

func (self *FiniteQueue) legalRank(rank int) bool {
	return rank >= self.min && rank <= self.max
}

func (self *FiniteQueue) getList(rank int) []RankedObject {
	if !self.legalRank(rank) {
		return nil
	}
	return self.objects[rank-self.min]
}

func (self *FiniteQueue) setList(rank int, list []RankedObject) {
	if !self.legalRank(rank) {
		return
	}
	self.objects[rank-self.min] = list
}
<|endoftext|>


<commit_before>package gbench

import (
	"context"
	"fmt"
	"math/rand"
	"strings"
	"time"

	rpc "github.com/relab/gorums/dev"
	"github.com/tylertreat/bench"

	"google.golang.org/grpc"
)

// RequesterFactory implements RequesterFactory by creating a Requester which
// issues requests to a register using the gRPC framework.
type GrpcRequesterFactory struct {
	Addr              string
	PayloadSize       int
	Timeout           time.Duration
	WriteRatioPercent int
}

// GetRequester returns a new Requester, called for each Benchmark connection.
func (r *GrpcRequesterFactory) GetRequester(uint64) bench.Requester {
	return &grpcRequester{
		addr: r.Addr,
		dialOpts: []grpc.DialOption{
			grpc.WithInsecure(),
			grpc.WithBlock(),
			grpc.WithTimeout(time.Second),
		},
		payloadSize: r.PayloadSize,
		timeout:     r.Timeout,
		writeRatio:  r.WriteRatioPercent,
	}
}

type grpcRequester struct {
	addr        string
	dialOpts    []grpc.DialOption
	payloadSize int
	timeout     time.Duration
	writeRatio  int

	conn   *grpc.ClientConn
	client rpc.RegisterClient
	state  *rpc.State
	ctx    context.Context
}

func (gr *grpcRequester) Setup() error {
	var err error
	gr.conn, err = grpc.Dial(gr.addr, gr.dialOpts...)
	if err != nil {
		return err
	}

	gr.ctx = context.Background()
	gr.client = rpc.NewRegisterClient(gr.conn)

	// Set initial state.
	gr.state = &rpc.State{
		Value:     strings.Repeat("x", gr.payloadSize),
		Timestamp: time.Now().UnixNano(),
	}

	wreply, err := gr.client.Write(gr.ctx, gr.state)
	if err != nil {
		return fmt.Errorf("write rpc error: %v", err)
	}
	if !wreply.New {
		return fmt.Errorf("intital write reply was not marked as new")
	}

	return nil
}

func (gr *grpcRequester) Request() error {
	var err error
	switch gr.writeRatio {
	case 0:
		_, err = gr.client.Read(gr.ctx, &rpc.ReadRequest{})
	case 100:
		gr.state.Timestamp = time.Now().UnixNano()
		_, err = gr.client.Write(gr.ctx, gr.state)
	default:
		x := rand.Intn(100)
		if x < gr.writeRatio {
			gr.state.Timestamp = time.Now().UnixNano()
			_, err = gr.client.Write(gr.ctx, gr.state)
		} else {
			_, err = gr.client.Read(gr.ctx, &rpc.ReadRequest{})
		}
	}

	return err
}

func (gr *grpcRequester) Teardown() error {
	return gr.conn.Close()
}
<commit_msg>gbench: grpc teardown<commit_after>package gbench

import (
	"context"
	"fmt"
	"math/rand"
	"strings"
	"time"

	rpc "github.com/relab/gorums/dev"
	"github.com/tylertreat/bench"

	"google.golang.org/grpc"
)

// RequesterFactory implements RequesterFactory by creating a Requester which
// issues requests to a register using the gRPC framework.
type GrpcRequesterFactory struct {
	Addr              string
	PayloadSize       int
	Timeout           time.Duration
	WriteRatioPercent int
}

// GetRequester returns a new Requester, called for each Benchmark connection.
func (r *GrpcRequesterFactory) GetRequester(uint64) bench.Requester {
	return &grpcRequester{
		addr: r.Addr,
		dialOpts: []grpc.DialOption{
			grpc.WithInsecure(),
			grpc.WithBlock(),
			grpc.WithTimeout(time.Second),
		},
		payloadSize: r.PayloadSize,
		timeout:     r.Timeout,
		writeRatio:  r.WriteRatioPercent,
	}
}

type grpcRequester struct {
	addr        string
	dialOpts    []grpc.DialOption
	payloadSize int
	timeout     time.Duration
	writeRatio  int

	conn   *grpc.ClientConn
	client rpc.RegisterClient
	state  *rpc.State
	ctx    context.Context
}

func (gr *grpcRequester) Setup() error {
	var err error
	gr.conn, err = grpc.Dial(gr.addr, gr.dialOpts...)
	if err != nil {
		return err
	}

	gr.ctx = context.Background()
	gr.client = rpc.NewRegisterClient(gr.conn)

	// Set initial state.
	gr.state = &rpc.State{
		Value:     strings.Repeat("x", gr.payloadSize),
		Timestamp: time.Now().UnixNano(),
	}

	wreply, err := gr.client.Write(gr.ctx, gr.state)
	if err != nil {
		return fmt.Errorf("write rpc error: %v", err)
	}
	if !wreply.New {
		return fmt.Errorf("intital write reply was not marked as new")
	}

	return nil
}

func (gr *grpcRequester) Request() error {
	var err error
	switch gr.writeRatio {
	case 0:
		_, err = gr.client.Read(gr.ctx, &rpc.ReadRequest{})
	case 100:
		gr.state.Timestamp = time.Now().UnixNano()
		_, err = gr.client.Write(gr.ctx, gr.state)
	default:
		x := rand.Intn(100)
		if x < gr.writeRatio {
			gr.state.Timestamp = time.Now().UnixNano()
			_, err = gr.client.Write(gr.ctx, gr.state)
		} else {
			_, err = gr.client.Read(gr.ctx, &rpc.ReadRequest{})
		}
	}

	return err
}

func (gr *grpcRequester) Teardown() error {
	gr.conn.Close()
	gr.conn = nil
	return nil
}
<|endoftext|>


<commit_before>package csql

import (
	"database/sql"
	"fmt"
	"strings"
)

type PrepExecer interface {
	Exec(query string, args ...interface{}) (sql.Result, error)
	Prepare(query string) (*sql.Stmt, error)
}

type Inserter struct {
	db            PrepExecer
	current, size int
	table         string
	columns       []string
	data          []interface{}
}

func NewInserter(
	db PrepExecer,
	size int,
	table string,
	columns ...string) *Inserter {
	if size < 1 {
		size = 1
	}
	return &Inserter{db, 0, size, table, columns, make([]interface{}, 0, 100)}
}

func (in *Inserter) Add(args ...interface{}) error {
	if len(args) != len(in.columns) {
		panic("len(args) != len(columns)")
	}
	in.data = append(in.data, args...)
	in.current++
	if in.current == in.size {
		return in.Exec()
	}
	return nil
}

func (in *Inserter) Exec() error {
	err := Safe(func() {
		q := in.prepare()
		if len(q) == 0 {
			return // nothing to insert! success!
		}
		Exec(in.db, q, in.data...)
	})
	in.current = 0
	in.data = in.data[:0]
	return err
}

func (in *Inserter) prepare() string {
	if in.current == 0 {
		return ""
	}

	makePlaceholders := func(start int) string {
		oneInsert := make([]string, len(in.columns))
		for i := range oneInsert {
			oneInsert[i] = fmt.Sprintf("$%d", start+i)
		}
		return fmt.Sprintf("(%s)", strings.Join(oneInsert, ", "))
	}

	rows := make([]string, in.current)
	for i := range rows {
		rows[i] = makePlaceholders(1 + i * len(in.columns))
	}
	return fmt.Sprintf("INSERT INTO %s (%s) VALUES %s",
		in.table, strings.Join(in.columns, ", "), strings.Join(rows, ", "))
}
<commit_msg>Simplify the Inserter API.<commit_after>package csql

import (
	"database/sql"
	"fmt"
	"strings"

	"github.com/lib/pq"
)

// TODO: Modify this so it uses COPY IN from github.com/lib/pq when it's
// available.
//
// But how do we know if the pq package is being used? Hmm...

type Inserter struct {
	tx            *sql.Tx
	current, size int
	table         string
	columns       []string

	stmt *sql.Stmt
	data []interface{}
}

func NewInserter(
	tx *sql.Tx,
	driver string,
	size int,
	table string,
	columns ...string,
) (*Inserter, error) {
	if size < 1 {
		size = 1
	}
	var ins *Inserter
	err := Safe(func() {
		ins = &Inserter{tx, 0, size, table, columns, nil, nil}
		if driver == "postgres" {
			ins.stmt = Prepare(tx, pq.CopyIn(table, columns...))
		} else {
			ins.data = make([]interface{}, 0, 100)
		}
	})
	return ins, err
}

func (in *Inserter) Exec(args ...interface{}) error {
	if in.stmt != nil {
		_, err := in.stmt.Exec(args...)
		if err != nil {
			return err
		}
		if len(args) == 0 {
			return in.stmt.Close()
		}
		return nil
	}
	if len(args) == 0 {
		return in.run()
	}
	if len(args) != len(in.columns) {
		panic("len(args) != len(columns)")
	}
	in.data = append(in.data, args...)
	in.current++
	if in.current == in.size {
		return in.run()
	}
	return nil
}

func (in *Inserter) run() error {
	err := Safe(func() {
		q := in.prepare()
		if len(q) == 0 {
			return // nothing to insert! success!
		}
		Exec(in.tx, q, in.data...)
	})
	in.current = 0
	in.data = in.data[:0]
	return err
}

func (in *Inserter) prepare() string {
	if in.current == 0 {
		return ""
	}

	makePlaceholders := func(start int) string {
		oneInsert := make([]string, len(in.columns))
		for i := range oneInsert {
			oneInsert[i] = fmt.Sprintf("$%d", start+i)
		}
		return fmt.Sprintf("(%s)", strings.Join(oneInsert, ", "))
	}

	rows := make([]string, in.current)
	for i := range rows {
		rows[i] = makePlaceholders(1 + i*len(in.columns))
	}
	return fmt.Sprintf("INSERT INTO %s (%s) VALUES %s",
		in.table, strings.Join(in.columns, ", "), strings.Join(rows, ", "))
}
<|endoftext|>


<commit_before>package iqfeed

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"net"
	"os"
	"strings"
	"time"
)

// IQC provides the main struct for the the IQ Client interface into what IQFeed will be sending us.
type IQC struct {
	System       chan *SystemMessage
	News         chan *NewsMsg
	Errors       chan *ErrorMsg
	Fundamental  chan *FundamentalMsg
	Regional     chan *RegionalMsg
	Time         chan *TimeMsg
	Updates      chan *UpdSummaryMsg
	CreateBackup bool
	BackupFile   string
	Conn         net.Conn
	Quit         chan bool
	DynFields    map[int]string
}

func (c *IQC) connect(cs string) {
	c.DynFields = make(map[int]string)
	if cs == "" {
		cs = "localhost:5009"
	}
	conn, err := net.Dial("tcp", cs)
	if err != nil {
		log.Fatal("Could not connect to IQFeed")
	}
	c.Conn = conn
}

// ProcessSysMsg handles system messages, field definitions are available here: http://www.iqfeed.net/dev/api/docs/Level1SystemMessage.cfm.
func (c *IQC) processSysMsg(d []byte) {
	s := &SystemMessage{}

	pfx := strings.Split(string(d), ",")
	switch pfx[0] {
	case "CURRENT UPDATE FIELDNAMES":
		/* We use a map here to preserve the actual order as it's important with marshalling dynamic fields */
		for i := 1; i < len(pfx); i++ {
			c.DynFields[i-1] = pfx[i]
		}
	default:
		s.UnMarshall(d)
		c.System <- s
	}
}

// ProcessSumMsg handles summary messages, field definitions are available here: http://www.iqfeed.net/dev/api/docs/Level1UpdateSummaryMessage.cfm.
func (c *IQC) processSumMsg(d []byte) {
	s := &UpdSummaryMsg{}
	items := strings.Split(string(d), ",")
	s.UnMarshall(items, c.DynFields)
	c.Updates <- s
}

// ProcessUpdMsg handles update messages, field definitions are available here: http://www.iqfeed.net/dev/api/docs/Level1UpdateSummaryMessage.cfm.
func (c *IQC) processUpdMsg(d []byte) {
	u := &UpdSummaryMsg{}
	items := strings.Split(string(d), ",")
	if items[2] == "Not Found" {
		c.process404Msg([]byte(items[0]))
		return
	}
	u.UnMarshall(items, c.DynFields)
	c.Updates <- u
}

// ProcessTimeMsg handles timestamp updates, field definitions are available here: http://www.iqfeed.net/dev/api/docs/TimeMessageFormat.cfm.
func (c *IQC) processTimeMsg(d []byte) {
	t := &TimeMsg{}
	t.UnMarshall(d)

	c.Time <- t
}

// ProcessRegUpdMsg handles regional updates field definitions are available here: http://www.iqfeed.net/dev/api/docs/RegionalMessageFormat.cfm.
func (c *IQC) processRegUpdMsg(d []byte) {
	r := &RegionalMsg{}
	r.UnMarshall(d)
	c.Regional <- r
}

// ProcessFndMsg handles fundamental messages, field descriptions are available here: http://www.iqfeed.net/dev/api/docs/Level1FundamentalMessage.cfm.
func (c *IQC) processFndMsg(d []byte) {
	f := &FundamentalMsg{}
	f.UnMarshall(d)
	c.Fundamental <- f

}

// ProcessNewsMsg handles summary messages, field definitions are available here: http://www.iqfeed.net/dev/api/docs/StreamingNewsMessageFormat.cfm.
func (c *IQC) processNewsMsg(d []byte) {
	n := &NewsMsg{}
	n.UnMarshall(d)
	c.News <- n
}

// Process404Msg handles messages indicating that a symbol was not found.
func (c *IQC) process404Msg(d []byte) {
	e := &ErrorMsg{}
	e.UnMarshall(true, d, 404)
	c.Errors <- e
}

// ProcessErrorMsg handles error messages in the form of error text.
func (c *IQC) processErrorMsg(d []byte) {
	e := &ErrorMsg{}
	e.UnMarshall(false, d, 500)
	c.Errors <- e
}

// ProcessReceiver is one of the main reciever functions that interprets data received by IQFeed and processes it in sub functions.
func (c *IQC) processReceiver(d []byte) {
	data := d[2:]
	switch d[0] {
	case 0x53: // Start letter is S, indicating System message (Unicode representation in integer value).
		c.processSysMsg(data)
	case 0x50: // Start letter is P, indicating a summary message.
		c.processSumMsg(data)
	case 0x51: // Start letter is Q, indicating an update message.
		c.processUpdMsg(data)
	case 0x54: // Start letter is T, indicating Time message.
		c.processTimeMsg(data)
	case 0x52: // Start letter is R, indicating regional update message
		c.processRegUpdMsg(data)
	case 0x46: // Start letter is F, indicating a fundamental message
		c.processFndMsg(data)
	case 0x4e: // Start letter is N, indicating a news message
		c.processNewsMsg(data)
	case 0x6E: // Start letter is n, indicating Symbol not found message
		c.process404Msg(data)
	case 0x45: // Start letter is E, error message
		c.processErrorMsg(data)
	}

}

// Read function does as expected and reads data from the network stream.
func (c *IQC) read() {
	r := bufio.NewReader(c.Conn)
	for {
		select {
		case <-c.Quit:
			log.Println("Client quitting")
			c.Conn.Close()
			break
		default:
			line, isPrefix, err := r.ReadLine()
			for err == nil && !isPrefix {
				if c.CreateBackup {
					bld := fmt.Sprintf("%s\r\n", string(line))
					c.writeBackup([]byte(bld))
				}
				c.processReceiver(line)
				line, isPrefix, err = r.ReadLine()
			}
			if isPrefix {
				log.Println("buffer size to small")
				//return Do not return and break the loop
			}
			if err != io.EOF {
				log.Println("Pipe closed exiting...")
				c.Conn.Close()
				os.Exit(0)
			}
		}
	}

}

func (c *IQC) getCallChar(t time.Time) string {
	switch t.Month().String() {
	case "January":
		return "A"
	case "February":
		return "B"
	case "March":
		return "C"
	case "April":
		return "D"
	case "May":
		return "E"
	case "June":
		return "F"
	case "July":
		return "G"
	case "August":
		return "H"
	case "September":
		return "I"
	case "October":
		return "J"
	case "November":
		return "K"
	case "December":
		return "L"
	}
	return "A"
}

func (c *IQC) getPutChar(t time.Time) string {
	switch t.Month().String() {
	case "January":
		return "M"
	case "February":
		return "N"
	case "March":
		return "O"
	case "April":
		return "P"
	case "May":
		return "Q"
	case "June":
		return "R"
	case "July":
		return "S"
	case "August":
		return "T"
	case "September":
		return "U"
	case "October":
		return "V"
	case "November":
		return "W"
	case "December":
		return "X"
	}
	return "M"
}

// Start function will start the concurrent functions to read and write data to the and from the network stream.
func (c *IQC) Start(connectString string) *IQC {
	c.connect(connectString)
	c.System = make(chan *SystemMessage)
	c.News = make(chan *NewsMsg)
	c.Errors = make(chan *ErrorMsg)
	c.Fundamental = make(chan *FundamentalMsg)
	c.Regional = make(chan *RegionalMsg)
	c.Time = make(chan *TimeMsg)
	c.Updates = make(chan *UpdSummaryMsg)
	go c.read()
	return c

}
<commit_msg>Incorrect string validation for calculating field namen order.<commit_after>package iqfeed

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"net"
	"os"
	"strings"
	"time"
)

// IQC provides the main struct for the the IQ Client interface into what IQFeed will be sending us.
type IQC struct {
	System       chan *SystemMessage
	News         chan *NewsMsg
	Errors       chan *ErrorMsg
	Fundamental  chan *FundamentalMsg
	Regional     chan *RegionalMsg
	Time         chan *TimeMsg
	Updates      chan *UpdSummaryMsg
	CreateBackup bool
	BackupFile   string
	Conn         net.Conn
	Quit         chan bool
	DynFields    map[int]string
}

func (c *IQC) connect(cs string) {
	c.DynFields = make(map[int]string)
	if cs == "" {
		cs = "localhost:5009"
	}
	conn, err := net.Dial("tcp", cs)
	if err != nil {
		log.Fatal("Could not connect to IQFeed")
	}
	c.Conn = conn
}

// ProcessSysMsg handles system messages, field definitions are available here: http://www.iqfeed.net/dev/api/docs/Level1SystemMessage.cfm.
func (c *IQC) processSysMsg(d []byte) {
	s := &SystemMessage{}

	pfx := strings.Split(string(d), ",")
	switch pfx[0] {
	case "UPDATE FIELDNAMES":
		/* We use a map here to preserve the actual order as it's important with marshalling dynamic fields */
		for i := 1; i < len(pfx); i++ {
			c.DynFields[i-1] = pfx[i]
		}
	default:
		s.UnMarshall(d)
		c.System <- s
	}
}

// ProcessSumMsg handles summary messages, field definitions are available here: http://www.iqfeed.net/dev/api/docs/Level1UpdateSummaryMessage.cfm.
func (c *IQC) processSumMsg(d []byte) {
	s := &UpdSummaryMsg{}
	items := strings.Split(string(d), ",")
	s.UnMarshall(items, c.DynFields)
	c.Updates <- s
}

// ProcessUpdMsg handles update messages, field definitions are available here: http://www.iqfeed.net/dev/api/docs/Level1UpdateSummaryMessage.cfm.
func (c *IQC) processUpdMsg(d []byte) {
	u := &UpdSummaryMsg{}
	items := strings.Split(string(d), ",")
	if items[2] == "Not Found" {
		c.process404Msg([]byte(items[0]))
		return
	}
	u.UnMarshall(items, c.DynFields)
	c.Updates <- u
}

// ProcessTimeMsg handles timestamp updates, field definitions are available here: http://www.iqfeed.net/dev/api/docs/TimeMessageFormat.cfm.
func (c *IQC) processTimeMsg(d []byte) {
	t := &TimeMsg{}
	t.UnMarshall(d)

	c.Time <- t
}

// ProcessRegUpdMsg handles regional updates field definitions are available here: http://www.iqfeed.net/dev/api/docs/RegionalMessageFormat.cfm.
func (c *IQC) processRegUpdMsg(d []byte) {
	r := &RegionalMsg{}
	r.UnMarshall(d)
	c.Regional <- r
}

// ProcessFndMsg handles fundamental messages, field descriptions are available here: http://www.iqfeed.net/dev/api/docs/Level1FundamentalMessage.cfm.
func (c *IQC) processFndMsg(d []byte) {
	f := &FundamentalMsg{}
	f.UnMarshall(d)
	c.Fundamental <- f

}

// ProcessNewsMsg handles summary messages, field definitions are available here: http://www.iqfeed.net/dev/api/docs/StreamingNewsMessageFormat.cfm.
func (c *IQC) processNewsMsg(d []byte) {
	n := &NewsMsg{}
	n.UnMarshall(d)
	c.News <- n
}

// Process404Msg handles messages indicating that a symbol was not found.
func (c *IQC) process404Msg(d []byte) {
	e := &ErrorMsg{}
	e.UnMarshall(true, d, 404)
	c.Errors <- e
}

// ProcessErrorMsg handles error messages in the form of error text.
func (c *IQC) processErrorMsg(d []byte) {
	e := &ErrorMsg{}
	e.UnMarshall(false, d, 500)
	c.Errors <- e
}

// ProcessReceiver is one of the main reciever functions that interprets data received by IQFeed and processes it in sub functions.
func (c *IQC) processReceiver(d []byte) {
	data := d[2:]
	switch d[0] {
	case 0x53: // Start letter is S, indicating System message (Unicode representation in integer value).
		c.processSysMsg(data)
	case 0x50: // Start letter is P, indicating a summary message.
		c.processSumMsg(data)
	case 0x51: // Start letter is Q, indicating an update message.
		c.processUpdMsg(data)
	case 0x54: // Start letter is T, indicating Time message.
		c.processTimeMsg(data)
	case 0x52: // Start letter is R, indicating regional update message
		c.processRegUpdMsg(data)
	case 0x46: // Start letter is F, indicating a fundamental message
		c.processFndMsg(data)
	case 0x4e: // Start letter is N, indicating a news message
		c.processNewsMsg(data)
	case 0x6E: // Start letter is n, indicating Symbol not found message
		c.process404Msg(data)
	case 0x45: // Start letter is E, error message
		c.processErrorMsg(data)
	}

}

// Read function does as expected and reads data from the network stream.
func (c *IQC) read() {
	r := bufio.NewReader(c.Conn)
	for {
		select {
		case <-c.Quit:
			log.Println("Client quitting")
			c.Conn.Close()
			break
		default:
			line, isPrefix, err := r.ReadLine()
			for err == nil && !isPrefix {
				if c.CreateBackup {
					bld := fmt.Sprintf("%s\r\n", string(line))
					c.writeBackup([]byte(bld))
				}
				c.processReceiver(line)
				line, isPrefix, err = r.ReadLine()
			}
			if isPrefix {
				log.Println("buffer size to small")
				//return Do not return and break the loop
			}
			if err != io.EOF {
				log.Println("Pipe closed exiting...")
				c.Conn.Close()
				os.Exit(0)
			}
		}
	}

}

func (c *IQC) getCallChar(t time.Time) string {
	switch t.Month().String() {
	case "January":
		return "A"
	case "February":
		return "B"
	case "March":
		return "C"
	case "April":
		return "D"
	case "May":
		return "E"
	case "June":
		return "F"
	case "July":
		return "G"
	case "August":
		return "H"
	case "September":
		return "I"
	case "October":
		return "J"
	case "November":
		return "K"
	case "December":
		return "L"
	}
	return "A"
}

func (c *IQC) getPutChar(t time.Time) string {
	switch t.Month().String() {
	case "January":
		return "M"
	case "February":
		return "N"
	case "March":
		return "O"
	case "April":
		return "P"
	case "May":
		return "Q"
	case "June":
		return "R"
	case "July":
		return "S"
	case "August":
		return "T"
	case "September":
		return "U"
	case "October":
		return "V"
	case "November":
		return "W"
	case "December":
		return "X"
	}
	return "M"
}

// Start function will start the concurrent functions to read and write data to the and from the network stream.
func (c *IQC) Start(connectString string) *IQC {
	c.connect(connectString)
	c.System = make(chan *SystemMessage)
	c.News = make(chan *NewsMsg)
	c.Errors = make(chan *ErrorMsg)
	c.Fundamental = make(chan *FundamentalMsg)
	c.Regional = make(chan *RegionalMsg)
	c.Time = make(chan *TimeMsg)
	c.Updates = make(chan *UpdSummaryMsg)
	go c.read()
	return c

}
<|endoftext|>


<commit_before>package stripe

import (
	"encoding/json"
)

// BitcoinReceiverListParams is the set of parameters that can be used when listing BitcoinReceivers.
// For more details see https://stripe.com/docs/api/#list_bitcoin_receivers.
type BitcoinReceiverListParams struct {
	ListParams `form:"*"`
	NotActive  bool `form:"active,invert"`
	NotFilled  bool `form:"filled,invert"`
	Uncaptured bool `form:"uncaptured_funds"`
}

// BitcoinReceiverParams is the set of parameters that can be used when creating a BitcoinReceiver.
// For more details see https://stripe.com/docs/api/#create_bitcoin_receiver.
type BitcoinReceiverParams struct {
	Params   `form:"*"`
	Amount   uint64   `form:"amount"`
	Currency Currency `form:"currency"`
	Desc     string   `form:"description"`
	Email    string   `form:"email"`
}

// BitcoinReceiverUpdateParams is the set of parameters that can be used when
// updating a BitcoinReceiver. For more details see
// https://stripe.com/docs/api/#update_bitcoin_receiver.
type BitcoinReceiverUpdateParams struct {
	Params     `form:"*"`
	Desc       string `form:"description"`
	Email      string `form:"email"`
	RefundAddr string `form:"refund_address"`
}

// BitcoinReceiver is the resource representing a Stripe bitcoin receiver.
// For more details see https://stripe.com/docs/api/#bitcoin_receivers
type BitcoinReceiver struct {
	Active                bool                    `json:"active"`
	Amount                uint64                  `json:"amount"`
	AmountReceived        uint64                  `json:"amount_received"`
	BitcoinAmount         uint64                  `json:"bitcoin_amount"`
	BitcoinAmountReceived uint64                  `json:"bitcoin_amount_received"`
	BitcoinUri            string                  `json:"bitcoin_uri"`
	Created               int64                   `json:"created"`
	Currency              Currency                `json:"currency"`
	Customer              string                  `json:"customer"`
	Desc                  string                  `json:"description"`
	Email                 string                  `json:"email"`
	Filled                bool                    `json:"filled"`
	ID                    string                  `json:"id"`
	InboundAddress        string                  `json:"inbound_address"`
	Meta                  map[string]string       `json:"metadata"`
	Payment               string                  `json:"payment"`
	RejectTransactions    bool                    `json:"reject_transactions"`
	RefundAddress         string                  `json:"refund_address"`
	Transactions          *BitcoinTransactionList `json:"transactions"`
}

// BitcoinReceiverList is a list of bitcoin receivers as retrieved from a list endpoint.
type BitcoinReceiverList struct {
	ListMeta
	Values []*BitcoinReceiver `json:"data"`
}

// UnmarshalJSON handles deserialization of a BitcoinReceiver.
// This custom unmarshaling is needed because the resulting
// property may be an id or the full struct if it was expanded.
func (br *BitcoinReceiver) UnmarshalJSON(data []byte) error {
	type bitcoinReceiver BitcoinReceiver
	var r bitcoinReceiver
	err := json.Unmarshal(data, &r)
	if err == nil {
		*br = BitcoinReceiver(r)
	} else {
		// the id is surrounded by "\" characters, so strip them
		br.ID = string(data[1 : len(data)-1])
	}

	return nil
}
<commit_msg>Properly sort RejectTransactions/RefundAddress<commit_after>package stripe

import (
	"encoding/json"
)

// BitcoinReceiverListParams is the set of parameters that can be used when listing BitcoinReceivers.
// For more details see https://stripe.com/docs/api/#list_bitcoin_receivers.
type BitcoinReceiverListParams struct {
	ListParams `form:"*"`
	NotActive  bool `form:"active,invert"`
	NotFilled  bool `form:"filled,invert"`
	Uncaptured bool `form:"uncaptured_funds"`
}

// BitcoinReceiverParams is the set of parameters that can be used when creating a BitcoinReceiver.
// For more details see https://stripe.com/docs/api/#create_bitcoin_receiver.
type BitcoinReceiverParams struct {
	Params   `form:"*"`
	Amount   uint64   `form:"amount"`
	Currency Currency `form:"currency"`
	Desc     string   `form:"description"`
	Email    string   `form:"email"`
}

// BitcoinReceiverUpdateParams is the set of parameters that can be used when
// updating a BitcoinReceiver. For more details see
// https://stripe.com/docs/api/#update_bitcoin_receiver.
type BitcoinReceiverUpdateParams struct {
	Params     `form:"*"`
	Desc       string `form:"description"`
	Email      string `form:"email"`
	RefundAddr string `form:"refund_address"`
}

// BitcoinReceiver is the resource representing a Stripe bitcoin receiver.
// For more details see https://stripe.com/docs/api/#bitcoin_receivers
type BitcoinReceiver struct {
	Active                bool                    `json:"active"`
	Amount                uint64                  `json:"amount"`
	AmountReceived        uint64                  `json:"amount_received"`
	BitcoinAmount         uint64                  `json:"bitcoin_amount"`
	BitcoinAmountReceived uint64                  `json:"bitcoin_amount_received"`
	BitcoinUri            string                  `json:"bitcoin_uri"`
	Created               int64                   `json:"created"`
	Currency              Currency                `json:"currency"`
	Customer              string                  `json:"customer"`
	Desc                  string                  `json:"description"`
	Email                 string                  `json:"email"`
	Filled                bool                    `json:"filled"`
	ID                    string                  `json:"id"`
	InboundAddress        string                  `json:"inbound_address"`
	Meta                  map[string]string       `json:"metadata"`
	Payment               string                  `json:"payment"`
	RefundAddress         string                  `json:"refund_address"`
	RejectTransactions    bool                    `json:"reject_transactions"`
	Transactions          *BitcoinTransactionList `json:"transactions"`
}

// BitcoinReceiverList is a list of bitcoin receivers as retrieved from a list endpoint.
type BitcoinReceiverList struct {
	ListMeta
	Values []*BitcoinReceiver `json:"data"`
}

// UnmarshalJSON handles deserialization of a BitcoinReceiver.
// This custom unmarshaling is needed because the resulting
// property may be an id or the full struct if it was expanded.
func (br *BitcoinReceiver) UnmarshalJSON(data []byte) error {
	type bitcoinReceiver BitcoinReceiver
	var r bitcoinReceiver
	err := json.Unmarshal(data, &r)
	if err == nil {
		*br = BitcoinReceiver(r)
	} else {
		// the id is surrounded by "\" characters, so strip them
		br.ID = string(data[1 : len(data)-1])
	}

	return nil
}
<|endoftext|>


<commit_before>package aeon

import (
	"fmt"

	"github.com/bitly/go-simplejson"

	"github.com/ninjasphere/go-ninja"
	"github.com/ninjasphere/go-ninja/devices"
	"github.com/ninjasphere/go-openzwave"
	"github.com/ninjasphere/go-openzwave/CC"
)

const (
	maxDeviceBrightness = 99 // by experiment, a level of 100 does not work for this device
)

type illuminator struct {
	api        openzwave.API
	node       openzwave.Node
	bus        *ninja.DriverBus
	light      *devices.LightDevice
	brightness uint8
}

func IlluminatorFactory(api openzwave.API, node openzwave.Node, bus *ninja.DriverBus) openzwave.Device {
	device := &illuminator{api, node, bus, nil, 0}
	return device
}

func (device *illuminator) NodeAdded() {
	var ok bool

	node := device.node
	api := device.api

	productId := node.GetProductId()
	productDescription := node.GetProductDescription()

	sigs := simplejson.New()
	sigs.Set("ninja:manufacturer", productDescription.ManufacturerName)
	sigs.Set("ninja:productName", productDescription.ProductName)
	sigs.Set("ninja:productType", productDescription.ProductType)
	sigs.Set("ninja:thingType", "light")

	address := fmt.Sprintf(
		"%08x:%03d:%s:%s",
		node.GetHomeId(),
		node.GetId(),
		productId.ManufacturerId,
		productId.ProductId)

	label := node.GetNodeName()

	// initialize brightness from the current level

	device.brightness, ok = device.node.GetValue(CC.SWITCH_MULTILEVEL, 1, 0).GetUint8()
	if !ok || device.brightness == 0 {
		// we have to reset brightness to 100 since we apply brightness when
		// we switch it on

		//
		// one implication of this is that if the controller is removed
		// then replaced, while the light is off, the original brightness
		// will be lost
		//

		device.brightness = maxDeviceBrightness
	}

	deviceBus, err := device.bus.AnnounceDevice(address, "light", label, sigs)
	if err != nil {
		api.Logger().Infof("failed to announce device: %v", node)
		return
	}

	device.light, err = devices.CreateLightDevice(label, deviceBus)
	if err != nil {
		api.Logger().Infof("failed to create light device: %v", node)
		return
	}

	if err := device.light.EnableOnOffChannel(); err != nil {
		api.Logger().Infof("Failed to enable on/off channel: %v", node)
		return
	}

	if err := device.light.EnableBrightnessChannel(); err != nil {
		api.Logger().Infof("Failed to brightness channel: %v", node)
		return
	}

	device.light.ApplyOnOff = func(state bool) error {

		// TODO: synchronize with notification thread

		level := uint8(0)
		if state {
			level = device.brightness
		}
		if device.node.GetValue(CC.SWITCH_MULTILEVEL, 1, 0).SetUint8(level) {
			return nil
		} else {
			return fmt.Errorf("Failed to change on/off state")
		}
	}

	device.light.ApplyBrightness = func(state float64) error {

		// TODO: synchronize with notification thread

		var err error = nil
		if state < 0 {
			state = 0
		} else if state > 1.0 {
			state = 1.0
		}
		level, ok := device.node.GetValue(CC.SWITCH_MULTILEVEL, 1, 0).GetUint8()
		if ok {
			device.brightness = uint8(state * maxDeviceBrightness)
			if level > 0 {
				if !device.node.GetValue(CC.SWITCH_MULTILEVEL, 1, 0).SetUint8(device.brightness) {
					err = fmt.Errorf("Failed to change brightness")
				}
			}
		} else {
			err = fmt.Errorf("Failed to read existing level from device")
		}
		return err
	}

	device.light.ApplyLightState = func(state *devices.LightDeviceState) error {

		// TODO: synchronize with notification thread

		device.light.ApplyOnOff(*state.OnOff)
		device.light.ApplyBrightness(*state.Brightness)
		return nil
	}
}

func (device *illuminator) NodeChanged() {
	state := &devices.LightDeviceState{}
	level, ok := device.node.GetValue(CC.SWITCH_MULTILEVEL, 1, 0).GetUint8()
	if ok {
		if level > maxDeviceBrightness {
			level = maxDeviceBrightness
		}

		if level != 0 {
			device.brightness = level
		}

		onOff := level != 0
		brightness := float64(device.brightness) / maxDeviceBrightness

		state.OnOff = &onOff
		state.Brightness = &brightness

		device.light.SetLightState(state)
	}
}

func (device *illuminator) NodeRemoved() {
}

func (device *illuminator) ValueChanged(openzwave.Value) {
}
<commit_msg>Set a minimum brightness.<commit_after>package aeon

import (
	"fmt"

	"github.com/bitly/go-simplejson"

	"github.com/ninjasphere/go-ninja"
	"github.com/ninjasphere/go-ninja/devices"
	"github.com/ninjasphere/go-openzwave"
	"github.com/ninjasphere/go-openzwave/CC"
)

const (
	maxDeviceBrightness = 99 // by experiment, a level of 100 does not work for this device
)

type illuminator struct {
	api        openzwave.API
	node       openzwave.Node
	bus        *ninja.DriverBus
	light      *devices.LightDevice
	brightness uint8
}

func IlluminatorFactory(api openzwave.API, node openzwave.Node, bus *ninja.DriverBus) openzwave.Device {
	device := &illuminator{api, node, bus, nil, 0}
	return device
}

func (device *illuminator) NodeAdded() {
	var ok bool

	node := device.node
	api := device.api

	productId := node.GetProductId()
	productDescription := node.GetProductDescription()

	sigs := simplejson.New()
	sigs.Set("ninja:manufacturer", productDescription.ManufacturerName)
	sigs.Set("ninja:productName", productDescription.ProductName)
	sigs.Set("ninja:productType", productDescription.ProductType)
	sigs.Set("ninja:thingType", "light")

	address := fmt.Sprintf(
		"%08x:%03d:%s:%s",
		node.GetHomeId(),
		node.GetId(),
		productId.ManufacturerId,
		productId.ProductId)

	label := node.GetNodeName()

	// initialize brightness from the current level

	device.brightness, ok = device.node.GetValue(CC.SWITCH_MULTILEVEL, 1, 0).GetUint8()
	if !ok || device.brightness == 0 {
		// we have to reset brightness to 100 since we apply brightness when
		// we switch it on

		//
		// one implication of this is that if the controller is removed
		// then replaced, while the light is off, the original brightness
		// will be lost
		//

		device.brightness = maxDeviceBrightness
	}

	deviceBus, err := device.bus.AnnounceDevice(address, "light", label, sigs)
	if err != nil {
		api.Logger().Infof("failed to announce device: %v", node)
		return
	}

	device.light, err = devices.CreateLightDevice(label, deviceBus)
	if err != nil {
		api.Logger().Infof("failed to create light device: %v", node)
		return
	}

	if err := device.light.EnableOnOffChannel(); err != nil {
		api.Logger().Infof("Failed to enable on/off channel: %v", node)
		return
	}

	if err := device.light.EnableBrightnessChannel(); err != nil {
		api.Logger().Infof("Failed to brightness channel: %v", node)
		return
	}

	device.light.ApplyOnOff = func(state bool) error {

		// TODO: synchronize with notification thread

		level := uint8(0)
		if state {
			level = device.brightness
		}
		if device.node.GetValue(CC.SWITCH_MULTILEVEL, 1, 0).SetUint8(level) {
			return nil
		} else {
			return fmt.Errorf("Failed to change on/off state")
		}
	}

	device.light.ApplyBrightness = func(state float64) error {

		// TODO: synchronize with notification thread

		var err error = nil
		if state < 0 {
			state = 0
		} else if state > 1.0 {
			state = 1.0
		}
		level, ok := device.node.GetValue(CC.SWITCH_MULTILEVEL, 1, 0).GetUint8()
		if ok {
			device.brightness = uint8(state * maxDeviceBrightness)
			if device.brightness < 1 {
				device.brightness = 1
			}
			if level > 0 {
				if !device.node.GetValue(CC.SWITCH_MULTILEVEL, 1, 0).SetUint8(device.brightness) {
					err = fmt.Errorf("Failed to change brightness")
				}
			}
		} else {
			err = fmt.Errorf("Failed to read existing level from device")
		}
		return err
	}

	device.light.ApplyLightState = func(state *devices.LightDeviceState) error {

		// TODO: synchronize with notification thread

		device.light.ApplyOnOff(*state.OnOff)
		device.light.ApplyBrightness(*state.Brightness)
		return nil
	}
}

func (device *illuminator) NodeChanged() {
	state := &devices.LightDeviceState{}
	level, ok := device.node.GetValue(CC.SWITCH_MULTILEVEL, 1, 0).GetUint8()
	if ok {
		if level > maxDeviceBrightness {
			level = maxDeviceBrightness
		}

		if level != 0 {
			device.brightness = level
		}

		onOff := level != 0
		brightness := float64(device.brightness) / maxDeviceBrightness

		state.OnOff = &onOff
		state.Brightness = &brightness

		device.light.SetLightState(state)
	}
}

func (device *illuminator) NodeRemoved() {
}

func (device *illuminator) ValueChanged(openzwave.Value) {
}
<|endoftext|>