<commit_before>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"runtime"
	"sort"
	"strconv"
	"strings"
	"syscall"

	"github.com/dickeyxxx/golock"
	"github.com/heroku/heroku-cli/gode"
)

// Plugin represents a javascript plugin
type Plugin struct {
	Name     string     `json:"name"`
	Version  string     `json:"version"`
	Topics   TopicSet   `json:"topics"`
	Topic    *Topic     `json:"topic"`
	Commands CommandSet `json:"commands"`
}

// SetupNode sets up node and npm in ~/.heroku
func SetupNode() {
	gode.SetRootPath(AppDir())
	setup, err := gode.IsSetup()
	PrintError(err)
	if !setup {
		setupNode()
	}
}

func setupNode() {
	Err("heroku-cli: Adding dependencies...")
	PrintError(gode.Setup())
	Errln(" done")
}

func updateNode() {
	gode.SetRootPath(AppDir())
	needsUpdate, err := gode.NeedsUpdate()
	PrintError(err)
	if needsUpdate {
		setupNode()
	}
}

// LoadPlugins loads the topics and commands from the JavaScript plugins into the CLI
func (cli *Cli) LoadPlugins(plugins map[string]*Plugin) {
	for _, plugin := range plugins {
		for _, topic := range plugin.Topics {
			cli.AddTopic(topic)
		}
		if plugin.Topic != nil {
			cli.AddTopic(plugin.Topic)
		}
		for _, command := range plugin.Commands {
			if !cli.AddCommand(command) {
				Errf("WARNING: command %s has already been defined\n", command)
			}
		}
	}
	sort.Sort(cli.Topics)
	sort.Sort(cli.Commands)
}

var pluginsTopic = &Topic{
	Name:        "plugins",
	Description: "manage plugins",
}

var pluginsInstallCmd = &Command{
	Topic:       "plugins",
	Command:     "install",
	Hidden:      true,
	Args:        []Arg{{Name: "name"}},
	Description: "Installs a plugin into the CLI",
	Help: `Install a Heroku plugin

  Example:
  $ heroku plugins:install dickeyxxx/heroku-production-status`,

	Run: func(ctx *Context) {
		name := ctx.Args.(map[string]string)["name"]
		if len(name) == 0 {
			Errln("Must specify a plugin name")
			return
		}
		Errf("Installing plugin %s... ", name)
		err := installPlugins(name)
		ExitIfError(err)
		plugin := getPlugin(name, false)
		if plugin == nil || len(plugin.Commands) == 0 {
			Err("\nThis does not appear to be a Heroku plugin, uninstalling... ")
			ExitIfError(gode.RemovePackage(name))
		}
		Errln("done")
	},
}

var pluginsLinkCmd = &Command{
	Topic:       "plugins",
	Command:     "link",
	Description: "Links a local plugin into CLI",
	Args:        []Arg{{Name: "path", Optional: true}},
	Help: `Links a local plugin into CLI.
	This is useful when developing plugins locally.
	It simply symlinks the specified path into ~/.heroku/node_modules

  Example:
	$ heroku plugins:link .`,

	Run: func(ctx *Context) {
		path := ctx.Args.(map[string]string)["path"]
		if path == "" {
			path = "."
		}
		path, err := filepath.Abs(path)
		if err != nil {
			panic(err)
		}
		if _, err := os.Stat(path); err != nil {
			panic(err)
		}
		name := filepath.Base(path)
		newPath := filepath.Join(ctx.HerokuDir, "node_modules", name)
		os.Remove(newPath)
		os.RemoveAll(newPath)
		err = os.Symlink(path, newPath)
		if err != nil {
			panic(err)
		}
		plugin := getPlugin(name, false)
		if plugin == nil || len(plugin.Commands) == 0 {
			Errln(name + " does not appear to be a Heroku plugin.\nDid you run `npm install`?")
			if err := os.Remove(newPath); err != nil {
				panic(err)
			}
			return
		}
		if name != plugin.Name {
			path = newPath
			newPath = filepath.Join(ctx.HerokuDir, "node_modules", plugin.Name)
			os.Remove(newPath)
			os.RemoveAll(newPath)
			os.Rename(path, newPath)
		}
		Println("symlinked", plugin.Name)
		Err("Updating plugin cache... ")
		AddPluginsToCache(plugin)
		Errln("done")
	},
}

var pluginsUninstallCmd = &Command{
	Topic:       "plugins",
	Command:     "uninstall",
	Hidden:      true,
	Args:        []Arg{{Name: "name"}},
	Description: "Uninstalls a plugin from the CLI",
	Help: `Uninstalls a Heroku plugin

  Example:
  $ heroku plugins:uninstall heroku-production-status`,

	Run: func(ctx *Context) {
		name := ctx.Args.(map[string]string)["name"]
		Errf("Uninstalling plugin %s... ", name)
		err := gode.RemovePackage(name)
		ExitIfError(err)
		Errln("done")
	},
}

var pluginsListCmd = &Command{
	Topic:       "plugins",
	Hidden:      true,
	Description: "Lists installed plugins",
	Help: `
Example:
  $ heroku plugins`,

	Run: func(ctx *Context) {
		for _, plugin := range GetPlugins() {
			if len(plugin.Commands) > 0 {
				Println(plugin.Name, plugin.Version)
			}
		}
	},
}

func runFn(plugin *Plugin, topic, command string) func(ctx *Context) {
	return func(ctx *Context) {
		lockfile := updateLockPath + "." + plugin.Name
		if exists, _ := fileExists(lockfile); exists {
			golock.Lock(lockfile)
			golock.Unlock(lockfile)
		}
		checkIfPluginIsInstalled(plugin.Name)
		ctx.Dev = isPluginSymlinked(plugin.Name)
		ctxJSON, err := json.Marshal(ctx)
		if err != nil {
			panic(err)
		}
		title, _ := json.Marshal(processTitle(ctx))
		script := fmt.Sprintf(`
		'use strict';
		var moduleName = '%s';
		var moduleVersion = '%s';
		var topic = '%s';
		var command = '%s';
		process.title = %s;
		var ctx = %s;
		ctx.version = ctx.version + ' ' + moduleName + '/' + moduleVersion + ' node-' + process.version;
		var logPath = %s;
		process.chdir(ctx.cwd);
		function repair (name) {
			console.error('Attempting to repair ' + name + '...');
			require('child_process')
			.spawnSync('heroku', ['plugins:install', name],
			{stdio: [0,1,2]});
			console.error('Repair complete. Try running your command again.');
		}
		if (!ctx.dev) {
			process.on('uncaughtException', function (err) {
				console.error(' !   Error in ' + moduleName + ':')
				if (err.message) {
					console.error(' !   ' + err.message);
					if (err.message.indexOf('Cannot find module') != -1) {
						repair(moduleName);
						process.exit(1);
					}
				} else {
					console.error(' !   ' + err);
				}
				if (err.stack) {
					var fs = require('fs');
					var log = function (line) {
						var d = new Date().toISOString()
						.replace(/T/, ' ')
						.replace(/-/g, '/')
						.replace(/\..+/, '');
						fs.appendFileSync(logPath, d + ' ' + line + '\n');
					}
					log('Error during ' + topic + ':' + command);
					log(err.stack);
					console.error(' !   See ' + logPath + ' for more info.');
				}
				process.exit(1);
			});
		}
		if (command === '') { command = null }
		var module = require(moduleName);
		var cmd = module.commands.filter(function (c) {
			return c.topic === topic && c.command == command;
		})[0];
		cmd.run(ctx);`, plugin.Name, plugin.Version, topic, command, string(title), ctxJSON, strconv.Quote(ErrLogPath))

		// swallow sigint since the plugin will handle it
		swallowSignal(os.Interrupt)

		cmd := gode.RunScript(script)
		if ctx.Flags["debugger"] == true {
			cmd = gode.DebugScript(script)
		}
		os.Chdir(cmd.Dir)
		execBin(cmd.Path, cmd.Args)
	}
}

func execBin(bin string, args []string) {
	if runtime.GOOS == "windows" {
		cmd := exec.Command(bin, args[1:]...)
		cmd.Stdin = os.Stdin
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		if err := cmd.Run(); err != nil {
			os.Exit(getExitCode(err))
		}
	} else {
		if err := syscall.Exec(bin, args, os.Environ()); err != nil {
			panic(err)
		}
	}
}

func swallowSignal(s os.Signal) {
	c := make(chan os.Signal, 1)
	signal.Notify(c, s)
	go func() {
		<-c
	}()
}

func getExitCode(err error) int {
	switch e := err.(type) {
	case *exec.ExitError:
		status, ok := e.Sys().(syscall.WaitStatus)
		if !ok {
			panic(err)
		}
		return status.ExitStatus()
	default:
		panic(err)
	}
}

func getPlugin(name string, attemptReinstall bool) *Plugin {
	script := `
	var plugin = require('` + name + `');
	if (!plugin.commands) plugin = {}; // not a real plugin
	var pjson  = require('` + name + `/package.json');

	plugin.name    = pjson.name;
	plugin.version = pjson.version;

	console.log(JSON.stringify(plugin))`
	cmd := gode.RunScript(script)
	output, err := cmd.CombinedOutput()
	if err != nil {
		if attemptReinstall && strings.Contains(string(output), "Error: Cannot find module") {
			Errf("Error reading plugin %s. Reinstalling... ", name)
			if err := installPlugins(name); err != nil {
				panic(errors.New(name + ": " + string(output)))
			}
			Errln("done")
			return getPlugin(name, false)
		}
		Errf("Error reading plugin: %s. See %s for more information.\n", name, ErrLogPath)
		Logln(err, "\n", string(output))
		return nil
	}
	var plugin Plugin
	json.Unmarshal([]byte(output), &plugin)
	for _, command := range plugin.Commands {
		command.Plugin = plugin.Name
		command.Help = strings.TrimSpace(command.Help)
	}
	return &plugin
}

// GetPlugins goes through all the node plugins and returns them in Go stucts
func GetPlugins() map[string]*Plugin {
	plugins := FetchPluginCache()
	for _, plugin := range plugins {
		for _, command := range plugin.Commands {
			command.Run = runFn(plugin, command.Topic, command.Command)
		}
	}
	return plugins
}

// PluginNames lists all the plugin names
func PluginNames() []string {
	plugins := FetchPluginCache()
	names := make([]string, 0, len(plugins))
	for _, plugin := range plugins {
		if plugin != nil {
			names = append(names, plugin.Name)
		}
	}
	return names
}

// PluginNamesNotSymlinked returns all the plugins that are not symlinked
func PluginNamesNotSymlinked() []string {
	a := PluginNames()
	b := make([]string, 0, len(a))
	for _, plugin := range a {
		if !isPluginSymlinked(plugin) {
			b = append(b, plugin)
		}
	}
	return b
}

func isPluginSymlinked(plugin string) bool {
	path := filepath.Join(AppDir(), "node_modules", plugin)
	fi, err := os.Lstat(path)
	if err != nil {
		panic(err)
	}
	return fi.Mode()&os.ModeSymlink != 0
}

// SetupBuiltinPlugins ensures all the builtinPlugins are installed
func SetupBuiltinPlugins() {
	pluginNames := difference(BuiltinPlugins, PluginNames())
	if len(pluginNames) == 0 {
		return
	}
	Err("heroku-cli: Installing core plugins...")
	err := installPlugins(pluginNames...)
	if err != nil {
		Errln()
		PrintError(err)
		return
	}
	Errln(" done")
}

func difference(a, b []string) []string {
	res := make([]string, 0, len(a))
	for _, aa := range a {
		if !contains(b, aa) {
			res = append(res, aa)
		}
	}
	return res
}

func contains(arr []string, s string) bool {
	for _, a := range arr {
		if a == s {
			return true
		}
	}
	return false
}

func installPlugins(names ...string) error {
	for _, name := range names {
		lockfile := updateLockPath + "." + name
		LogIfError(golock.Lock(lockfile))
	}
	err := gode.InstallPackage(names...)
	plugins := make([]*Plugin, 0, len(names))
	for _, name := range names {
		plugins = append(plugins, getPlugin(name, false))
	}
	AddPluginsToCache(plugins...)
	for _, name := range names {
		lockfile := updateLockPath + "." + name
		LogIfError(golock.Unlock(lockfile))
	}
	return err
}

func checkIfPluginIsInstalled(plugin string) {
	if exists, _ := fileExists(filepath.Join(AppDir(), "node_modules", plugin)); !exists {
		installPlugins(plugin)
	}
}
<commit_msg>prevent against null plugins<commit_after>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"runtime"
	"sort"
	"strconv"
	"strings"
	"syscall"

	"github.com/dickeyxxx/golock"
	"github.com/heroku/heroku-cli/gode"
)

// Plugin represents a javascript plugin
type Plugin struct {
	Name     string     `json:"name"`
	Version  string     `json:"version"`
	Topics   TopicSet   `json:"topics"`
	Topic    *Topic     `json:"topic"`
	Commands CommandSet `json:"commands"`
}

// SetupNode sets up node and npm in ~/.heroku
func SetupNode() {
	gode.SetRootPath(AppDir())
	setup, err := gode.IsSetup()
	PrintError(err)
	if !setup {
		setupNode()
	}
}

func setupNode() {
	Err("heroku-cli: Adding dependencies...")
	PrintError(gode.Setup())
	Errln(" done")
}

func updateNode() {
	gode.SetRootPath(AppDir())
	needsUpdate, err := gode.NeedsUpdate()
	PrintError(err)
	if needsUpdate {
		setupNode()
	}
}

// LoadPlugins loads the topics and commands from the JavaScript plugins into the CLI
func (cli *Cli) LoadPlugins(plugins map[string]*Plugin) {
	for _, plugin := range plugins {
		for _, topic := range plugin.Topics {
			cli.AddTopic(topic)
		}
		if plugin.Topic != nil {
			cli.AddTopic(plugin.Topic)
		}
		for _, command := range plugin.Commands {
			if !cli.AddCommand(command) {
				Errf("WARNING: command %s has already been defined\n", command)
			}
		}
	}
	sort.Sort(cli.Topics)
	sort.Sort(cli.Commands)
}

var pluginsTopic = &Topic{
	Name:        "plugins",
	Description: "manage plugins",
}

var pluginsInstallCmd = &Command{
	Topic:       "plugins",
	Command:     "install",
	Hidden:      true,
	Args:        []Arg{{Name: "name"}},
	Description: "Installs a plugin into the CLI",
	Help: `Install a Heroku plugin

  Example:
  $ heroku plugins:install dickeyxxx/heroku-production-status`,

	Run: func(ctx *Context) {
		name := ctx.Args.(map[string]string)["name"]
		if len(name) == 0 {
			Errln("Must specify a plugin name")
			return
		}
		Errf("Installing plugin %s... ", name)
		err := installPlugins(name)
		ExitIfError(err)
		plugin := getPlugin(name, false)
		if plugin == nil || len(plugin.Commands) == 0 {
			Err("\nThis does not appear to be a Heroku plugin, uninstalling... ")
			ExitIfError(gode.RemovePackage(name))
		}
		Errln("done")
	},
}

var pluginsLinkCmd = &Command{
	Topic:       "plugins",
	Command:     "link",
	Description: "Links a local plugin into CLI",
	Args:        []Arg{{Name: "path", Optional: true}},
	Help: `Links a local plugin into CLI.
	This is useful when developing plugins locally.
	It simply symlinks the specified path into ~/.heroku/node_modules

  Example:
	$ heroku plugins:link .`,

	Run: func(ctx *Context) {
		path := ctx.Args.(map[string]string)["path"]
		if path == "" {
			path = "."
		}
		path, err := filepath.Abs(path)
		if err != nil {
			panic(err)
		}
		if _, err := os.Stat(path); err != nil {
			panic(err)
		}
		name := filepath.Base(path)
		newPath := filepath.Join(ctx.HerokuDir, "node_modules", name)
		os.Remove(newPath)
		os.RemoveAll(newPath)
		err = os.Symlink(path, newPath)
		if err != nil {
			panic(err)
		}
		plugin := getPlugin(name, false)
		if plugin == nil || len(plugin.Commands) == 0 {
			Errln(name + " does not appear to be a Heroku plugin.\nDid you run `npm install`?")
			if err := os.Remove(newPath); err != nil {
				panic(err)
			}
			return
		}
		if name != plugin.Name {
			path = newPath
			newPath = filepath.Join(ctx.HerokuDir, "node_modules", plugin.Name)
			os.Remove(newPath)
			os.RemoveAll(newPath)
			os.Rename(path, newPath)
		}
		Println("symlinked", plugin.Name)
		Err("Updating plugin cache... ")
		AddPluginsToCache(plugin)
		Errln("done")
	},
}

var pluginsUninstallCmd = &Command{
	Topic:       "plugins",
	Command:     "uninstall",
	Hidden:      true,
	Args:        []Arg{{Name: "name"}},
	Description: "Uninstalls a plugin from the CLI",
	Help: `Uninstalls a Heroku plugin

  Example:
  $ heroku plugins:uninstall heroku-production-status`,

	Run: func(ctx *Context) {
		name := ctx.Args.(map[string]string)["name"]
		Errf("Uninstalling plugin %s... ", name)
		err := gode.RemovePackage(name)
		ExitIfError(err)
		Errln("done")
	},
}

var pluginsListCmd = &Command{
	Topic:       "plugins",
	Hidden:      true,
	Description: "Lists installed plugins",
	Help: `
Example:
  $ heroku plugins`,

	Run: func(ctx *Context) {
		for _, plugin := range GetPlugins() {
			if len(plugin.Commands) > 0 {
				Println(plugin.Name, plugin.Version)
			}
		}
	},
}

func runFn(plugin *Plugin, topic, command string) func(ctx *Context) {
	return func(ctx *Context) {
		lockfile := updateLockPath + "." + plugin.Name
		if exists, _ := fileExists(lockfile); exists {
			golock.Lock(lockfile)
			golock.Unlock(lockfile)
		}
		checkIfPluginIsInstalled(plugin.Name)
		ctx.Dev = isPluginSymlinked(plugin.Name)
		ctxJSON, err := json.Marshal(ctx)
		if err != nil {
			panic(err)
		}
		title, _ := json.Marshal(processTitle(ctx))
		script := fmt.Sprintf(`
		'use strict';
		var moduleName = '%s';
		var moduleVersion = '%s';
		var topic = '%s';
		var command = '%s';
		process.title = %s;
		var ctx = %s;
		ctx.version = ctx.version + ' ' + moduleName + '/' + moduleVersion + ' node-' + process.version;
		var logPath = %s;
		process.chdir(ctx.cwd);
		function repair (name) {
			console.error('Attempting to repair ' + name + '...');
			require('child_process')
			.spawnSync('heroku', ['plugins:install', name],
			{stdio: [0,1,2]});
			console.error('Repair complete. Try running your command again.');
		}
		if (!ctx.dev) {
			process.on('uncaughtException', function (err) {
				console.error(' !   Error in ' + moduleName + ':')
				if (err.message) {
					console.error(' !   ' + err.message);
					if (err.message.indexOf('Cannot find module') != -1) {
						repair(moduleName);
						process.exit(1);
					}
				} else {
					console.error(' !   ' + err);
				}
				if (err.stack) {
					var fs = require('fs');
					var log = function (line) {
						var d = new Date().toISOString()
						.replace(/T/, ' ')
						.replace(/-/g, '/')
						.replace(/\..+/, '');
						fs.appendFileSync(logPath, d + ' ' + line + '\n');
					}
					log('Error during ' + topic + ':' + command);
					log(err.stack);
					console.error(' !   See ' + logPath + ' for more info.');
				}
				process.exit(1);
			});
		}
		if (command === '') { command = null }
		var module = require(moduleName);
		var cmd = module.commands.filter(function (c) {
			return c.topic === topic && c.command == command;
		})[0];
		cmd.run(ctx);`, plugin.Name, plugin.Version, topic, command, string(title), ctxJSON, strconv.Quote(ErrLogPath))

		// swallow sigint since the plugin will handle it
		swallowSignal(os.Interrupt)

		cmd := gode.RunScript(script)
		if ctx.Flags["debugger"] == true {
			cmd = gode.DebugScript(script)
		}
		os.Chdir(cmd.Dir)
		execBin(cmd.Path, cmd.Args)
	}
}

func execBin(bin string, args []string) {
	if runtime.GOOS == "windows" {
		cmd := exec.Command(bin, args[1:]...)
		cmd.Stdin = os.Stdin
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		if err := cmd.Run(); err != nil {
			os.Exit(getExitCode(err))
		}
	} else {
		if err := syscall.Exec(bin, args, os.Environ()); err != nil {
			panic(err)
		}
	}
}

func swallowSignal(s os.Signal) {
	c := make(chan os.Signal, 1)
	signal.Notify(c, s)
	go func() {
		<-c
	}()
}

func getExitCode(err error) int {
	switch e := err.(type) {
	case *exec.ExitError:
		status, ok := e.Sys().(syscall.WaitStatus)
		if !ok {
			panic(err)
		}
		return status.ExitStatus()
	default:
		panic(err)
	}
}

func getPlugin(name string, attemptReinstall bool) *Plugin {
	script := `
	var plugin = require('` + name + `');
	if (!plugin.commands) plugin = {}; // not a real plugin
	var pjson  = require('` + name + `/package.json');

	plugin.name    = pjson.name;
	plugin.version = pjson.version;

	console.log(JSON.stringify(plugin))`
	cmd := gode.RunScript(script)
	output, err := cmd.CombinedOutput()
	if err != nil {
		if attemptReinstall && strings.Contains(string(output), "Error: Cannot find module") {
			Errf("Error reading plugin %s. Reinstalling... ", name)
			if err := installPlugins(name); err != nil {
				panic(errors.New(name + ": " + string(output)))
			}
			Errln("done")
			return getPlugin(name, false)
		}
		Errf("Error reading plugin: %s. See %s for more information.\n", name, ErrLogPath)
		Logln(err, "\n", string(output))
		return nil
	}
	var plugin Plugin
	json.Unmarshal([]byte(output), &plugin)
	for _, command := range plugin.Commands {
		command.Plugin = plugin.Name
		command.Help = strings.TrimSpace(command.Help)
	}
	return &plugin
}

// GetPlugins goes through all the node plugins and returns them in Go stucts
func GetPlugins() map[string]*Plugin {
	plugins := FetchPluginCache()
	for _, plugin := range plugins {
		if plugin != nil {
			for _, command := range plugin.Commands {
				command.Run = runFn(plugin, command.Topic, command.Command)
			}
		}
	}
	return plugins
}

// PluginNames lists all the plugin names
func PluginNames() []string {
	plugins := FetchPluginCache()
	names := make([]string, 0, len(plugins))
	for _, plugin := range plugins {
		if plugin != nil {
			names = append(names, plugin.Name)
		}
	}
	return names
}

// PluginNamesNotSymlinked returns all the plugins that are not symlinked
func PluginNamesNotSymlinked() []string {
	a := PluginNames()
	b := make([]string, 0, len(a))
	for _, plugin := range a {
		if !isPluginSymlinked(plugin) {
			b = append(b, plugin)
		}
	}
	return b
}

func isPluginSymlinked(plugin string) bool {
	path := filepath.Join(AppDir(), "node_modules", plugin)
	fi, err := os.Lstat(path)
	if err != nil {
		panic(err)
	}
	return fi.Mode()&os.ModeSymlink != 0
}

// SetupBuiltinPlugins ensures all the builtinPlugins are installed
func SetupBuiltinPlugins() {
	pluginNames := difference(BuiltinPlugins, PluginNames())
	if len(pluginNames) == 0 {
		return
	}
	Err("heroku-cli: Installing core plugins...")
	err := installPlugins(pluginNames...)
	if err != nil {
		Errln()
		PrintError(err)
		return
	}
	Errln(" done")
}

func difference(a, b []string) []string {
	res := make([]string, 0, len(a))
	for _, aa := range a {
		if !contains(b, aa) {
			res = append(res, aa)
		}
	}
	return res
}

func contains(arr []string, s string) bool {
	for _, a := range arr {
		if a == s {
			return true
		}
	}
	return false
}

func installPlugins(names ...string) error {
	for _, name := range names {
		lockfile := updateLockPath + "." + name
		LogIfError(golock.Lock(lockfile))
	}
	err := gode.InstallPackage(names...)
	plugins := make([]*Plugin, 0, len(names))
	for _, name := range names {
		plugins = append(plugins, getPlugin(name, false))
	}
	AddPluginsToCache(plugins...)
	for _, name := range names {
		lockfile := updateLockPath + "." + name
		LogIfError(golock.Unlock(lockfile))
	}
	return err
}

func checkIfPluginIsInstalled(plugin string) {
	if exists, _ := fileExists(filepath.Join(AppDir(), "node_modules", plugin)); !exists {
		installPlugins(plugin)
	}
}
<|endoftext|>


<commit_before>package plyfile

/*
#cgo LDFLAGS: -L${SRCDIR}/lib -lplyfile
#include <string.h>
#include "lib/ply.h"
*/
import "C"

import (
  "os"
  "unsafe"
  "bytes"
  //"encoding/gob"
  "encoding/binary"
  "fmt"
)

const (
  PLY_ASCII = 1        /* ascii PLY file */
  PLY_BINARY_BE = 2        /* binary PLY file, big endian */
  PLY_BINARY_LE = 3        /* binary PLY file, little endian */

  PLY_OKAY = 0           /* ply routine worked okay */
  PLY_ERROR = -1           /* error in ply routine */

  /* scalar data types supported by PLY format */
  PLY_START_TYPE = 0
  PLY_CHAR = 1
  PLY_SHORT = 2
  PLY_INT = 3
  PLY_UCHAR = 4
  PLY_USHORT = 5
  PLY_UINT = 6
  PLY_FLOAT = 7
  PLY_DOUBLE = 8
  PLY_END_TYPE = 9

  PLY_SCALAR = 0
  PLY_LIST = 1
)

type CPlyProperty C.struct_PlyProperty
type PlyProperty struct {
  name string               /* property name */
  external_type int         /* file's data type */
  internal_type int                    /* program's data type */
  offset int                           /* offset bytes of prop in a struct */

  is_list int                          /* 1 = list, 0 = scalar */
  count_external int                  /* file's count type */
  count_internal int                   /* program's count type */
  count_offset int                 /* offset byte for list count */
}
/* ToC converts a PlyProperty go structure to a PlyProperty C structure for passing to C functions */
func (prop *PlyProperty) ToC() CPlyProperty {
  var cprop CPlyProperty
  cprop.name = C.CString(prop.name)
  cprop.external_type = C.int(prop.external_type)
  cprop.internal_type = C.int(prop.internal_type)
  cprop.offset = C.int(prop.offset)
  cprop.is_list = C.int(prop.is_list)
  cprop.count_external = C.int(prop.count_external)
  cprop.count_internal = C.int(prop.count_internal)
  cprop.count_offset = C.int(prop.count_offset)
  return cprop
}
func (prop *PlyProperty) FromC(cprop CPlyProperty) {
  prop.name = C.GoString(cprop.name)
  prop.external_type = int(cprop.external_type)
  prop.internal_type = int(cprop.internal_type)
  prop.offset = int(cprop.offset)

  prop.is_list = int(cprop.is_list)
  prop.count_external = int(cprop.count_external)
  prop.count_internal = int(cprop.count_internal)
  prop.count_offset = int(cprop.count_offset)
}

type PlyElement struct {
  name string                   /* element name */
  num int                      /* number of elements in this object */
  size int                     /* size of element (bytes) or -1 if variable */
  nprops int                   /* number of properties for this element */
  props []PlyProperty          /* list of properties in the file */
  store_prop string             /* flags: property wanted by user? */
  other_offset int             /* offset to un-asked-for props, or -1 if none*/
  other_size int               /* size of other_props structure */
}

type PlyOtherProp struct {   /* describes other properties in an element */
  name string                   /* element name */
  size int                     /* size of other_props */
  nprops int                   /* number of properties in other_props */
  props []PlyProperty          /* list of properties in other_props */
}

type PlyFile struct {        /* description of PLY file */
  FILE *os.File                     /* file pointer */
  file_type int                /* ascii or binary */
  version float64                /* version number of file */
  nelems int                   /* number of elements of object */
  elems []PlyElement           /* list of elements */
  num_comments int             /* number of comments */
  comments []string;              /* list of comments */
  num_obj_info int             /* number of items of object information */
  obj_info []string              /* list of object info items */
  which_elem PlyElement       /* which element we're currently writing */
}

type CPlyFile *C.struct_PlyFile
type CPlyElement *C.struct_PlyElement

func PlyOpenForWriting(filename string, nelems int, elem_names []string, file_type int, version *float32) CPlyFile {

  c_elem_names := make([]*C.char, nelems)
  for i := 0; i < nelems; i++ {
    c_elem_names[i] = C.CString(elem_names[i])
  }

  plyfile := C.ply_open_for_writing(C.CString(filename), C.int(nelems), &c_elem_names[0], C.int(file_type), (*C.float)(version))

  return plyfile
}

func PlyOpenForReading(filename string) (CPlyFile, []string) {

  plyfile := C.ply_open_and_read_header(C.CString(filename))

  nelems := int(plyfile.nelems)

  elements := make([]CPlyElement, nelems)
  elem_names := make([]string, nelems)

  for i := 0; i < nelems; i++ {
    elements[i] = C.ply_get_element_by_index(plyfile, C.int(i))
    elem_names[i] = C.GoString(elements[i].name)
  }

  return plyfile, elem_names
}

func PlyClose(plyfile CPlyFile) {
  C.ply_close(plyfile)
}

/* Writing Functions */

func PlyElementCount(plyfile CPlyFile, element_name string, nelems int) {
  C.ply_element_count(plyfile, C.CString(element_name), C.int(nelems))
}

func PlyDescribeProperty(plyfile CPlyFile, element_name string, prop PlyProperty) {
  propertyptr := prop.ToC()
  C.ply_describe_property(plyfile, C.CString(element_name), &propertyptr)
}

func PlyPutComment(plyfile CPlyFile, comment string) {
  C.ply_put_comment(plyfile, C.CString(comment))
}

func PlyPutObjInfo(plyfile CPlyFile, obj_info string) {
  C.ply_put_obj_info(plyfile, C.CString(obj_info))
}

func PlyHeaderComplete(plyfile CPlyFile) {
  C.ply_header_complete(plyfile)
}

func PlyPutElementSetup(plyfile CPlyFile, element_name string) {
  C.ply_put_element_setup(plyfile, C.CString(element_name))
}

func PlyPutElement(plyfile CPlyFile, element interface{}) {
  // write the passed in element to a buffer
  buf := new(bytes.Buffer)
  err := binary.Write(buf, binary.LittleEndian, element)
  if err != nil {
    panic(err)
  }
  element_bytes := buf.Bytes()

  // pass a pointer to the buffer
  C.ply_put_element(plyfile, unsafe.Pointer(&element_bytes[0]))
}

/* Reading Functions */
func PlyGetElementDescription(plyfile CPlyFile, element_name string) ([]PlyProperty, int) {
  var nelems int
  var nprops int

  cnelems := C.int(nelems)
  cnprops := C.int(nprops)

  cplist_ptr := C.ply_get_element_description(plyfile, C.CString(element_name), &cnelems, &cnprops)

  nprops = int(cnprops)

  // convert cplist_ptr to a go slice of pointers
  cplist_ptr_go := (*[1<<30]*CPlyProperty)(unsafe.Pointer(cplist_ptr))[:nprops]

  // iterate through the slice of pointers, converting from CPlyProperty to PlyProperty
  plist := make([]PlyProperty, nprops)
  for i := 0; i < nprops; i++ {
    tmp := *cplist_ptr_go[i]
    plist[i].FromC(tmp)
  }

  return plist, int(cnelems)
}

func pointerToInt(ptr uintptr) []byte {
  size := unsafe.Sizeof(ptr)
  buf := make([]byte, size)
  switch size {
  case 4:
    binary.LittleEndian.PutUint32(buf, uint32(ptr))
  case 8:
    binary.LittleEndian.PutUint64(buf, uint64(ptr))
  default:
    panic(fmt.Sprintf("Error: unknown ptr size: %v", size))
  }
  return buf
}

func copyByteSliceToArray(barray *[8]byte, bslice []byte) {
  for i := 0; i < len(bslice); i++ {
    barray[i] = bslice[i]
  }
}
<commit_msg>Working version of PlyGetElementDescription. Committing old code in fcn for possible future use.<commit_after>package plyfile

/*
#cgo LDFLAGS: -L${SRCDIR}/lib -lplyfile
#include <string.h>
#include "lib/ply.h"
*/
import "C"

import (
  "os"
  "unsafe"
  "bytes"
  //"encoding/gob"
  "encoding/binary"
  "fmt"
  "reflect"
)

const (
  PLY_ASCII = 1        /* ascii PLY file */
  PLY_BINARY_BE = 2        /* binary PLY file, big endian */
  PLY_BINARY_LE = 3        /* binary PLY file, little endian */

  PLY_OKAY = 0           /* ply routine worked okay */
  PLY_ERROR = -1           /* error in ply routine */

  /* scalar data types supported by PLY format */
  PLY_START_TYPE = 0
  PLY_CHAR = 1
  PLY_SHORT = 2
  PLY_INT = 3
  PLY_UCHAR = 4
  PLY_USHORT = 5
  PLY_UINT = 6
  PLY_FLOAT = 7
  PLY_DOUBLE = 8
  PLY_END_TYPE = 9

  PLY_SCALAR = 0
  PLY_LIST = 1
)

type CPlyProperty C.struct_PlyProperty
type PlyProperty struct {
  name string               /* property name */
  external_type int         /* file's data type */
  internal_type int                    /* program's data type */
  offset int                           /* offset bytes of prop in a struct */

  is_list int                          /* 1 = list, 0 = scalar */
  count_external int                  /* file's count type */
  count_internal int                   /* program's count type */
  count_offset int                 /* offset byte for list count */
}
/* ToC converts a PlyProperty go structure to a PlyProperty C structure for passing to C functions */
func (prop *PlyProperty) ToC() CPlyProperty {
  var cprop CPlyProperty
  cprop.name = C.CString(prop.name)
  cprop.external_type = C.int(prop.external_type)
  cprop.internal_type = C.int(prop.internal_type)
  cprop.offset = C.int(prop.offset)
  cprop.is_list = C.int(prop.is_list)
  cprop.count_external = C.int(prop.count_external)
  cprop.count_internal = C.int(prop.count_internal)
  cprop.count_offset = C.int(prop.count_offset)
  return cprop
}
func (prop *PlyProperty) FromC(cprop CPlyProperty) {
  prop.name = C.GoString(cprop.name)
  prop.external_type = int(cprop.external_type)
  prop.internal_type = int(cprop.internal_type)
  prop.offset = int(cprop.offset)

  prop.is_list = int(cprop.is_list)
  prop.count_external = int(cprop.count_external)
  prop.count_internal = int(cprop.count_internal)
  prop.count_offset = int(cprop.count_offset)
}

type PlyElement struct {
  name string                   /* element name */
  num int                      /* number of elements in this object */
  size int                     /* size of element (bytes) or -1 if variable */
  nprops int                   /* number of properties for this element */
  props []PlyProperty          /* list of properties in the file */
  store_prop string             /* flags: property wanted by user? */
  other_offset int             /* offset to un-asked-for props, or -1 if none*/
  other_size int               /* size of other_props structure */
}

type PlyOtherProp struct {   /* describes other properties in an element */
  name string                   /* element name */
  size int                     /* size of other_props */
  nprops int                   /* number of properties in other_props */
  props []PlyProperty          /* list of properties in other_props */
}

type PlyFile struct {        /* description of PLY file */
  FILE *os.File                     /* file pointer */
  file_type int                /* ascii or binary */
  version float64                /* version number of file */
  nelems int                   /* number of elements of object */
  elems []PlyElement           /* list of elements */
  num_comments int             /* number of comments */
  comments []string;              /* list of comments */
  num_obj_info int             /* number of items of object information */
  obj_info []string              /* list of object info items */
  which_elem PlyElement       /* which element we're currently writing */
}

type CPlyFile *C.struct_PlyFile
type CPlyElement *C.struct_PlyElement

func PlyOpenForWriting(filename string, nelems int, elem_names []string, file_type int, version *float32) CPlyFile {

  c_elem_names := make([]*C.char, nelems)
  for i := 0; i < nelems; i++ {
    c_elem_names[i] = C.CString(elem_names[i])
  }

  plyfile := C.ply_open_for_writing(C.CString(filename), C.int(nelems), &c_elem_names[0], C.int(file_type), (*C.float)(version))

  return plyfile
}

func PlyOpenForReading(filename string) (CPlyFile, []string) {

  plyfile := C.ply_open_and_read_header(C.CString(filename))

  nelems := int(plyfile.nelems)

  elements := make([]CPlyElement, nelems)
  elem_names := make([]string, nelems)

  for i := 0; i < nelems; i++ {
    elements[i] = C.ply_get_element_by_index(plyfile, C.int(i))
    elem_names[i] = C.GoString(elements[i].name)
  }

  return plyfile, elem_names
}

func PlyClose(plyfile CPlyFile) {
  C.ply_close(plyfile)
}

/* Writing Functions */

func PlyElementCount(plyfile CPlyFile, element_name string, nelems int) {
  C.ply_element_count(plyfile, C.CString(element_name), C.int(nelems))
}

func PlyDescribeProperty(plyfile CPlyFile, element_name string, prop PlyProperty) {
  propertyptr := prop.ToC()
  C.ply_describe_property(plyfile, C.CString(element_name), &propertyptr)
}

func PlyPutComment(plyfile CPlyFile, comment string) {
  C.ply_put_comment(plyfile, C.CString(comment))
}

func PlyPutObjInfo(plyfile CPlyFile, obj_info string) {
  C.ply_put_obj_info(plyfile, C.CString(obj_info))
}

func PlyHeaderComplete(plyfile CPlyFile) {
  C.ply_header_complete(plyfile)
}

func PlyPutElementSetup(plyfile CPlyFile, element_name string) {
  C.ply_put_element_setup(plyfile, C.CString(element_name))
}

func PlyPutElement(plyfile CPlyFile, element interface{}) {
  // write the passed in element to a buffer
  buf := new(bytes.Buffer)
  err := binary.Write(buf, binary.LittleEndian, element)
  if err != nil {
    panic(err)
  }
  element_bytes := buf.Bytes()

  // pass a pointer to the buffer
  C.ply_put_element(plyfile, unsafe.Pointer(&element_bytes[0]))
}

/* Reading Functions */
func PlyGetElementDescription(plyfile CPlyFile, element_name string) ([]PlyProperty, int) {
  var nelems int
  var nprops int

  cnelems := C.int(nelems)
  cnprops := C.int(nprops)

  cplist_ptr := C.ply_get_element_description(plyfile, C.CString(element_name), &cnelems, &cnprops)

  nprops = int(cnprops)

  // convert cplist_ptr to a go slice of pointers
  cplist_ptr_go := (*[1<<30]*CPlyProperty)(unsafe.Pointer(cplist_ptr))[:nprops]

  // iterate through the slice of pointers, converting from CPlyProperty to PlyProperty
  plist := make([]PlyProperty, nprops)
  for i := 0; i < nprops; i++ {
    tmp := *cplist_ptr_go[i]
    plist[i].FromC(tmp)
  }

  return plist, int(cnelems)
}

func PlyGetProperty(plyfile CPlyFile, elem_name string, prop PlyProperty) {
  cprop := prop.ToC()
  C.ply_get_property(plyfile, C.CString(elem_name), &cprop)
}

func PlyGetElement(plyfile CPlyFile, element interface {}, size uintptr) {

  var ptr C.char

  C.ply_get_element(plyfile, unsafe.Pointer(&ptr))
  //fmt.Println(&ptr)

  // convert the pointer into a number, so we can do pointer arithmetic
  ptrval := uintptr(unsafe.Pointer(&ptr))

  // convert the *C.char array into a byte slice
  var byteSlice = make([]byte, size)
  for i := 0; i < len(byteSlice); i++ {
    byteSlice[i] = byte(*(*C.char)(unsafe.Pointer(ptrval)))
    ptrval++
  }

  /*
  // create a new temporary element
  elem_type := reflect.TypeOf(element)
  fmt.Println(elem_type)
  //elem_tmp := reflect.New(elem_type).Elem().Addr().Interface()
  elem_tmp := reflect.New(elem_type).Elem().Addr() //.Elem().Addr().Interface()
  fmt.Println(elem_tmp)
  fmt.Printf("type: %T\n", elem_tmp)
  */

  // copy byte slice into temporary element
  elem_ptr := reflect.ValueOf(element).Elem().Addr().Interface() // should be elem_ptr

  r := bytes.NewReader(byteSlice)
  err := binary.Read(r, binary.LittleEndian, elem_ptr)
  if err != nil {
    panic(err)
  }

  // set input element to be temporary element (perform deep copy)
  /*
  for i := 0; i < reflect.ValueOf(element).NumField(); i++ {
    reflect.ValueOf(element).Ptr().Field(i).Set( reflect.ValueOf(elem_tmp).Field(i) )
  }
  */
  //fmt.Printf("1", reflect.ValueOf(element).Interface())

  //reflect.ValueOf(element) = elem_tmp

  //var v reflect.TypeOf(element)
  //fmt.Printf("%T\n\n", v)

  /* OLDDD!!!!!!!!!!!!!! */
  /*
  element_value := reflect.ValueOf(&element).Elem()
  fmt.Println(element_value.CanSet())

  fmt.Println(element_value.Kind())
  fmt.Println("before", element_value)
  err := binary.Read(r, binary.LittleEndian, element_value)
  if err != nil {
    panic(err)
  }
  fmt.Println("after", element_value)

  fmt.Println(element)
  */
  /*
  t := reflect.ValueOf(element)
  fmt.Printf("type %T\n\n", t)
  fmt.Println("t is ", t)
  */

  /*
  // copy byte slice into

  fmt.Println(byteSlice)

  t := reflect.New(reflect.TypeOf(element)).Elem().Interface()
  fmt.Println(t)
  fmt.Printf("Type: %T\n\n", t)

  element = t
  */

  /*
  buf := new(bytes.Buffer)
  for i := 0; i < int(size); i++ {
    buf.WriteByte(ptr[i])
  }

  */
  //fmt.Println(size)

  ///fmt.Println(element)

}

/* misc functions */

func pointerToInt(ptr uintptr) []byte {
  size := unsafe.Sizeof(ptr)
  buf := make([]byte, size)
  switch size {
  case 4:
    binary.LittleEndian.PutUint32(buf, uint32(ptr))
  case 8:
    binary.LittleEndian.PutUint64(buf, uint64(ptr))
  default:
    panic(fmt.Sprintf("Error: unknown ptr size: %v", size))
  }
  return buf
}

func copyByteSliceToArray(barray *[8]byte, bslice []byte) {
  for i := 0; i < len(bslice); i++ {
    barray[i] = bslice[i]
  }
}
<|endoftext|>


<commit_before>package resource

import (
	"errors"
	"reflect"

	"github.com/jinzhu/gorm"
	"github.com/qor/qor"
	"github.com/qor/qor/utils"
	"github.com/qor/roles"
)

// ErrProcessorSkipLeft skip left processors error, if returned this error in validation, before callbacks, then qor will stop process following processors
var ErrProcessorSkipLeft = errors.New("resource: skip left")

type processor struct {
	Result     interface{}
	Resource   Resourcer
	Context    *qor.Context
	MetaValues *MetaValues
	SkipLeft   bool
	newRecord  bool
}

// DecodeToResource decode meta values to resource result
func DecodeToResource(res Resourcer, result interface{}, metaValues *MetaValues, context *qor.Context) *processor {
	scope := &gorm.Scope{Value: result}
	return &processor{Resource: res, Result: result, Context: context, MetaValues: metaValues, newRecord: scope.PrimaryKeyZero()}
}

func (processor *processor) checkSkipLeft(errs ...error) bool {
	if processor.SkipLeft {
		return true
	}

	for _, err := range errs {
		if err == ErrProcessorSkipLeft {
			processor.SkipLeft = true
			break
		}
	}
	return processor.SkipLeft
}

func (processor *processor) Initialize() error {
	err := processor.Resource.CallFindOne(processor.Result, processor.MetaValues, processor.Context)
	processor.checkSkipLeft(err)
	return err
}

func (processor *processor) Validate() error {
	var errors qor.Errors
	if processor.checkSkipLeft() {
		return nil
	}

	for _, fc := range processor.Resource.GetResource().Validators {
		if errors.AddError(fc(processor.Result, processor.MetaValues, processor.Context)); !errors.HasError() {
			if processor.checkSkipLeft(errors.GetErrors()...) {
				break
			}
		}
	}
	return errors
}

func (processor *processor) decode() (errors []error) {
	if processor.checkSkipLeft() || processor.MetaValues == nil {
		return
	}

	for _, metaValue := range processor.MetaValues.Values {
		meta := metaValue.Meta
		if meta == nil {
			continue
		}

		if processor.newRecord && !meta.HasPermission(roles.Create, processor.Context) {
			continue
		} else if !meta.HasPermission(roles.Update, processor.Context) {
			continue
		}

		if setter := meta.GetSetter(); setter != nil {
			setter(processor.Result, metaValue, processor.Context)
		}

		if res := metaValue.Meta.GetResource(); res != nil && !reflect.ValueOf(res).IsNil() {
			field := reflect.Indirect(reflect.ValueOf(processor.Result)).FieldByName(meta.GetFieldName())
			if utils.ModelType(field.Addr().Interface()) == utils.ModelType(res.NewStruct()) {
				decodeMetaValuesToField(res, field, metaValue, processor.Context)
			}
		}
	}

	return
}

func (processor *processor) Commit() error {
	var errors qor.Errors
	errors.AddError(processor.decode()...)
	if processor.checkSkipLeft(errors.GetErrors()...) {
		return nil
	}

	for _, fc := range processor.Resource.GetResource().Processors {
		if err := fc(processor.Result, processor.MetaValues, processor.Context); err != nil {
			if processor.checkSkipLeft(err) {
				break
			}
			errors.AddError(err)
		}
	}
	return errors
}

func (processor *processor) Start() error {
	var errors qor.Errors
	processor.Initialize()
	if errors.AddError(processor.Validate()); !errors.HasError() {
		errors.AddError(processor.Commit())
	}
	if errors.HasError() {
		return errors
	}
	return nil
}
<commit_msg>Only decode nested resource when has nested meta values<commit_after>package resource

import (
	"errors"
	"reflect"

	"github.com/jinzhu/gorm"
	"github.com/qor/qor"
	"github.com/qor/qor/utils"
	"github.com/qor/roles"
)

// ErrProcessorSkipLeft skip left processors error, if returned this error in validation, before callbacks, then qor will stop process following processors
var ErrProcessorSkipLeft = errors.New("resource: skip left")

type processor struct {
	Result     interface{}
	Resource   Resourcer
	Context    *qor.Context
	MetaValues *MetaValues
	SkipLeft   bool
	newRecord  bool
}

// DecodeToResource decode meta values to resource result
func DecodeToResource(res Resourcer, result interface{}, metaValues *MetaValues, context *qor.Context) *processor {
	scope := &gorm.Scope{Value: result}
	return &processor{Resource: res, Result: result, Context: context, MetaValues: metaValues, newRecord: scope.PrimaryKeyZero()}
}

func (processor *processor) checkSkipLeft(errs ...error) bool {
	if processor.SkipLeft {
		return true
	}

	for _, err := range errs {
		if err == ErrProcessorSkipLeft {
			processor.SkipLeft = true
			break
		}
	}
	return processor.SkipLeft
}

func (processor *processor) Initialize() error {
	err := processor.Resource.CallFindOne(processor.Result, processor.MetaValues, processor.Context)
	processor.checkSkipLeft(err)
	return err
}

func (processor *processor) Validate() error {
	var errors qor.Errors
	if processor.checkSkipLeft() {
		return nil
	}

	for _, fc := range processor.Resource.GetResource().Validators {
		if errors.AddError(fc(processor.Result, processor.MetaValues, processor.Context)); !errors.HasError() {
			if processor.checkSkipLeft(errors.GetErrors()...) {
				break
			}
		}
	}
	return errors
}

func (processor *processor) decode() (errors []error) {
	if processor.checkSkipLeft() || processor.MetaValues == nil {
		return
	}

	for _, metaValue := range processor.MetaValues.Values {
		meta := metaValue.Meta
		if meta == nil {
			continue
		}

		if processor.newRecord && !meta.HasPermission(roles.Create, processor.Context) {
			continue
		} else if !meta.HasPermission(roles.Update, processor.Context) {
			continue
		}

		if setter := meta.GetSetter(); setter != nil {
			setter(processor.Result, metaValue, processor.Context)
		}

		if metaValue.MetaValues != nil && len(metaValue.MetaValues.Values) > 0 {
			if res := metaValue.Meta.GetResource(); res != nil && !reflect.ValueOf(res).IsNil() {
				field := reflect.Indirect(reflect.ValueOf(processor.Result)).FieldByName(meta.GetFieldName())
				if utils.ModelType(field.Addr().Interface()) == utils.ModelType(res.NewStruct()) {
					decodeMetaValuesToField(res, field, metaValue, processor.Context)
				}
			}
		}
	}

	return
}

func (processor *processor) Commit() error {
	var errors qor.Errors
	errors.AddError(processor.decode()...)
	if processor.checkSkipLeft(errors.GetErrors()...) {
		return nil
	}

	for _, fc := range processor.Resource.GetResource().Processors {
		if err := fc(processor.Result, processor.MetaValues, processor.Context); err != nil {
			if processor.checkSkipLeft(err) {
				break
			}
			errors.AddError(err)
		}
	}
	return errors
}

func (processor *processor) Start() error {
	var errors qor.Errors
	processor.Initialize()
	if errors.AddError(processor.Validate()); !errors.HasError() {
		errors.AddError(processor.Commit())
	}
	if errors.HasError() {
		return errors
	}
	return nil
}
<|endoftext|>


<commit_before>package jsonpointer

import (
	"errors"
	"strconv"
	"strings"

	"github.com/dustin/gojson"
)

var unparsable = errors.New("I can't parse this")

func arreq(a, b []string) bool {
	if len(a) == len(b) {
		for i := range a {
			if a[i] != b[i] {
				return false
			}
		}
		return true
	}

	return false
}

func parsePointer(s string) []string {
	a := strings.Split(s[1:], "/")

	for i := range a {
		if strings.Contains(a[i], "~") {
			a[i] = strings.Replace(a[i], "~1", "/", -1)
			a[i] = strings.Replace(a[i], "~0", "~", -1)
		}
	}
	return a
}

func encodePointer(p []string) string {
	a := make([]string, 0, len(p))
	for _, s := range p {
		s = strings.Replace(s, "~", "~0", -1)
		s = strings.Replace(s, "/", "~1", -1)
		a = append(a, s)
	}
	return "/" + strings.Join(a, "/")
}

func grokLiteral(b []byte) string {
	s, ok := json.UnquoteBytes(b)
	if !ok {
		panic("could not grok literal " + string(b))
	}
	return string(s)
}

// Find a section of raw JSON by specifying a JSONPointer.
func Find(data []byte, path string) ([]byte, error) {
	if path == "" {
		return data, nil
	}

	needle := parsePointer(path)

	scanner := &json.Scanner{}
	scanner.Reset()

	offset := 0
	beganLiteral := 0
	current := []string{}
	for {
		var newOp int
		if offset >= len(data) {
			newOp = scanner.EOF()
			break
			offset = len(data) + 1 // mark processed EOF with len+1
		} else {
			c := int(data[offset])
			offset++
			newOp = scanner.Step(scanner, c)
		}

		switch newOp {
		case json.ScanBeginArray:
			current = append(current, "0")
		case json.ScanObjectKey:
			current = append(current, grokLiteral(data[beganLiteral-1:offset-1]))
		case json.ScanBeginLiteral:
			beganLiteral = offset
		case json.ScanArrayValue:
			n, err := strconv.Atoi(current[len(current)-1])
			if err != nil {
				return nil, err
			}
			current[len(current)-1] = strconv.Itoa(n + 1)
		case json.ScanObjectValue:
			current = current[:len(current)-1]
		case json.ScanEndArray:
			current = current[:len(current)-1]
		}

		if (newOp == json.ScanBeginArray || newOp == json.ScanArrayValue ||
			newOp == json.ScanObjectKey) && arreq(needle, current) {
			val, _, err := json.NextValue(data[offset:], scanner)
			return val, err
		}
	}

	return nil, nil
}

// Find a section of raw JSON by specifying a JSONPointer.
func FindMany(data []byte, paths []string) (map[string][]byte, error) {
	todo := map[string]bool{}
	m := map[string][]byte{}
	for _, p := range paths {
		if p == "" {
			m[p] = data
		} else {
			todo[p] = true
		}
	}

	scan := &json.Scanner{}
	scan.Reset()

	offset := 0
	beganLiteral := 0
	var current []string
	currentStr := ""
	for {
		var newOp int
		if offset >= len(data) {
			newOp = scan.EOF()
			break
			offset = len(data) + 1 // mark processed EOF with len+1
		} else {
			c := int(data[offset])
			offset++
			newOp = scan.Step(scan, c)
		}

		switch newOp {
		case json.ScanBeginArray:
			current = append(current, "0")
			currentStr = encodePointer(current)
		case json.ScanObjectKey:
			current = append(current, grokLiteral(data[beganLiteral-1:offset-1]))
			currentStr = encodePointer(current)
		case json.ScanBeginLiteral:
			beganLiteral = offset
		case json.ScanArrayValue:
			n, err := strconv.Atoi(current[len(current)-1])
			if err != nil {
				return nil, err
			}
			current[len(current)-1] = strconv.Itoa(n + 1)
			currentStr = encodePointer(current)
		case json.ScanObjectValue, json.ScanEndArray, json.ScanEndObject:
			current = current[:len(current)-1]
			currentStr = encodePointer(current)
		}

		if (newOp == json.ScanBeginArray || newOp == json.ScanArrayValue ||
			newOp == json.ScanObjectKey) && todo[currentStr] {

			stmp := &json.Scanner{}
			val, _, err := json.NextValue(data[offset:], stmp)
			if err != nil {
				return m, err
			}
			m[currentStr] = val
			delete(todo, currentStr)
			if len(todo) == 0 {
				return m, nil
			}
		}
	}

	return m, nil
}
<commit_msg>perf: don't encode currentStr unless we need it.<commit_after>package jsonpointer

import (
	"errors"
	"strconv"
	"strings"

	"github.com/dustin/gojson"
)

var unparsable = errors.New("I can't parse this")

func arreq(a, b []string) bool {
	if len(a) == len(b) {
		for i := range a {
			if a[i] != b[i] {
				return false
			}
		}
		return true
	}

	return false
}

func parsePointer(s string) []string {
	a := strings.Split(s[1:], "/")

	for i := range a {
		if strings.Contains(a[i], "~") {
			a[i] = strings.Replace(a[i], "~1", "/", -1)
			a[i] = strings.Replace(a[i], "~0", "~", -1)
		}
	}
	return a
}

func encodePointer(p []string) string {
	a := make([]string, 0, len(p))
	for _, s := range p {
		s = strings.Replace(s, "~", "~0", -1)
		s = strings.Replace(s, "/", "~1", -1)
		a = append(a, s)
	}
	return "/" + strings.Join(a, "/")
}

func grokLiteral(b []byte) string {
	s, ok := json.UnquoteBytes(b)
	if !ok {
		panic("could not grok literal " + string(b))
	}
	return string(s)
}

// Find a section of raw JSON by specifying a JSONPointer.
func Find(data []byte, path string) ([]byte, error) {
	if path == "" {
		return data, nil
	}

	needle := parsePointer(path)

	scanner := &json.Scanner{}
	scanner.Reset()

	offset := 0
	beganLiteral := 0
	current := []string{}
	for {
		var newOp int
		if offset >= len(data) {
			newOp = scanner.EOF()
			break
			offset = len(data) + 1 // mark processed EOF with len+1
		} else {
			c := int(data[offset])
			offset++
			newOp = scanner.Step(scanner, c)
		}

		switch newOp {
		case json.ScanBeginArray:
			current = append(current, "0")
		case json.ScanObjectKey:
			current = append(current, grokLiteral(data[beganLiteral-1:offset-1]))
		case json.ScanBeginLiteral:
			beganLiteral = offset
		case json.ScanArrayValue:
			n, err := strconv.Atoi(current[len(current)-1])
			if err != nil {
				return nil, err
			}
			current[len(current)-1] = strconv.Itoa(n + 1)
		case json.ScanObjectValue:
			current = current[:len(current)-1]
		case json.ScanEndArray:
			current = current[:len(current)-1]
		}

		if (newOp == json.ScanBeginArray || newOp == json.ScanArrayValue ||
			newOp == json.ScanObjectKey) && arreq(needle, current) {
			val, _, err := json.NextValue(data[offset:], scanner)
			return val, err
		}
	}

	return nil, nil
}

// Find a section of raw JSON by specifying a JSONPointer.
func FindMany(data []byte, paths []string) (map[string][]byte, error) {
	todo := map[string]bool{}
	m := map[string][]byte{}
	for _, p := range paths {
		if p == "" {
			m[p] = data
		} else {
			todo[p] = true
		}
	}

	scan := &json.Scanner{}
	scan.Reset()

	offset := 0
	beganLiteral := 0
	var current []string
	for {
		var newOp int
		if offset >= len(data) {
			newOp = scan.EOF()
			break
			offset = len(data) + 1 // mark processed EOF with len+1
		} else {
			c := int(data[offset])
			offset++
			newOp = scan.Step(scan, c)
		}

		switch newOp {
		case json.ScanBeginArray:
			current = append(current, "0")
		case json.ScanObjectKey:
			current = append(current, grokLiteral(data[beganLiteral-1:offset-1]))
		case json.ScanBeginLiteral:
			beganLiteral = offset
		case json.ScanArrayValue:
			n, err := strconv.Atoi(current[len(current)-1])
			if err != nil {
				return nil, err
			}
			current[len(current)-1] = strconv.Itoa(n + 1)
		case json.ScanObjectValue, json.ScanEndArray, json.ScanEndObject:
			current = current[:len(current)-1]
		}

		if newOp == json.ScanBeginArray || newOp == json.ScanArrayValue ||
			newOp == json.ScanObjectKey {
			currentStr := encodePointer(current)
			if !todo[currentStr] {
				continue
			}

			stmp := &json.Scanner{}
			val, _, err := json.NextValue(data[offset:], stmp)
			if err != nil {
				return m, err
			}
			m[currentStr] = val
			delete(todo, currentStr)
			if len(todo) == 0 {
				return m, nil
			}
		}
	}

	return m, nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2013 Conformal Systems LLC.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package btcwire

import (
	"crypto/rand"
	"crypto/sha256"
	"encoding/binary"
	"fmt"
	"io"
	"math"
)

// Maximum payload size for a variable length integer.
const maxVarIntPayload = 9

// readElement reads the next sequence of bytes from r using little endian
// depending on the concrete type of element pointed to.
func readElement(r io.Reader, element interface{}) error {
	return binary.Read(r, binary.LittleEndian, element)
}

// readElements reads multiple items from r.  It is equivalent to multiple
// calls to readElement.
func readElements(r io.Reader, elements ...interface{}) error {
	for _, element := range elements {
		err := readElement(r, element)
		if err != nil {
			return err
		}
	}
	return nil
}

// writeElement writes the little endian representation of element to w.
func writeElement(w io.Writer, element interface{}) error {
	return binary.Write(w, binary.LittleEndian, element)
}

// writeElements writes multiple items to w.  It is equivalent to multiple
// calls to writeElement.
func writeElements(w io.Writer, elements ...interface{}) error {
	for _, element := range elements {
		err := writeElement(w, element)
		if err != nil {
			return err
		}
	}
	return nil
}

// readVarInt reads a variable length integer from r and returns it as a uint64.
func readVarInt(r io.Reader, pver uint32) (uint64, error) {
	b := make([]byte, 8)
	_, err := io.ReadFull(r, b[0:1])
	if err != nil {
		return 0, err
	}

	var rv uint64
	discriminant := uint8(b[0])
	switch discriminant {
	case 0xff:
		_, err := io.ReadFull(r, b)
		if err != nil {
			return 0, err
		}
		rv = binary.LittleEndian.Uint64(b)

	case 0xfe:
		_, err := io.ReadFull(r, b[0:4])
		if err != nil {
			return 0, err
		}
		rv = uint64(binary.LittleEndian.Uint32(b))

	case 0xfd:
		_, err := io.ReadFull(r, b[0:2])
		if err != nil {
			return 0, err
		}
		rv = uint64(binary.LittleEndian.Uint16(b))

	default:
		rv = uint64(discriminant)
	}

	return rv, nil
}

// writeVarInt serializes val to w using a variable number of bytes depending
// on its value.
func writeVarInt(w io.Writer, pver uint32, val uint64) error {
	if val < 0xfd {
		_, err := w.Write([]byte{uint8(val)})
		return err
	}

	if val <= math.MaxUint16 {
		buf := make([]byte, 3)
		buf[0] = 0xfd
		binary.LittleEndian.PutUint16(buf[1:], uint16(val))
		_, err := w.Write(buf)
		return err
	}

	if val <= math.MaxUint32 {
		buf := make([]byte, 5)
		buf[0] = 0xfe
		binary.LittleEndian.PutUint32(buf[1:], uint32(val))
		_, err := w.Write(buf)
		return err
	}

	buf := make([]byte, 9)
	buf[0] = 0xff
	binary.LittleEndian.PutUint64(buf[1:], val)
	_, err := w.Write(buf)
	return err
}

// varIntSerializeSize returns the number of bytes it would take to serialize
// val as a variable length integer.
func varIntSerializeSize(val uint64) int {
	// The value is small enough to be represented by itself, so it's
	// just 1 byte.
	if val < 0xfd {
		return 1
	}

	// Discriminant 1 byte plus 2 bytes for the uint16.
	if val <= math.MaxUint16 {
		return 3
	}

	// Discriminant 1 byte plus 4 bytes for the uint32.
	if val <= math.MaxUint32 {
		return 5
	}

	// Discriminant 1 byte plus 8 bytes for the uint64.
	return 9
}

// readVarString reads a variable length string from r and returns it as a Go
// string.  A varString is encoded as a varInt containing the length of the
// string, and the bytes that represent the string itself.  An error is returned
// if the length is greater than the maximum block payload size, since it would
// not be possible to put a varString of that size into a block anyways and it
// also helps protect against memory exhuastion attacks and forced panics
// through malformed messages.
func readVarString(r io.Reader, pver uint32) (string, error) {
	count, err := readVarInt(r, pver)
	if err != nil {
		return "", err
	}

	// Prevent variable length strings that are larger than the maximum
	// message size.  It would be possible to cause memory exhaustion and
	// panics without a sane upper bound on this count.
	if count > maxMessagePayload {
		str := fmt.Sprintf("variable length string is too long "+
			"[count %d, max %d]", count, maxMessagePayload)
		return "", messageError("readVarString", str)
	}

	buf := make([]byte, count)
	_, err = io.ReadFull(r, buf)
	if err != nil {
		return "", err
	}
	return string(buf), nil
}

// writeVarString serializes str to w as a varInt containing the length of the
// string followed by the bytes that represent the string itself.
func writeVarString(w io.Writer, pver uint32, str string) error {
	err := writeVarInt(w, pver, uint64(len(str)))
	if err != nil {
		return err
	}
	err = writeElement(w, []byte(str))
	if err != nil {
		return err
	}
	return nil
}

// randomUint64 returns a cryptographically random uint64 value.  This
// unexported version takes a reader primarily to ensure the error paths
// can be properly tested by passing a fake reader in the tests.
func randomUint64(r io.Reader) (uint64, error) {
	b := make([]byte, 8)
	n, err := r.Read(b)
	if n != len(b) {
		return 0, io.ErrShortBuffer
	}
	if err != nil {
		return 0, err
	}
	return binary.BigEndian.Uint64(b), nil
}

// RandomUint64 returns a cryptographically random uint64 value.
func RandomUint64() (uint64, error) {
	return randomUint64(rand.Reader)
}

// DoubleSha256 calculates sha256(sha256(b)) and returns the resulting bytes.
func DoubleSha256(b []byte) []byte {
	hasher := sha256.New()
	hasher.Write(b)
	sum := hasher.Sum(nil)
	hasher.Reset()
	hasher.Write(sum)
	return hasher.Sum(nil)
}
<commit_msg>Optimize writeVarString.<commit_after>// Copyright (c) 2013 Conformal Systems LLC.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package btcwire

import (
	"crypto/rand"
	"crypto/sha256"
	"encoding/binary"
	"fmt"
	"io"
	"math"
)

// Maximum payload size for a variable length integer.
const maxVarIntPayload = 9

// readElement reads the next sequence of bytes from r using little endian
// depending on the concrete type of element pointed to.
func readElement(r io.Reader, element interface{}) error {
	return binary.Read(r, binary.LittleEndian, element)
}

// readElements reads multiple items from r.  It is equivalent to multiple
// calls to readElement.
func readElements(r io.Reader, elements ...interface{}) error {
	for _, element := range elements {
		err := readElement(r, element)
		if err != nil {
			return err
		}
	}
	return nil
}

// writeElement writes the little endian representation of element to w.
func writeElement(w io.Writer, element interface{}) error {
	return binary.Write(w, binary.LittleEndian, element)
}

// writeElements writes multiple items to w.  It is equivalent to multiple
// calls to writeElement.
func writeElements(w io.Writer, elements ...interface{}) error {
	for _, element := range elements {
		err := writeElement(w, element)
		if err != nil {
			return err
		}
	}
	return nil
}

// readVarInt reads a variable length integer from r and returns it as a uint64.
func readVarInt(r io.Reader, pver uint32) (uint64, error) {
	b := make([]byte, 8)
	_, err := io.ReadFull(r, b[0:1])
	if err != nil {
		return 0, err
	}

	var rv uint64
	discriminant := uint8(b[0])
	switch discriminant {
	case 0xff:
		_, err := io.ReadFull(r, b)
		if err != nil {
			return 0, err
		}
		rv = binary.LittleEndian.Uint64(b)

	case 0xfe:
		_, err := io.ReadFull(r, b[0:4])
		if err != nil {
			return 0, err
		}
		rv = uint64(binary.LittleEndian.Uint32(b))

	case 0xfd:
		_, err := io.ReadFull(r, b[0:2])
		if err != nil {
			return 0, err
		}
		rv = uint64(binary.LittleEndian.Uint16(b))

	default:
		rv = uint64(discriminant)
	}

	return rv, nil
}

// writeVarInt serializes val to w using a variable number of bytes depending
// on its value.
func writeVarInt(w io.Writer, pver uint32, val uint64) error {
	if val < 0xfd {
		_, err := w.Write([]byte{uint8(val)})
		return err
	}

	if val <= math.MaxUint16 {
		buf := make([]byte, 3)
		buf[0] = 0xfd
		binary.LittleEndian.PutUint16(buf[1:], uint16(val))
		_, err := w.Write(buf)
		return err
	}

	if val <= math.MaxUint32 {
		buf := make([]byte, 5)
		buf[0] = 0xfe
		binary.LittleEndian.PutUint32(buf[1:], uint32(val))
		_, err := w.Write(buf)
		return err
	}

	buf := make([]byte, 9)
	buf[0] = 0xff
	binary.LittleEndian.PutUint64(buf[1:], val)
	_, err := w.Write(buf)
	return err
}

// varIntSerializeSize returns the number of bytes it would take to serialize
// val as a variable length integer.
func varIntSerializeSize(val uint64) int {
	// The value is small enough to be represented by itself, so it's
	// just 1 byte.
	if val < 0xfd {
		return 1
	}

	// Discriminant 1 byte plus 2 bytes for the uint16.
	if val <= math.MaxUint16 {
		return 3
	}

	// Discriminant 1 byte plus 4 bytes for the uint32.
	if val <= math.MaxUint32 {
		return 5
	}

	// Discriminant 1 byte plus 8 bytes for the uint64.
	return 9
}

// readVarString reads a variable length string from r and returns it as a Go
// string.  A varString is encoded as a varInt containing the length of the
// string, and the bytes that represent the string itself.  An error is returned
// if the length is greater than the maximum block payload size, since it would
// not be possible to put a varString of that size into a block anyways and it
// also helps protect against memory exhuastion attacks and forced panics
// through malformed messages.
func readVarString(r io.Reader, pver uint32) (string, error) {
	count, err := readVarInt(r, pver)
	if err != nil {
		return "", err
	}

	// Prevent variable length strings that are larger than the maximum
	// message size.  It would be possible to cause memory exhaustion and
	// panics without a sane upper bound on this count.
	if count > maxMessagePayload {
		str := fmt.Sprintf("variable length string is too long "+
			"[count %d, max %d]", count, maxMessagePayload)
		return "", messageError("readVarString", str)
	}

	buf := make([]byte, count)
	_, err = io.ReadFull(r, buf)
	if err != nil {
		return "", err
	}
	return string(buf), nil
}

// writeVarString serializes str to w as a varInt containing the length of the
// string followed by the bytes that represent the string itself.
func writeVarString(w io.Writer, pver uint32, str string) error {
	err := writeVarInt(w, pver, uint64(len(str)))
	if err != nil {
		return err
	}
	_, err = w.Write([]byte(str))
	if err != nil {
		return err
	}
	return nil
}

// randomUint64 returns a cryptographically random uint64 value.  This
// unexported version takes a reader primarily to ensure the error paths
// can be properly tested by passing a fake reader in the tests.
func randomUint64(r io.Reader) (uint64, error) {
	b := make([]byte, 8)
	n, err := r.Read(b)
	if n != len(b) {
		return 0, io.ErrShortBuffer
	}
	if err != nil {
		return 0, err
	}
	return binary.BigEndian.Uint64(b), nil
}

// RandomUint64 returns a cryptographically random uint64 value.
func RandomUint64() (uint64, error) {
	return randomUint64(rand.Reader)
}

// DoubleSha256 calculates sha256(sha256(b)) and returns the resulting bytes.
func DoubleSha256(b []byte) []byte {
	hasher := sha256.New()
	hasher.Write(b)
	sum := hasher.Sum(nil)
	hasher.Reset()
	hasher.Write(sum)
	return hasher.Sum(nil)
}
<|endoftext|>


<commit_before>package crawler

import (
	"io"
	"log"
	"os"
	"strings"

	"golang.org/x/net/html"
)

func parseHTML(body io.Reader, baseURL string) ([]string, error) {
	var result []string

	doc, err := html.Parse(body)
	if err != nil {
		return nil, err
	}

	var f func(*html.Node)
	f = func(node *html.Node) {
		if node.Type == html.ElementNode && node.Data == "a" {
			for _, v := range node.Attr {
				if v.Key == "href" {
					if strings.Contains(v.Val, "javascript:") {
						break
					}
					if strings.HasPrefix(v.Val, "http") {
						result = append(result, v.Val)
					} else {
						result = append(result, baseURL+v.Val)
					}
					break
				}
			}
		}

		for c := node.FirstChild; c != nil; c = c.NextSibling {
			f(c)
		}
	}

	f(doc)

	return result, nil
}
<commit_msg>Fix compilation<commit_after>package crawler

import (
	"io"
	"strings"

	"golang.org/x/net/html"
)

func parseHTML(body io.Reader, baseURL string) ([]string, error) {
	var result []string

	doc, err := html.Parse(body)
	if err != nil {
		return nil, err
	}

	var f func(*html.Node)
	f = func(node *html.Node) {
		if node.Type == html.ElementNode && node.Data == "a" {
			for _, v := range node.Attr {
				if v.Key == "href" {
					if strings.Contains(v.Val, "javascript:") {
						break
					}
					if strings.HasPrefix(v.Val, "http") {
						result = append(result, v.Val)
					} else {
						result = append(result, baseURL+v.Val)
					}
					break
				}
			}
		}

		for c := node.FirstChild; c != nil; c = c.NextSibling {
			f(c)
		}
	}

	f(doc)

	return result, nil
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"fmt"
	"net/http"
	"net/http/httptest"
	"testing"

	"github.com/coopernurse/gorp"
	. "gopkg.in/check.v1"
)

func createTestDatabase() *gorp.DbMap {
	db, err := sql.Open("sqlite3", ":memory:")
	checkErr(err, "sql.Open failed")

	// construct a gorp DbMap
	dbmap := &gorp.DbMap{Db: db, Dialect: gorp.SqliteDialect{}}

	return dbmap
}

func Test(t *testing.T) { TestingT(t) }

type ElectionSuite struct {
	dbmap *gorp.DbMap
}

var _ = Suite(&ElectionSuite{})

func (s *ElectionSuite) SetUpSuite(c *C) {
	s.dbmap = createTestDatabase()
}

func (s *ElectionSuite) SetUpTest(c *C) {
	err := s.dbmap.TruncateTables()
	if err != nil {
		c.Error(err)
	}
}

func (s *ElectionSuite) TearDownSuite(c *C) {
	s.dbmap.Db.Close()
}

func (s *ElectionSuite) PerformRequest(method string, relativePath string) *httptest.ResponseRecorder {
	path := fmt.Sprintf("http://test.example.com%s", relativePath)
	w := httptest.NewRecorder()
	r, err := http.NewRequest(method, path, nil)
	checkErr(err, "Request creation failed")

	testApp := CreateApplication(s.dbmap)
	testApp.router.ServeHTTP(w, r)
	return w
}

func (s *ElectionSuite) TestAddElectionCreatesOneElection(c *C) {
	s.PerformRequest("POST", "/elections")

	count, err := s.dbmap.SelectInt("select count(*) from elections")
	checkErr(err, "Getting count failed")
	c.Assert(count, Equals, int64(1))
}

func (s *ElectionSuite) TestAddElectionCreatesElectionWithCorrectName(c *C) {
	s.PerformRequest("POST", "/elections")

	var createdElection Election
	err := s.dbmap.SelectOne(&createdElection, "select * from elections")
	if err != nil {
		c.Error(err)
	}
	c.Assert(createdElection.Name, Matches, "foo")
}
<commit_msg>Add test for getting elections.<commit_after>package main

import (
	"database/sql"
	"fmt"
	"net/http"
	"net/http/httptest"
	"testing"

	"github.com/coopernurse/gorp"
	. "gopkg.in/check.v1"
)

func createTestDatabase() *gorp.DbMap {
	db, err := sql.Open("sqlite3", ":memory:")
	checkErr(err, "sql.Open failed")

	// construct a gorp DbMap
	dbmap := &gorp.DbMap{Db: db, Dialect: gorp.SqliteDialect{}}

	return dbmap
}

func Test(t *testing.T) { TestingT(t) }

type ElectionSuite struct {
	dbmap *gorp.DbMap
}

var _ = Suite(&ElectionSuite{})

func (s *ElectionSuite) SetUpSuite(c *C) {
	s.dbmap = createTestDatabase()
}

func (s *ElectionSuite) SetUpTest(c *C) {
	err := s.dbmap.TruncateTables()
	if err != nil {
		c.Error(err)
	}
}

func (s *ElectionSuite) TearDownSuite(c *C) {
	s.dbmap.Db.Close()
}

func (s *ElectionSuite) PerformRequest(method string, relativePath string) *httptest.ResponseRecorder {
	path := fmt.Sprintf("http://test.example.com%s", relativePath)
	w := httptest.NewRecorder()
	r, err := http.NewRequest(method, path, nil)
	checkErr(err, "Request creation failed")

	testApp := CreateApplication(s.dbmap)
	testApp.router.ServeHTTP(w, r)
	return w
}

func (s *ElectionSuite) TestAddElectionCreatesOneElection(c *C) {
	s.PerformRequest("POST", "/elections")

	count, err := s.dbmap.SelectInt("select count(*) from elections")
	checkErr(err, "Getting count failed")
	c.Assert(count, Equals, int64(1))
}

func (s *ElectionSuite) TestAddElectionCreatesElectionWithCorrectName(c *C) {
	s.PerformRequest("POST", "/elections")

	var createdElection Election
	err := s.dbmap.SelectOne(&createdElection, "select * from elections")
	if err != nil {
		c.Error(err)
	}
	c.Assert(createdElection.Name, Matches, "foo")
}

func (s *ElectionSuite) TestGetElectionReturnsElectionName(c *C) {
	election := Election{Name: "my test name"}
	s.dbmap.Insert(&election)

	recorder := s.PerformRequest("GET", fmt.Sprintf("/elections/%d", election.Id))
	c.Assert(recorder.Body.String(), Matches, "my test name")
}
<|endoftext|>


<commit_before>package scipipe

import (
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
)

// Process is the central component in SciPipe after Workflow. Processes are
// long-running "services" that schedules and executes Tasks based on the IPs
// and parameters received on its in-ports and parameter ports
type Process struct {
	BaseProcess
	CommandPattern string
	PathFuncs      map[string]func(*Task) string
	CustomExecute  func(*Task)
	CoresPerTask   int
	Prepend        string
	Spawn          bool
	PortInfo       map[string]*PortInfo
}

// ------------------------------------------------------------------------
// Factory method(s)
// ------------------------------------------------------------------------

// NewProc returns a new Process, and initializes its ports based on the
// command pattern.
func NewProc(workflow *Workflow, name string, cmd string) *Process {
	p := &Process{
		BaseProcess: NewBaseProcess(
			workflow,
			name,
		),
		CommandPattern: cmd,
		PathFuncs:      make(map[string]func(*Task) string),
		Spawn:          true,
		CoresPerTask:   1,
		PortInfo:       map[string]*PortInfo{},
	}
	workflow.AddProc(p)
	p.initPortsFromCmdPattern(cmd, nil)
	p.initDefaultPathFuncs()
	return p
}

// PortInfo is a container for various information about process ports
type PortInfo struct {
	portType  string
	extension string
	doStream  bool
	join      bool
	joinSep   string
}

// initPortsFromCmdPattern is a helper function for NewProc, that sets up in-
// and out-ports based on the shell command pattern used to create the Process.
// Ports are set up in this way:
// `{i:PORTNAME}` specifies an in-port
// `{o:PORTNAME}` specifies an out-port
// `{os:PORTNAME}` specifies an out-port that streams via a FIFO file
// `{p:PORTNAME}` a "parameter (in-)port", which means a port where parameters can be "streamed"
func (p *Process) initPortsFromCmdPattern(cmd string, params map[string]string) {
	// Find in/out port names and params and set up ports
	r := getShellCommandPlaceHolderRegex()
	ms := r.FindAllStringSubmatch(cmd, -1)

	for _, m := range ms {
		portType := m[1]
		portRest := m[2]
		splitParts := strings.Split(portRest, "|")
		portName := splitParts[0]

		p.PortInfo[portName] = &PortInfo{portType: portType}

		for _, part := range splitParts[1:] {
			// If the |-separated part starts with a dot, treat it as a
			// configuration for file extenion to use
			fileExtPtn := regexp.MustCompile("\\.([a-z0-9\\.\\-\\_]+)")
			if fileExtPtn.MatchString(part) {
				m := fileExtPtn.FindStringSubmatch(part)
				p.PortInfo[portName].extension = m[1]
			}
			// If the |-separated part starts with "join:"
			// then treat the character following that as the character to use
			// when joining multiple files received on a sub-stream
			joinPtn := regexp.MustCompile("join:([^{}|]+)")
			if joinPtn.MatchString(part) {
				m := joinPtn.FindStringSubmatch(part)
				p.PortInfo[portName].join = true
				p.PortInfo[portName].joinSep = m[1]
			}
		}
	}

	for portName, pInfo := range p.PortInfo {
		if pInfo.portType == "o" || pInfo.portType == "os" {
			p.InitOutPort(p, portName)
			if pInfo.portType == "os" {
				p.PortInfo[portName].doStream = true
			}
		} else if pInfo.portType == "i" {
			p.InitInPort(p, portName)
		} else if pInfo.portType == "p" {
			if params == nil {
				p.InitInParamPort(p, portName)
			} else if _, ok := params[portName]; !ok {
				p.InitInParamPort(p, portName)
			}
		}
	}
}

// initDefaultPathFuncs does exactly what it name says: Initializes default
// path formatters for processes, that is used if no explicit path is set, using
// the proc.SetPath[...] methods
func (p *Process) initDefaultPathFuncs() {
	for outName := range p.OutPorts() {
		outName := outName
		p.PathFuncs[outName] = func(t *Task) string {
			pathPcs := []string{}
			for _, ipName := range sortedFileIPMapKeys(t.InIPs) {
				pathPcs = append(pathPcs, filepath.Base(t.InIP(ipName).Path()))
			}
			procName := sanitizePathFragment(t.Process.Name())
			pathPcs = append(pathPcs, procName)
			for _, paramName := range sortedStringMapKeys(t.Params) {
				pathPcs = append(pathPcs, paramName+"_"+t.Param(paramName))
			}
			for _, tagName := range sortedStringMapKeys(t.Tags) {
				pathPcs = append(pathPcs, tagName+"_"+t.Tag(tagName))
			}
			pathPcs = append(pathPcs, outName)
			fileExt := p.PortInfo[outName].extension
			if fileExt != "" {
				pathPcs = append(pathPcs, fileExt)
			}
			return strings.Join(pathPcs, ".")
		}
	}
}

func sortedFileIPMapKeys(kv map[string]*FileIP) []string {
	keys := []string{}
	for k := range kv {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	return keys
}

func sortedStringMapKeys(kv map[string]string) []string {
	keys := []string{}
	for k := range kv {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	return keys
}

// ------------------------------------------------------------------------
// Main API methods: Port accessor methods
// ------------------------------------------------------------------------

// In is a short-form for InPort() (of BaseProcess), which works only on Process
// processes
func (p *Process) In(portName string) *InPort {
	if portName == "" && len(p.InPorts()) == 1 {
		for _, inPort := range p.InPorts() {
			return inPort // Return the (only) in-port available
		}
	}
	return p.InPort(portName)
}

// Out is a short-form for OutPort() (of BaseProcess), which works only on
// Process processes
func (p *Process) Out(portName string) *OutPort {
	if portName == "" && len(p.OutPorts()) == 1 {
		for _, outPort := range p.OutPorts() {
			return outPort // Return the (only) out-port available
		}
	}
	return p.OutPort(portName)
}

// InParam is a short-form for InParamPort() (of BaseProcess), which works only on Process
// processes
func (p *Process) InParam(portName string) *InParamPort {
	if _, ok := p.inParamPorts[portName]; !ok {
		p.InitInParamPort(p, portName)
	}
	return p.InParamPort(portName)
}

// OutParam is a short-form for OutParamPort() (of BaseProcess), which works only on
// Process processes
func (p *Process) OutParam(portName string) *OutParamPort {
	return p.OutParamPort(portName)
}

// ------------------------------------------------------------------------
// Main API methods: Configure path formatting
// ------------------------------------------------------------------------

// SetOut initializes a port (if it does not already exist), and takes a
// configuration for its outputs paths via a pattern similar to the command
// pattern used to create new processes, with placeholder tags. Available
// placeholder tags to use are:
// {i:inport_name}
// {p:param_name}
// {t:tag_name}
// An example might be: {i:foo}.replace_with_{p:replacement}.txt
// ... given that the process contains an in-port named 'foo', and a parameter
// named 'replacement'.
// If an out-port with the specified name does not exist, it will be created.
// This allows to create out-ports for filenames that are created without explicitly
// stating a filename on the commandline, such as when only submitting a prefix.
func (p *Process) SetOut(outPortName string, pathPattern string) {
	p.SetOutFunc(outPortName, func(t *Task) string {
		path := pathPattern // Avoiding reusing the same variable in multiple instances of this func

		r := getShellCommandPlaceHolderRegex()
		matches := r.FindAllStringSubmatch(path, -1)
		for _, match := range matches {
			var replacement string

			placeHolder := match[0]
			phType := match[1]
			restMatch := match[2]

			parts := strings.Split(restMatch, "|")
			portName := parts[0]
			restParts := parts[1:]

			switch phType {
			case "i":
				replacement = t.InPath(portName)
			case "o":
				if _, ok := t.Process.PathFuncs[portName]; !ok {
					Fail("No such pathfunc for out-port " + portName + " in task " + t.Name)
				}
				replacement = t.Process.PathFuncs[portName](t)
			case "p":
				replacement = t.Param(portName)
			case "t":
				replacement = t.Tag(portName)
			default:
				Fail("Replace failed for placeholder ", portName, " for path patterh '", path, "'")
			}

			if len(restParts) > 0 {
				substPtn := regexp.MustCompile("s\\/([^\\/]+)\\/([^\\/]*)\\/")
				trimEndPtn := regexp.MustCompile("%(.*)")
				basenamePtn := regexp.MustCompile(`.*\/`)

				for _, restPart := range restParts {
					// If the |-separated part looks like a search/replace
					// pattern on the form of: s/SEARCHSTRING/REPLACESTRING/
					// then execute the replacement.
					if substPtn.MatchString(restPart) {
						mbits := substPtn.FindStringSubmatch(restPart)
						search := mbits[1]
						replace := mbits[2]
						replacement = strings.Replace(replacement, search, replace, 1)
					}
					// If the |-separated part starts iwth a %-character, then
					// trim everything following that character, from the end
					// of the path.
					if trimEndPtn.MatchString(restPart) {
						mbits := trimEndPtn.FindStringSubmatch(restPart)
						end := mbits[1]
						if end == replacement[len(replacement)-len(end):] {
							replacement = replacement[:len(replacement)-len(end)]
						}
					}
					// If the |-separated part is "basename", then remove all leading
					// folders up to the actual file name.
					if restPart == "basename" {
						replacement = basenamePtn.ReplaceAllString(replacement, "")
					}
				}
			}

			// Replace placeholder with concrete value
			path = strings.Replace(path, placeHolder, replacement, -1)
		}
		return path
	})
}

// SetOutFunc takes a function which produces a file path based on data
// available in *Task, such as concrete file paths and parameter values,
func (p *Process) SetOutFunc(outPortName string, pathFmtFunc func(task *Task) (path string)) {
	if _, ok := p.outPorts[outPortName]; !ok {
		p.InitOutPort(p, outPortName)
	}
	p.PathFuncs[outPortName] = pathFmtFunc
}

// ------------------------------------------------------------------------
// Run method
// ------------------------------------------------------------------------

// Run runs the process by instantiating and executing Tasks for all inputs
// and parameter values on its in-ports. in the case when there are no inputs
// or parameter values on the in-ports, it will run just once before it
// terminates. note that the actual execution of shell commands are done inside
// Task.Execute, not here.
func (p *Process) Run() {
	defer p.CloseOutPorts()
	// Check that CoresPerTask is a sane number
	if p.CoresPerTask > cap(p.workflow.concurrentTasks) {
		Failf("%s: CoresPerTask (%d) can't be greater than maxConcurrentTasks of workflow (%d)\n", p.Name(), p.CoresPerTask, cap(p.workflow.concurrentTasks))
	}

	tasks := make(chan *Task, BUFSIZE)
	go func() {
		defer close(tasks)
		for t := range p.createTasks() {
			tasks <- t

			// Sending FIFOs for the task
			for oname, oip := range t.OutIPs {
				if oip.doStream {
					if oip.FifoFileExists() {
						Fail("Fifo file exists, so exiting (clean up fifo files before restarting the workflow): ", oip.FifoPath())
					}
					oip.CreateFifo()
					p.Out(oname).Send(oip)
				}
			}

			// Execute task in separate go-routine
			go t.Execute()
		}
	}()

	// Wait for tasks to finish, in the order they were started (thus maintaining
	// order of IPs), and then sending output IPs
	for t := range tasks {
		<-t.Done
		for oname, oip := range t.OutIPs {
			if !oip.doStream { // Streaming (FIFO) outputs have been sent earlier
				p.Out(oname).Send(oip)
			}
			// Remove any FIFO file
			if oip.doStream && oip.FifoFileExists() {
				os.Remove(oip.FifoPath())
			}
		}
	}
}

// createTasks is a helper method for Run that creates tasks based on incoming
// IPs on in-ports, and feeds them to the Run method on the returned channel ch
func (p *Process) createTasks() (ch chan *Task) {
	ch = make(chan *Task)
	go func() {
		defer close(ch)

		inIPs := map[string]*FileIP{}
		params := map[string]string{}
		tags := map[string]string{}

		inPortsOpen := true
		paramPortsOpen := true
		for {
			// Only read on in-ports if we have any
			if len(p.inPorts) > 0 {
				inIPs, inPortsOpen = p.receiveOnInPorts()
				// If in-port is closed, that means we got the last params on last iteration, so break
				if !inPortsOpen {
					break
				}
			}
			// Only read on param in-ports if we have any
			if len(p.inParamPorts) > 0 {
				params, paramPortsOpen = p.receiveOnInParamPorts()
				// If param-port is closed, that means we got the last params on last iteration, so break
				if !paramPortsOpen {
					break
				}
			}

			for iname, ip := range inIPs {
				for k, v := range ip.Tags() {
					tags[iname+"."+k] = v
				}
			}

			// Create task and send on the channel we are about to return
			ch <- NewTask(p.workflow, p, p.Name(), p.CommandPattern, inIPs, p.PathFuncs, p.PortInfo, params, tags, p.Prepend, p.CustomExecute, p.CoresPerTask)

			// If we have no in-ports nor param in-ports, we should break after the first iteration
			if len(p.inPorts) == 0 && len(p.inParamPorts) == 0 {
				break
			}
		}
	}()
	return ch
}
<commit_msg>Add comments explaining the need for processing completed Tasks<commit_after>package scipipe

import (
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
)

// Process is the central component in SciPipe after Workflow. Processes are
// long-running "services" that schedules and executes Tasks based on the IPs
// and parameters received on its in-ports and parameter ports
type Process struct {
	BaseProcess
	CommandPattern string
	PathFuncs      map[string]func(*Task) string
	CustomExecute  func(*Task)
	CoresPerTask   int
	Prepend        string
	Spawn          bool
	PortInfo       map[string]*PortInfo
}

// ------------------------------------------------------------------------
// Factory method(s)
// ------------------------------------------------------------------------

// NewProc returns a new Process, and initializes its ports based on the
// command pattern.
func NewProc(workflow *Workflow, name string, cmd string) *Process {
	p := &Process{
		BaseProcess: NewBaseProcess(
			workflow,
			name,
		),
		CommandPattern: cmd,
		PathFuncs:      make(map[string]func(*Task) string),
		Spawn:          true,
		CoresPerTask:   1,
		PortInfo:       map[string]*PortInfo{},
	}
	workflow.AddProc(p)
	p.initPortsFromCmdPattern(cmd, nil)
	p.initDefaultPathFuncs()
	return p
}

// PortInfo is a container for various information about process ports
type PortInfo struct {
	portType  string
	extension string
	doStream  bool
	join      bool
	joinSep   string
}

// initPortsFromCmdPattern is a helper function for NewProc, that sets up in-
// and out-ports based on the shell command pattern used to create the Process.
// Ports are set up in this way:
// `{i:PORTNAME}` specifies an in-port
// `{o:PORTNAME}` specifies an out-port
// `{os:PORTNAME}` specifies an out-port that streams via a FIFO file
// `{p:PORTNAME}` a "parameter (in-)port", which means a port where parameters can be "streamed"
func (p *Process) initPortsFromCmdPattern(cmd string, params map[string]string) {
	// Find in/out port names and params and set up ports
	r := getShellCommandPlaceHolderRegex()
	ms := r.FindAllStringSubmatch(cmd, -1)

	for _, m := range ms {
		portType := m[1]
		portRest := m[2]
		splitParts := strings.Split(portRest, "|")
		portName := splitParts[0]

		p.PortInfo[portName] = &PortInfo{portType: portType}

		for _, part := range splitParts[1:] {
			// If the |-separated part starts with a dot, treat it as a
			// configuration for file extenion to use
			fileExtPtn := regexp.MustCompile("\\.([a-z0-9\\.\\-\\_]+)")
			if fileExtPtn.MatchString(part) {
				m := fileExtPtn.FindStringSubmatch(part)
				p.PortInfo[portName].extension = m[1]
			}
			// If the |-separated part starts with "join:"
			// then treat the character following that as the character to use
			// when joining multiple files received on a sub-stream
			joinPtn := regexp.MustCompile("join:([^{}|]+)")
			if joinPtn.MatchString(part) {
				m := joinPtn.FindStringSubmatch(part)
				p.PortInfo[portName].join = true
				p.PortInfo[portName].joinSep = m[1]
			}
		}
	}

	for portName, pInfo := range p.PortInfo {
		if pInfo.portType == "o" || pInfo.portType == "os" {
			p.InitOutPort(p, portName)
			if pInfo.portType == "os" {
				p.PortInfo[portName].doStream = true
			}
		} else if pInfo.portType == "i" {
			p.InitInPort(p, portName)
		} else if pInfo.portType == "p" {
			if params == nil {
				p.InitInParamPort(p, portName)
			} else if _, ok := params[portName]; !ok {
				p.InitInParamPort(p, portName)
			}
		}
	}
}

// initDefaultPathFuncs does exactly what it name says: Initializes default
// path formatters for processes, that is used if no explicit path is set, using
// the proc.SetPath[...] methods
func (p *Process) initDefaultPathFuncs() {
	for outName := range p.OutPorts() {
		outName := outName
		p.PathFuncs[outName] = func(t *Task) string {
			pathPcs := []string{}
			for _, ipName := range sortedFileIPMapKeys(t.InIPs) {
				pathPcs = append(pathPcs, filepath.Base(t.InIP(ipName).Path()))
			}
			procName := sanitizePathFragment(t.Process.Name())
			pathPcs = append(pathPcs, procName)
			for _, paramName := range sortedStringMapKeys(t.Params) {
				pathPcs = append(pathPcs, paramName+"_"+t.Param(paramName))
			}
			for _, tagName := range sortedStringMapKeys(t.Tags) {
				pathPcs = append(pathPcs, tagName+"_"+t.Tag(tagName))
			}
			pathPcs = append(pathPcs, outName)
			fileExt := p.PortInfo[outName].extension
			if fileExt != "" {
				pathPcs = append(pathPcs, fileExt)
			}
			return strings.Join(pathPcs, ".")
		}
	}
}

func sortedFileIPMapKeys(kv map[string]*FileIP) []string {
	keys := []string{}
	for k := range kv {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	return keys
}

func sortedStringMapKeys(kv map[string]string) []string {
	keys := []string{}
	for k := range kv {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	return keys
}

// ------------------------------------------------------------------------
// Main API methods: Port accessor methods
// ------------------------------------------------------------------------

// In is a short-form for InPort() (of BaseProcess), which works only on Process
// processes
func (p *Process) In(portName string) *InPort {
	if portName == "" && len(p.InPorts()) == 1 {
		for _, inPort := range p.InPorts() {
			return inPort // Return the (only) in-port available
		}
	}
	return p.InPort(portName)
}

// Out is a short-form for OutPort() (of BaseProcess), which works only on
// Process processes
func (p *Process) Out(portName string) *OutPort {
	if portName == "" && len(p.OutPorts()) == 1 {
		for _, outPort := range p.OutPorts() {
			return outPort // Return the (only) out-port available
		}
	}
	return p.OutPort(portName)
}

// InParam is a short-form for InParamPort() (of BaseProcess), which works only on Process
// processes
func (p *Process) InParam(portName string) *InParamPort {
	if _, ok := p.inParamPorts[portName]; !ok {
		p.InitInParamPort(p, portName)
	}
	return p.InParamPort(portName)
}

// OutParam is a short-form for OutParamPort() (of BaseProcess), which works only on
// Process processes
func (p *Process) OutParam(portName string) *OutParamPort {
	return p.OutParamPort(portName)
}

// ------------------------------------------------------------------------
// Main API methods: Configure path formatting
// ------------------------------------------------------------------------

// SetOut initializes a port (if it does not already exist), and takes a
// configuration for its outputs paths via a pattern similar to the command
// pattern used to create new processes, with placeholder tags. Available
// placeholder tags to use are:
// {i:inport_name}
// {p:param_name}
// {t:tag_name}
// An example might be: {i:foo}.replace_with_{p:replacement}.txt
// ... given that the process contains an in-port named 'foo', and a parameter
// named 'replacement'.
// If an out-port with the specified name does not exist, it will be created.
// This allows to create out-ports for filenames that are created without explicitly
// stating a filename on the commandline, such as when only submitting a prefix.
func (p *Process) SetOut(outPortName string, pathPattern string) {
	p.SetOutFunc(outPortName, func(t *Task) string {
		path := pathPattern // Avoiding reusing the same variable in multiple instances of this func

		r := getShellCommandPlaceHolderRegex()
		matches := r.FindAllStringSubmatch(path, -1)
		for _, match := range matches {
			var replacement string

			placeHolder := match[0]
			phType := match[1]
			restMatch := match[2]

			parts := strings.Split(restMatch, "|")
			portName := parts[0]
			restParts := parts[1:]

			switch phType {
			case "i":
				replacement = t.InPath(portName)
			case "o":
				if _, ok := t.Process.PathFuncs[portName]; !ok {
					Fail("No such pathfunc for out-port " + portName + " in task " + t.Name)
				}
				replacement = t.Process.PathFuncs[portName](t)
			case "p":
				replacement = t.Param(portName)
			case "t":
				replacement = t.Tag(portName)
			default:
				Fail("Replace failed for placeholder ", portName, " for path patterh '", path, "'")
			}

			if len(restParts) > 0 {
				substPtn := regexp.MustCompile("s\\/([^\\/]+)\\/([^\\/]*)\\/")
				trimEndPtn := regexp.MustCompile("%(.*)")
				basenamePtn := regexp.MustCompile(`.*\/`)

				for _, restPart := range restParts {
					// If the |-separated part looks like a search/replace
					// pattern on the form of: s/SEARCHSTRING/REPLACESTRING/
					// then execute the replacement.
					if substPtn.MatchString(restPart) {
						mbits := substPtn.FindStringSubmatch(restPart)
						search := mbits[1]
						replace := mbits[2]
						replacement = strings.Replace(replacement, search, replace, 1)
					}
					// If the |-separated part starts iwth a %-character, then
					// trim everything following that character, from the end
					// of the path.
					if trimEndPtn.MatchString(restPart) {
						mbits := trimEndPtn.FindStringSubmatch(restPart)
						end := mbits[1]
						if end == replacement[len(replacement)-len(end):] {
							replacement = replacement[:len(replacement)-len(end)]
						}
					}
					// If the |-separated part is "basename", then remove all leading
					// folders up to the actual file name.
					if restPart == "basename" {
						replacement = basenamePtn.ReplaceAllString(replacement, "")
					}
				}
			}

			// Replace placeholder with concrete value
			path = strings.Replace(path, placeHolder, replacement, -1)
		}
		return path
	})
}

// SetOutFunc takes a function which produces a file path based on data
// available in *Task, such as concrete file paths and parameter values,
func (p *Process) SetOutFunc(outPortName string, pathFmtFunc func(task *Task) (path string)) {
	if _, ok := p.outPorts[outPortName]; !ok {
		p.InitOutPort(p, outPortName)
	}
	p.PathFuncs[outPortName] = pathFmtFunc
}

// ------------------------------------------------------------------------
// Run method
// ------------------------------------------------------------------------

// Run runs the process by instantiating and executing Tasks for all inputs
// and parameter values on its in-ports. in the case when there are no inputs
// or parameter values on the in-ports, it will run just once before it
// terminates. note that the actual execution of shell commands are done inside
// Task.Execute, not here.
func (p *Process) Run() {
	defer p.CloseOutPorts()
	// Check that CoresPerTask is a sane number
	if p.CoresPerTask > cap(p.workflow.concurrentTasks) {
		Failf("%s: CoresPerTask (%d) can't be greater than maxConcurrentTasks of workflow (%d)\n", p.Name(), p.CoresPerTask, cap(p.workflow.concurrentTasks))
	}

	// By using a channel to preserve Task ordering, completed Tasks can be
	// reaped while Tasks are still being created. Waiting for all Tasks to be
	// spawned before processing any can cause deadlock under certain workflow
	// architectures when there are more than BUFSIZE Tasks per process, see #81.
	// Note: This effectively limits concurrency to BUFSIZE Tasks per process.
	tasks := make(chan *Task, BUFSIZE)
	go func() {
		defer close(tasks)
		for t := range p.createTasks() {
			tasks <- t

			// Sending FIFOs for the task
			for oname, oip := range t.OutIPs {
				if oip.doStream {
					if oip.FifoFileExists() {
						Fail("Fifo file exists, so exiting (clean up fifo files before restarting the workflow): ", oip.FifoPath())
					}
					oip.CreateFifo()
					p.Out(oname).Send(oip)
				}
			}

			// Execute task in separate go-routine
			go t.Execute()
		}
	}()

	// Wait for tasks to finish, in the order they were started (thus maintaining
	// order of IPs), and then sending output IPs
	for t := range tasks {
		<-t.Done
		for oname, oip := range t.OutIPs {
			if !oip.doStream { // Streaming (FIFO) outputs have been sent earlier
				p.Out(oname).Send(oip)
			}
			// Remove any FIFO file
			if oip.doStream && oip.FifoFileExists() {
				os.Remove(oip.FifoPath())
			}
		}
	}
}

// createTasks is a helper method for Run that creates tasks based on incoming
// IPs on in-ports, and feeds them to the Run method on the returned channel ch
func (p *Process) createTasks() (ch chan *Task) {
	ch = make(chan *Task)
	go func() {
		defer close(ch)

		inIPs := map[string]*FileIP{}
		params := map[string]string{}
		tags := map[string]string{}

		inPortsOpen := true
		paramPortsOpen := true
		for {
			// Only read on in-ports if we have any
			if len(p.inPorts) > 0 {
				inIPs, inPortsOpen = p.receiveOnInPorts()
				// If in-port is closed, that means we got the last params on last iteration, so break
				if !inPortsOpen {
					break
				}
			}
			// Only read on param in-ports if we have any
			if len(p.inParamPorts) > 0 {
				params, paramPortsOpen = p.receiveOnInParamPorts()
				// If param-port is closed, that means we got the last params on last iteration, so break
				if !paramPortsOpen {
					break
				}
			}

			for iname, ip := range inIPs {
				for k, v := range ip.Tags() {
					tags[iname+"."+k] = v
				}
			}

			// Create task and send on the channel we are about to return
			ch <- NewTask(p.workflow, p, p.Name(), p.CommandPattern, inIPs, p.PathFuncs, p.PortInfo, params, tags, p.Prepend, p.CustomExecute, p.CoresPerTask)

			// If we have no in-ports nor param in-ports, we should break after the first iteration
			if len(p.inPorts) == 0 && len(p.inParamPorts) == 0 {
				break
			}
		}
	}()
	return ch
}
<|endoftext|>


<commit_before>package main

import (
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"time"
)

type Program struct {
	Name string `json:"name"`
}

// does the given directory contain a 'main' file?
func isProgram(parentDir, dir string) bool {
	_, err := os.Stat(filepath.Join(parentDir, dir, "main"))
	return err == nil
}

func readDir(dir string) ([]*Program, error) {
	programs := []*Program{}

	infos, err := ioutil.ReadDir(dir)
	if err != nil {
		return programs, err
	}

	for _, info := range infos {
		if err == nil && info.IsDir() && isProgram(dir, info.Name()) {
			programs = append(programs, &Program{info.Name()})
		}
	}
	return programs, nil
}

func MonitorPrograms(repo, workingDir string, delay time.Duration) (chan []*Program, error) {
	ch := make(chan []*Program)

	sha := ""

	go func() {
		for {
			defer func() {
				time.Sleep(delay)
			}()

			newSha, err := MasterSha(repo)

			if err != nil {
				log.Print(err)
				continue
			}

			if newSha != sha {
				err := Pull(workingDir)

				if err != nil {
					log.Print(err)
					continue
				}

				newPrograms, err := readDir(workingDir)

				if err != nil {
					log.Print(err)
					continue
				}

				ch <- newPrograms
				sha = newSha
			}
		}
	}()

	return ch, nil
}

func FindProgram(name string, programs []*Program) *Program {
	for _, program := range programs {
		if program.Name == name {
			return program
		}
	}

	return nil
}
<commit_msg>add some logging whilst looking for programs<commit_after>package main

import (
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"time"
)

type Program struct {
	Name string `json:"name"`
}

// does the given directory contain a 'main' file?
func isProgram(parentDir, dir string) bool {
	_, err := os.Stat(filepath.Join(parentDir, dir, "main"))
	return err == nil
}

func readDir(dir string) ([]*Program, error) {
	programs := []*Program{}

	log.Println("looking for programs in", dir)
	infos, err := ioutil.ReadDir(dir)
	if err != nil {
		return programs, err
	}

	for _, info := range infos {
		if err == nil && info.IsDir() && isProgram(dir, info.Name()) {
			programName := info.Name()
			log.Println("found program:", programName)

			programs = append(programs, &Program{info.Name()})
		}
	}

	return programs, nil
}

func MonitorPrograms(repo, workingDir string, delay time.Duration) (chan []*Program, error) {
	ch := make(chan []*Program)

	sha := ""

	go func() {
		for {
			defer func() {
				time.Sleep(delay)
			}()

			newSha, err := MasterSha(repo)

			if err != nil {
				log.Print(err)
				continue
			}

			if newSha != sha {
				log.Println("pulling from repository", repo)
				err := Pull(workingDir)

				if err != nil {
					log.Print(err)
					continue
				}

				newPrograms, err := readDir(workingDir)

				if err != nil {
					log.Print(err)
					continue
				}

				ch <- newPrograms
				sha = newSha
			}
		}
	}()

	return ch, nil
}

func FindProgram(name string, programs []*Program) *Program {
	for _, program := range programs {
		if program.Name == name {
			return program
		}
	}

	return nil
}
<|endoftext|>


<commit_before><commit_msg>Add level-guard to Recommender selector log statement<commit_after><|endoftext|>