<commit_before>package cloudinit

import (
	"encoding/base64"
	"fmt"
	"launchpad.net/goyaml"
	"launchpad.net/juju-core/cloudinit"
	"launchpad.net/juju-core/environs"
	"launchpad.net/juju-core/environs/config"
	"launchpad.net/juju-core/log"
	"launchpad.net/juju-core/state"
	"launchpad.net/juju-core/trivial"
	"launchpad.net/juju-core/upstart"
	"path"
	"strings"
)

// TODO(dfc) duplicated from environs/ec2

const mgoPort = 37017

var mgoPortSuffix = fmt.Sprintf(":%d", mgoPort)

// MachineConfig represents initialization information for a new juju machine.
// Creation of cloudinit data from this struct is largely provider-independent,
// but we'll keep it internal until we need to factor it out.
type MachineConfig struct {
	// StateServer specifies whether the new machine will run a ZooKeeper
	// or MongoDB instance.
	StateServer bool

	// ServerCertAndKey holds the state server certificate and private
	// key in PEM format; it is required when StateServer is set,
	// and ignored otherwise.
	ServerCertAndKey []byte

	// InstanceIdAccessor holds bash code that evaluates to the current instance id.
	InstanceIdAccessor string

	// ProviderType identifies the provider type so the host
	// knows which kind of provider to use.
	ProviderType string

	// StateInfo holds the means for the new instance to communicate with the
	// juju state. Unless the new machine is running a state server (StateServer is
	// set), there must be at least one state server address supplied.
	// The entity name must match that of the machine being started,
	// or be empty when starting a state server.
	StateInfo *state.Info

	// Tools is juju tools to be used on the new machine.
	Tools *state.Tools

	// DataDir holds the directory that juju state will be put in the new
	// machine.
	DataDir string

	// MachineId identifies the new machine. It must be non-negative.
	MachineId int

	// AuthorizedKeys specifies the keys that are allowed to
	// connect to the machine (see cloudinit.SSHAddAuthorizedKeys)
	// If no keys are supplied, there can be no ssh access to the node.
	// On a bootstrap machine, that is fatal. On other
	// machines it will mean that the ssh, scp and debug-hooks
	// commands cannot work.
	AuthorizedKeys string

	// Config holds the initial environment configuration.
	Config *config.Config
}

func addScripts(c *cloudinit.Config, scripts ...string) {
	for _, s := range scripts {
		c.AddRunCmd(s)
	}
}

func base64yaml(m *config.Config) string {
	data, err := goyaml.Marshal(m.AllAttrs())
	if err != nil {
		// can't happen, these values have been validated a number of times
		panic(err)
	}
	return base64.StdEncoding.EncodeToString(data)
}

const serverCertFile = "/var/lib/juju/server-cert.pem"

func New(cfg *MachineConfig) (*cloudinit.Config, error) {
	if err := verifyConfig(cfg); err != nil {
		return nil, err
	}
	c := cloudinit.New()

	c.AddSSHAuthorizedKeys(cfg.AuthorizedKeys)
	c.AddPackage("git")

	addScripts(c,
		fmt.Sprintf("mkdir -p %s", cfg.DataDir),
		"mkdir -p /var/log/juju")

	// Make a directory for the tools to live in, then fetch the
	// tools and unarchive them into it.
	addScripts(c,
		"bin="+shquote(cfg.jujuTools()),
		"mkdir -p $bin",
		fmt.Sprintf("wget --no-verbose -O - %s | tar xz -C $bin", shquote(cfg.Tools.URL)),
		fmt.Sprintf("echo -n %s > $bin/downloaded-url.txt", shquote(cfg.Tools.URL)),
	)

	debugFlag := ""
	// TODO: disable debug mode by default when the system is stable.
	if true || log.Debug {
		debugFlag = " --debug"
	}

	if cfg.StateServer {
		addScripts(c, fmt.Sprintf("echo %s > %s",
			shquote(string(cfg.ServerCertAndKey)), serverCertFile))
		// TODO The public bucket must come from the environment configuration.
		b := cfg.Tools.Binary
		url := fmt.Sprintf("http://juju-dist.s3.amazonaws.com/tools/mongo-2.2.0-%s-%s.tgz", b.Series, b.Arch)
		addScripts(c,
			"mkdir -p /opt",
			fmt.Sprintf("wget --no-verbose -O - %s | tar xz -C /opt", shquote(url)),
		)
		if err := addMongoToBoot(c); err != nil {
			return nil, err
		}
		addScripts(c, cfg.jujuTools()+"/jujud bootstrap-state"+
			" --instance-id "+cfg.InstanceIdAccessor+
			" --env-config "+shquote(base64yaml(cfg.Config))+
			" --state-servers localhost"+mgoPortSuffix+
			" --initial-password "+shquote(cfg.StateInfo.Password)+
			debugFlag,
		)

	}

	if err := addAgentToBoot(c, cfg, "machine",
		state.MachineEntityName(cfg.MachineId),
		fmt.Sprintf("--machine-id %d "+debugFlag, cfg.MachineId)); err != nil {
		return nil, err
	}

	// general options
	c.SetAptUpgrade(true)
	c.SetAptUpdate(true)
	c.SetOutput(cloudinit.OutAll, "| tee -a /var/log/cloud-init-output.log", "")
	return c, nil
}

func addAgentToBoot(c *cloudinit.Config, cfg *MachineConfig, kind, name, args string) error {
	// Make the agent run via a symbolic link to the actual tools
	// directory, so it can upgrade itself without needing to change
	// the upstart script.
	toolsDir := environs.AgentToolsDir(cfg.DataDir, name)
	// TODO(dfc) ln -nfs, so it doesn't fail if for some reason that the target already exists
	addScripts(c, fmt.Sprintf("ln -s %v %s", cfg.Tools.Binary, shquote(toolsDir)))

	agentDir := environs.AgentDir(cfg.DataDir, name)
	addScripts(c, fmt.Sprintf("mkdir -p %s", shquote(agentDir)))
	svc := upstart.NewService("jujud-" + name)
	logPath := fmt.Sprintf("/var/log/juju/%s.log", name)
	cmd := fmt.Sprintf(
		"%s/jujud %s"+
			" --state-servers '%s'"+
			" --log-file %s"+
			" --data-dir '%s'"+
			" --initial-password '%s'"+
			" %s",
		toolsDir, kind,
		cfg.stateHostAddrs(),
		logPath,
		cfg.DataDir,
		cfg.StateInfo.Password,
		args,
	)
	conf := &upstart.Conf{
		Service: *svc,
		Desc:    fmt.Sprintf("juju %s agent", name),
		Cmd:     cmd,
		Out:     logPath,
	}
	cmds, err := conf.InstallCommands()
	if err != nil {
		return fmt.Errorf("cannot make cloud-init upstart script for the %s agent: %v", name, err)
	}
	addScripts(c, cmds...)
	return nil
}

func addMongoToBoot(c *cloudinit.Config) error {
	addScripts(c,
		"mkdir -p /var/lib/juju/db/journal",
		// Otherwise we get three files with 100M+ each, which takes time.
		"dd bs=1M count=1 if=/dev/zero of=/var/lib/juju/db/journal/prealloc.0",
		"dd bs=1M count=1 if=/dev/zero of=/var/lib/juju/db/journal/prealloc.1",
		"dd bs=1M count=1 if=/dev/zero of=/var/lib/juju/db/journal/prealloc.2",
	)
	svc := upstart.NewService("juju-db")
	conf := &upstart.Conf{
		Service: *svc,
		Desc:    "juju state database",
		Cmd: "/opt/mongo/bin/mongod" +
			" --auth" +
			" --dbpath=/var/lib/juju/db" +
			" --bind_ip 0.0.0.0" +
			" --port " + fmt.Sprint(mgoPort) +
			" --noprealloc" +
			" --smallfiles",
	}
	cmds, err := conf.InstallCommands()
	if err != nil {
		return fmt.Errorf("cannot make cloud-init upstart script for the state database: %v", err)
	}
	addScripts(c, cmds...)
	return nil
}

// versionDir converts a tools URL into a name
// to use as a directory for storing the tools executables in
// by using the last element stripped of its extension.
func versionDir(toolsURL string) string {
	name := path.Base(toolsURL)
	ext := path.Ext(name)
	return name[:len(name)-len(ext)]
}

func (cfg *MachineConfig) jujuTools() string {
	return environs.ToolsDir(cfg.DataDir, cfg.Tools.Binary)
}

func (cfg *MachineConfig) stateHostAddrs() string {
	var hosts []string
	if cfg.StateServer {
		hosts = append(hosts, "localhost"+mgoPortSuffix)
	}
	if cfg.StateInfo != nil {
		hosts = append(hosts, cfg.StateInfo.Addrs...)
	}
	return strings.Join(hosts, ",")
}

// shquote quotes s so that when read by bash, no metacharacters
// within s will be interpreted as such.
func shquote(s string) string {
	// single-quote becomes single-quote, double-quote, single-quote, double-quote, single-quote
	return `'` + strings.Replace(s, `'`, `'"'"'`, -1) + `'`
}

type requiresError string

func (e requiresError) Error() string {
	return "invalid machine configuration: missing " + string(e)
}

func verifyConfig(cfg *MachineConfig) (err error) {
	defer trivial.ErrorContextf(&err, "invalid machine configuration")
	if cfg.MachineId < 0 {
		return fmt.Errorf("negative machine id")
	}
	if cfg.ProviderType == "" {
		return fmt.Errorf("missing provider type")
	}
	if cfg.DataDir == "" {
		return fmt.Errorf("missing var directory")
	}
	if cfg.Tools == nil {
		return fmt.Errorf("missing tools")
	}
	if cfg.Tools.URL == "" {
		return fmt.Errorf("missing tools URL")
	}
	if cfg.StateInfo == nil {
		return fmt.Errorf("missing state info")
	}
	if cfg.StateServer {
		if cfg.InstanceIdAccessor == "" {
			return fmt.Errorf("missing instance id accessor")
		}
		if cfg.Config == nil {
			return fmt.Errorf("missing environment configuration")
		}
		if cfg.StateInfo.EntityName != "" {
			return fmt.Errorf("entity name must be blank when starting a state server")
		}
		if len(cfg.ServerCertAndKey) == 0 {
			return fmt.Errorf("missing certificate")
		}
	} else {
		if len(cfg.StateInfo.Addrs) == 0 {
			return fmt.Errorf("missing state hosts")
		}
		if cfg.StateInfo.EntityName != state.MachineEntityName(cfg.MachineId) {
			return fmt.Errorf("entity name must match started machine")
		}
	}
	for _, r := range cfg.StateInfo.Password {
		if r == '\'' || r == '\\' || r < 32 {
			return fmt.Errorf("password has disallowed characters")
		}
	}
	return nil
}
<commit_msg>environs/cloudinit: chmod certfile 600<commit_after>package cloudinit

import (
	"encoding/base64"
	"fmt"
	"launchpad.net/goyaml"
	"launchpad.net/juju-core/cloudinit"
	"launchpad.net/juju-core/environs"
	"launchpad.net/juju-core/environs/config"
	"launchpad.net/juju-core/log"
	"launchpad.net/juju-core/state"
	"launchpad.net/juju-core/trivial"
	"launchpad.net/juju-core/upstart"
	"path"
	"strings"
)

// TODO(dfc) duplicated from environs/ec2

const mgoPort = 37017

var mgoPortSuffix = fmt.Sprintf(":%d", mgoPort)

// MachineConfig represents initialization information for a new juju machine.
// Creation of cloudinit data from this struct is largely provider-independent,
// but we'll keep it internal until we need to factor it out.
type MachineConfig struct {
	// StateServer specifies whether the new machine will run a ZooKeeper
	// or MongoDB instance.
	StateServer bool

	// ServerCertAndKey holds the state server certificate and private
	// key in PEM format; it is required when StateServer is set,
	// and ignored otherwise.
	ServerCertAndKey []byte

	// InstanceIdAccessor holds bash code that evaluates to the current instance id.
	InstanceIdAccessor string

	// ProviderType identifies the provider type so the host
	// knows which kind of provider to use.
	ProviderType string

	// StateInfo holds the means for the new instance to communicate with the
	// juju state. Unless the new machine is running a state server (StateServer is
	// set), there must be at least one state server address supplied.
	// The entity name must match that of the machine being started,
	// or be empty when starting a state server.
	StateInfo *state.Info

	// Tools is juju tools to be used on the new machine.
	Tools *state.Tools

	// DataDir holds the directory that juju state will be put in the new
	// machine.
	DataDir string

	// MachineId identifies the new machine. It must be non-negative.
	MachineId int

	// AuthorizedKeys specifies the keys that are allowed to
	// connect to the machine (see cloudinit.SSHAddAuthorizedKeys)
	// If no keys are supplied, there can be no ssh access to the node.
	// On a bootstrap machine, that is fatal. On other
	// machines it will mean that the ssh, scp and debug-hooks
	// commands cannot work.
	AuthorizedKeys string

	// Config holds the initial environment configuration.
	Config *config.Config
}

func addScripts(c *cloudinit.Config, scripts ...string) {
	for _, s := range scripts {
		c.AddRunCmd(s)
	}
}

func base64yaml(m *config.Config) string {
	data, err := goyaml.Marshal(m.AllAttrs())
	if err != nil {
		// can't happen, these values have been validated a number of times
		panic(err)
	}
	return base64.StdEncoding.EncodeToString(data)
}

const serverCertFile = "/var/lib/juju/server-cert.pem"

func New(cfg *MachineConfig) (*cloudinit.Config, error) {
	if err := verifyConfig(cfg); err != nil {
		return nil, err
	}
	c := cloudinit.New()

	c.AddSSHAuthorizedKeys(cfg.AuthorizedKeys)
	c.AddPackage("git")

	addScripts(c,
		fmt.Sprintf("mkdir -p %s", cfg.DataDir),
		"mkdir -p /var/log/juju")

	// Make a directory for the tools to live in, then fetch the
	// tools and unarchive them into it.
	addScripts(c,
		"bin="+shquote(cfg.jujuTools()),
		"mkdir -p $bin",
		fmt.Sprintf("wget --no-verbose -O - %s | tar xz -C $bin", shquote(cfg.Tools.URL)),
		fmt.Sprintf("echo -n %s > $bin/downloaded-url.txt", shquote(cfg.Tools.URL)),
	)

	debugFlag := ""
	// TODO: disable debug mode by default when the system is stable.
	if true || log.Debug {
		debugFlag = " --debug"
	}

	if cfg.StateServer {
		addScripts(c,
			fmt.Sprintf("echo %s > %s",
				shquote(string(cfg.ServerCertAndKey)), serverCertFile),
			"chmod 600 "+serverCertFile,
		)

		// TODO The public bucket must come from the environment configuration.
		b := cfg.Tools.Binary
		url := fmt.Sprintf("http://juju-dist.s3.amazonaws.com/tools/mongo-2.2.0-%s-%s.tgz", b.Series, b.Arch)
		addScripts(c,
			"mkdir -p /opt",
			fmt.Sprintf("wget --no-verbose -O - %s | tar xz -C /opt", shquote(url)),
		)
		if err := addMongoToBoot(c); err != nil {
			return nil, err
		}
		addScripts(c, cfg.jujuTools()+"/jujud bootstrap-state"+
			" --instance-id "+cfg.InstanceIdAccessor+
			" --env-config "+shquote(base64yaml(cfg.Config))+
			" --state-servers localhost"+mgoPortSuffix+
			" --initial-password "+shquote(cfg.StateInfo.Password)+
			debugFlag,
		)

	}

	if err := addAgentToBoot(c, cfg, "machine",
		state.MachineEntityName(cfg.MachineId),
		fmt.Sprintf("--machine-id %d "+debugFlag, cfg.MachineId)); err != nil {
		return nil, err
	}

	// general options
	c.SetAptUpgrade(true)
	c.SetAptUpdate(true)
	c.SetOutput(cloudinit.OutAll, "| tee -a /var/log/cloud-init-output.log", "")
	return c, nil
}

func addAgentToBoot(c *cloudinit.Config, cfg *MachineConfig, kind, name, args string) error {
	// Make the agent run via a symbolic link to the actual tools
	// directory, so it can upgrade itself without needing to change
	// the upstart script.
	toolsDir := environs.AgentToolsDir(cfg.DataDir, name)
	// TODO(dfc) ln -nfs, so it doesn't fail if for some reason that the target already exists
	addScripts(c, fmt.Sprintf("ln -s %v %s", cfg.Tools.Binary, shquote(toolsDir)))

	agentDir := environs.AgentDir(cfg.DataDir, name)
	addScripts(c, fmt.Sprintf("mkdir -p %s", shquote(agentDir)))
	svc := upstart.NewService("jujud-" + name)
	logPath := fmt.Sprintf("/var/log/juju/%s.log", name)
	cmd := fmt.Sprintf(
		"%s/jujud %s"+
			" --state-servers '%s'"+
			" --log-file %s"+
			" --data-dir '%s'"+
			" --initial-password '%s'"+
			" %s",
		toolsDir, kind,
		cfg.stateHostAddrs(),
		logPath,
		cfg.DataDir,
		cfg.StateInfo.Password,
		args,
	)
	conf := &upstart.Conf{
		Service: *svc,
		Desc:    fmt.Sprintf("juju %s agent", name),
		Cmd:     cmd,
		Out:     logPath,
	}
	cmds, err := conf.InstallCommands()
	if err != nil {
		return fmt.Errorf("cannot make cloud-init upstart script for the %s agent: %v", name, err)
	}
	addScripts(c, cmds...)
	return nil
}

func addMongoToBoot(c *cloudinit.Config) error {
	addScripts(c,
		"mkdir -p /var/lib/juju/db/journal",
		// Otherwise we get three files with 100M+ each, which takes time.
		"dd bs=1M count=1 if=/dev/zero of=/var/lib/juju/db/journal/prealloc.0",
		"dd bs=1M count=1 if=/dev/zero of=/var/lib/juju/db/journal/prealloc.1",
		"dd bs=1M count=1 if=/dev/zero of=/var/lib/juju/db/journal/prealloc.2",
	)
	svc := upstart.NewService("juju-db")
	conf := &upstart.Conf{
		Service: *svc,
		Desc:    "juju state database",
		Cmd: "/opt/mongo/bin/mongod" +
			" --auth" +
			" --dbpath=/var/lib/juju/db" +
			" --bind_ip 0.0.0.0" +
			" --port " + fmt.Sprint(mgoPort) +
			" --noprealloc" +
			" --smallfiles",
	}
	cmds, err := conf.InstallCommands()
	if err != nil {
		return fmt.Errorf("cannot make cloud-init upstart script for the state database: %v", err)
	}
	addScripts(c, cmds...)
	return nil
}

// versionDir converts a tools URL into a name
// to use as a directory for storing the tools executables in
// by using the last element stripped of its extension.
func versionDir(toolsURL string) string {
	name := path.Base(toolsURL)
	ext := path.Ext(name)
	return name[:len(name)-len(ext)]
}

func (cfg *MachineConfig) jujuTools() string {
	return environs.ToolsDir(cfg.DataDir, cfg.Tools.Binary)
}

func (cfg *MachineConfig) stateHostAddrs() string {
	var hosts []string
	if cfg.StateServer {
		hosts = append(hosts, "localhost"+mgoPortSuffix)
	}
	if cfg.StateInfo != nil {
		hosts = append(hosts, cfg.StateInfo.Addrs...)
	}
	return strings.Join(hosts, ",")
}

// shquote quotes s so that when read by bash, no metacharacters
// within s will be interpreted as such.
func shquote(s string) string {
	// single-quote becomes single-quote, double-quote, single-quote, double-quote, single-quote
	return `'` + strings.Replace(s, `'`, `'"'"'`, -1) + `'`
}

type requiresError string

func (e requiresError) Error() string {
	return "invalid machine configuration: missing " + string(e)
}

func verifyConfig(cfg *MachineConfig) (err error) {
	defer trivial.ErrorContextf(&err, "invalid machine configuration")
	if cfg.MachineId < 0 {
		return fmt.Errorf("negative machine id")
	}
	if cfg.ProviderType == "" {
		return fmt.Errorf("missing provider type")
	}
	if cfg.DataDir == "" {
		return fmt.Errorf("missing var directory")
	}
	if cfg.Tools == nil {
		return fmt.Errorf("missing tools")
	}
	if cfg.Tools.URL == "" {
		return fmt.Errorf("missing tools URL")
	}
	if cfg.StateInfo == nil {
		return fmt.Errorf("missing state info")
	}
	if cfg.StateServer {
		if cfg.InstanceIdAccessor == "" {
			return fmt.Errorf("missing instance id accessor")
		}
		if cfg.Config == nil {
			return fmt.Errorf("missing environment configuration")
		}
		if cfg.StateInfo.EntityName != "" {
			return fmt.Errorf("entity name must be blank when starting a state server")
		}
		if len(cfg.ServerCertAndKey) == 0 {
			return fmt.Errorf("missing certificate")
		}
	} else {
		if len(cfg.StateInfo.Addrs) == 0 {
			return fmt.Errorf("missing state hosts")
		}
		if cfg.StateInfo.EntityName != state.MachineEntityName(cfg.MachineId) {
			return fmt.Errorf("entity name must match started machine")
		}
	}
	for _, r := range cfg.StateInfo.Password {
		if r == '\'' || r == '\\' || r < 32 {
			return fmt.Errorf("password has disallowed characters")
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package errorsutil

import (
	"errors"
	"fmt"
	"strings"
)

// Append adds additional text to an existing error.
func Append(err error, str string) error {
	return errors.New(fmt.Sprint(err) + str)
}

func Join(inclNils bool, errs ...error) error {
	strs := []string{}
	for _, err := range errs {
		if err == nil {
			if inclNils {
				strs = append(strs, "nil")
			}
		} else {
			strs = append(strs, err.Error())
		}
	}
	if len(strs) > 0 {
		return errors.New(strings.Join(strs, ";"))
	}
	return nil
}

// PanicOnErr is a syntactic sugar function to panic on error.
func PanicOnErr(err error) {
	if err != nil {
		panic(err)
	}
}

type ErrorInfo struct {
	Error       error
	ErrorString string // must match Error
	Code        string
	Display     string
	Input       string
	Correct     string
}

type ErrorInfos []*ErrorInfo

func (eis ErrorInfos) Inflate() {
	for i, ei := range eis {
		if ei != nil && ei.Error != nil {
			ei.ErrorString = ei.Error.Error()
		}
		eis[i] = ei
	}
}

func (eis ErrorInfos) GoodInputs() []string {
	inputs := []string{}
	for _, ei := range eis {
		if ei.Error == nil {
			inputs = append(inputs, ei.Input)
		}
	}
	return inputs
}

func (eis ErrorInfos) ErrorsString() []string {
	estrings := []string{}
	for _, ei := range eis {
		if ei.Error != nil {
			estrings = append(estrings, ei.Error.Error())
		}
	}
	return estrings
}

func (eis ErrorInfos) Filter(isError bool) ErrorInfos {
	filtered := ErrorInfos{}
	for _, ei := range eis {
		if isError {
			if ei.Error != nil {
				filtered = append(filtered, ei)
			} else {
				filtered = append(filtered, nil)
			}
		} else if !isError {
			if ei.Error == nil {
				filtered = append(filtered, ei)
			} else {
				filtered = append(filtered, nil)
			}
		}
	}
	return filtered
}
<commit_msg>feat: add errorsutil.ErrorInfos.GoodCorrects()<commit_after>package errorsutil

import (
	"errors"
	"fmt"
	"strings"
)

// Append adds additional text to an existing error.
func Append(err error, str string) error {
	return errors.New(fmt.Sprint(err) + str)
}

func Join(inclNils bool, errs ...error) error {
	strs := []string{}
	for _, err := range errs {
		if err == nil {
			if inclNils {
				strs = append(strs, "nil")
			}
		} else {
			strs = append(strs, err.Error())
		}
	}
	if len(strs) > 0 {
		return errors.New(strings.Join(strs, ";"))
	}
	return nil
}

// PanicOnErr is a syntactic sugar function to panic on error.
func PanicOnErr(err error) {
	if err != nil {
		panic(err)
	}
}

type ErrorInfo struct {
	Error       error
	ErrorString string // must match Error
	Code        string
	Display     string
	Input       string
	Correct     string
}

type ErrorInfos []*ErrorInfo

func (eis ErrorInfos) Inflate() {
	for i, ei := range eis {
		if ei != nil && ei.Error != nil {
			ei.ErrorString = ei.Error.Error()
		}
		eis[i] = ei
	}
}

func (eis ErrorInfos) GoodInputs() []string {
	inputs := []string{}
	for _, ei := range eis {
		if ei.Error == nil {
			inputs = append(inputs, ei.Input)
		}
	}
	return inputs
}

func (eis ErrorInfos) GoodCorrects() []string {
	inputs := []string{}
	for _, ei := range eis {
		if ei.Error == nil {
			inputs = append(inputs, ei.Correct)
		}
	}
	return inputs
}

func (eis ErrorInfos) ErrorsString() []string {
	estrings := []string{}
	for _, ei := range eis {
		if ei.Error != nil {
			estrings = append(estrings, ei.Error.Error())
		}
	}
	return estrings
}

func (eis ErrorInfos) Filter(isError bool) ErrorInfos {
	filtered := ErrorInfos{}
	for _, ei := range eis {
		if isError {
			if ei.Error != nil {
				filtered = append(filtered, ei)
			} else {
				filtered = append(filtered, nil)
			}
		} else if !isError {
			if ei.Error == nil {
				filtered = append(filtered, ei)
			} else {
				filtered = append(filtered, nil)
			}
		}
	}
	return filtered
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"github.com/zerowidth/gh-shorthand/alfred"
	"os"
	"strings"
)

func main() {
	var input string
	if len(os.Args) < 2 {
		input = ""
	} else {
		input = strings.Join(os.Args[1:], " ")
	}

	fmt.Fprintf(os.Stderr, "input: %#v\n", input)

	item := alfred.Item{
		Title: "hello",
		Valid: false,
	}

	items := alfred.Items{Items: []alfred.Item{item}}
	encoded, _ := json.Marshal(items)

	os.Stdout.Write(encoded)
	os.Stdout.WriteString("\n")
}
<commit_msg>Panic when we can't encode the alfred items<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"github.com/zerowidth/gh-shorthand/alfred"
	"os"
	"strings"
)

func main() {
	var input string
	if len(os.Args) < 2 {
		input = ""
	} else {
		input = strings.Join(os.Args[1:], " ")
	}

	fmt.Fprintf(os.Stderr, "input: %#v\n", input)

	item := alfred.Item{
		Title: "hello",
		Valid: false,
	}

	items := alfred.Items{Items: []alfred.Item{item}}
	encoded, err := json.Marshal(items)
	if err != nil {
		panic(err.Error())
	}

	os.Stdout.Write(encoded)
	os.Stdout.WriteString("\n")
}
<|endoftext|>


<commit_before>package input_test

import (
	. "github.com/jutkko/mindown/input"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Input", func() {

})
<commit_msg>Remove unused test<commit_after><|endoftext|>


<commit_before>package glfw

import (
	"bytes"
	"github.com/ajhager/rog"
	"github.com/banthar/gl"
	"github.com/jteeuwen/glfw"
	"image"
	"image/color"
	"image/draw"
	_ "image/png"
)

func init() {
	rog.SetBackend(new(glfwBackend))
}

type glfwBackend struct {
	open  bool
	mouse *rog.MouseData
	key   int
	font  image.Image
	zoom  int
}

func (w *glfwBackend) Open(width, height, zoom int) {
	if err := glfw.Init(); err != nil {
		panic(err)
	}

	w.zoom = zoom

	glfw.OpenWindowHint(glfw.WindowNoResize, gl.TRUE)
	err := glfw.OpenWindow(width*16*zoom, height*16*zoom, 8, 8, 8, 8, 0, 0, glfw.Windowed)
	if err != nil {
		panic(err)
	}

	glfw.SetWindowCloseCallback(func() int { w.Close(); return 0 })
	glfw.SetKeyCallback(func(key, state int) { w.setKey(key, state) })
	glfw.Enable(glfw.KeyRepeat)

	w.mouse = new(rog.MouseData)
	glfw.SetMousePosCallback(func(x, y int) { w.mouseMove(x, y) })
	glfw.SetMouseButtonCallback(func(but, state int) { w.mousePress(but, state) })

	font, _, err := image.Decode(bytes.NewBuffer(rog.FontData()))
	if err != nil {
		panic(err)
	}
	w.font = font

	glInit(width*16*zoom, height*16*zoom, w.font)

	w.open = true
}

func (w *glfwBackend) IsOpen() bool {
	return w.open && glfw.WindowParam(glfw.Opened) == 1
}

func (w *glfwBackend) Close() {
	w.open = false
	glfw.CloseWindow()
	glfw.Terminate()
}

func (w *glfwBackend) Name(title string) {
	glfw.SetWindowTitle(title)
}

func (w *glfwBackend) Render(console *rog.Console) {
	if w.IsOpen() {
		w.mouse.Left.Released = false
		w.mouse.Right.Released = false
		w.mouse.Middle.Released = false
		w.key = -1

		gl.Clear(gl.COLOR_BUFFER_BIT)

		for y := 0; y < console.Height(); y++ {
			for x := 0; x < console.Width(); x++ {
				fg, bg, ch := console.Get(x, y)

				setColor(bg)
				w.letter(x, y, 0)

				if ch != 0 && ch != 32 {
					setColor(fg)
					w.letter(x, y, ch)
				}
			}
		}

		glfw.SwapBuffers()
	}
}

func (w *glfwBackend) Mouse() *rog.MouseData {
	return w.mouse
}

func (w *glfwBackend) mouseMove(x, y int) {
	w.mouse.Pos.X = x
	w.mouse.Pos.Y = y
	w.mouse.Cell.X = x / (16 * w.zoom)
	w.mouse.Cell.Y = y / (16 * w.zoom)
}

func (w *glfwBackend) mousePress(button, state int) {
	switch state {
	case glfw.KeyPress:
		switch button {
		case glfw.MouseLeft:
			w.mouse.Left.Pressed = true
		case glfw.MouseRight:
			w.mouse.Right.Pressed = true
		case glfw.MouseMiddle:
			w.mouse.Middle.Pressed = true
		}
	case glfw.KeyRelease:
		switch button {
		case glfw.MouseLeft:
			w.mouse.Left.Pressed = false
			w.mouse.Left.Released = true
		case glfw.MouseRight:
			w.mouse.Right.Pressed = false
			w.mouse.Right.Released = true
		case glfw.MouseMiddle:
			w.mouse.Middle.Pressed = false
			w.mouse.Middle.Released = true
		}
	}
}

func (w *glfwBackend) Key() int {
	return w.key
}

func (w *glfwBackend) setKey(key, state int) {
	if state == glfw.KeyPress {
		rogKey, exists := glfwToRogKey[key]
		if exists {
			w.key = rogKey
		}

		if key < 256 {
			w.key = key
		}
	}
}

func glInit(width, height int, font image.Image) {
	gl.Init()
	gl.Enable(gl.TEXTURE_2D)
	gl.PixelStorei(gl.UNPACK_ALIGNMENT, 1)
	gl.Viewport(0, 0, width, height)
	gl.MatrixMode(gl.PROJECTION)
	gl.LoadIdentity()
	gl.Ortho(0, float64(width), float64(height), 0, -1, 1)
	gl.MatrixMode(gl.MODELVIEW)
	gl.LoadIdentity()
	gl.Enable(gl.BLEND)
	gl.BlendFunc(gl.SRC_ALPHA, gl.ONE_MINUS_SRC_ALPHA)
	textures := make([]gl.Texture, 1)
	gl.GenTextures(textures)
	textures[0].Bind(gl.TEXTURE_2D)
	gl.TexParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST)
	gl.TexParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST)

	b := font.Bounds()
	m := image.NewRGBA(image.Rect(0, 0, b.Dx(), b.Dy()))
	draw.Draw(m, m.Bounds(), font, b.Min, draw.Src)

	gl.TexImage2D(gl.TEXTURE_2D, 0, gl.RGBA, m.Bounds().Max.X, m.Bounds().Max.Y, 0, gl.RGBA, gl.UNSIGNED_BYTE, m.Pix)
}

// Draw a letter at a certain coordinate
func (w *glfwBackend) letter(lx, ly int, c rune) {
	b := w.font.Bounds()
	fc := float32(16 * w.zoom)
	cx := float32(lx) * fc
	cy := float32(ly) * fc
	verts := []float32{cx, cy, cx, cy + fc, cx + fc, cy + fc, cx + fc, cy, cx, cy}
	x := float32(c % 256)
	y := float32(c / 256)
	s := float32(16) / float32(b.Max.X)
	t := float32(16) / float32(b.Max.Y)
	u := x * s
	v := y * t
	gl.EnableClientState(gl.VERTEX_ARRAY)
	gl.EnableClientState(gl.TEXTURE_COORD_ARRAY)
	gl.VertexPointer(2, 0, verts)
	gl.TexCoordPointer(2, 0, []float32{u, v, u, v + t, u + s, v + t, u + s, v, u, v})
	gl.DrawArrays(gl.POLYGON, 0, len(verts)/2-1)
	gl.DisableClientState(gl.VERTEX_ARRAY)
	gl.DisableClientState(gl.TEXTURE_COORD_ARRAY)
}

// Set the opengl drawing color
func setColor(c color.Color) {
	r, g, b, _ := c.RGBA()
	gl.Color3ub(uint8(r), uint8(g), uint8(b))
}

var glfwToRogKey = map[int]int{
	glfw.KeyBackspace:  rog.Backspace,
	glfw.KeyTab:        rog.Tab,
	glfw.KeyEsc:        rog.Escape,
	glfw.KeySpace:      rog.Space,
	glfw.KeyDel:        rog.Delete,
	glfw.KeyLsuper:     rog.LSuper,
	glfw.KeyRsuper:     rog.RSuper,
	glfw.KeyLshift:     rog.LShift,
	glfw.KeyRshift:     rog.RShift,
	glfw.KeyLctrl:      rog.LControl,
	glfw.KeyRctrl:      rog.RControl,
	glfw.KeyLalt:       rog.LAlt,
	glfw.KeyRalt:       rog.RAlt,
	glfw.KeyF1:         rog.F1,
	glfw.KeyF2:         rog.F2,
	glfw.KeyF3:         rog.F3,
	glfw.KeyF4:         rog.F4,
	glfw.KeyF5:         rog.F5,
	glfw.KeyF6:         rog.F6,
	glfw.KeyF7:         rog.F7,
	glfw.KeyF8:         rog.F8,
	glfw.KeyF9:         rog.F9,
	glfw.KeyF10:        rog.F10,
	glfw.KeyF11:        rog.F11,
	glfw.KeyF12:        rog.F12,
	glfw.KeyF13:        rog.F13,
	glfw.KeyF14:        rog.F14,
	glfw.KeyF15:        rog.F15,
	glfw.KeyF16:        rog.F16,
	glfw.KeyUp:         rog.Up,
	glfw.KeyDown:       rog.Down,
	glfw.KeyLeft:       rog.Left,
	glfw.KeyRight:      rog.Right,
	glfw.KeyEnter:      rog.Return,
	glfw.KeyInsert:     rog.Insert,
	glfw.KeyHome:       rog.Home,
	glfw.KeyEnd:        rog.End,
	glfw.KeyCapslock:   rog.Capslock,
	glfw.KeyKPDivide:  rog.KPDivide,
	glfw.KeyKPMultiply: rog.KPMultiply,
	glfw.KeyKPSubtract: rog.KPSubtract,
	glfw.KeyKPAdd:      rog.KPAdd,
	glfw.KeyKPDecimal:  rog.KPDecimal,
	glfw.KeyKPEqual:    rog.KPEqual,
	glfw.KeyKPEnter:    rog.KPEnter,
	glfw.KeyKPNumlock:  rog.KPNumlock,
	glfw.KeyKP0:        rog.KP0,
	glfw.KeyKP1:        rog.KP1,
	glfw.KeyKP2:        rog.KP2,
	glfw.KeyKP3:        rog.KP3,
	glfw.KeyKP4:        rog.KP4,
	glfw.KeyKP5:        rog.KP5,
	glfw.KeyKP6:        rog.KP6,
	glfw.KeyKP7:        rog.KP7,
	glfw.KeyKP8:        rog.KP8,
	glfw.KeyKP9:        rog.KP9}
<commit_msg>Faster set color in glfw backend<commit_after>package glfw

import (
	"bytes"
	"github.com/ajhager/rog"
	"github.com/banthar/gl"
	"github.com/jteeuwen/glfw"
	"image"
	"image/draw"
	_ "image/png"
)

func init() {
	rog.SetBackend(new(glfwBackend))
}

type glfwBackend struct {
	open  bool
	mouse *rog.MouseData
	key   int
	font  image.Image
	zoom  int
}

func (w *glfwBackend) Open(width, height, zoom int) {
	if err := glfw.Init(); err != nil {
		panic(err)
	}

	w.zoom = zoom

	glfw.OpenWindowHint(glfw.WindowNoResize, gl.TRUE)
	err := glfw.OpenWindow(width*16*zoom, height*16*zoom, 8, 8, 8, 8, 0, 0, glfw.Windowed)
	if err != nil {
		panic(err)
	}

	glfw.SetWindowCloseCallback(func() int { w.Close(); return 0 })
	glfw.SetKeyCallback(func(key, state int) { w.setKey(key, state) })
	glfw.Enable(glfw.KeyRepeat)

	w.mouse = new(rog.MouseData)
	glfw.SetMousePosCallback(func(x, y int) { w.mouseMove(x, y) })
	glfw.SetMouseButtonCallback(func(but, state int) { w.mousePress(but, state) })

	font, _, err := image.Decode(bytes.NewBuffer(rog.FontData()))
	if err != nil {
		panic(err)
	}
	w.font = font

	glInit(width*16*zoom, height*16*zoom, w.font)

	w.open = true
}

func (w *glfwBackend) IsOpen() bool {
	return w.open && glfw.WindowParam(glfw.Opened) == 1
}

func (w *glfwBackend) Close() {
	w.open = false
	glfw.CloseWindow()
	glfw.Terminate()
}

func (w *glfwBackend) Name(title string) {
	glfw.SetWindowTitle(title)
}

func (w *glfwBackend) Render(console *rog.Console) {
	if w.IsOpen() {
		w.mouse.Left.Released = false
		w.mouse.Right.Released = false
		w.mouse.Middle.Released = false
		w.key = -1

		gl.Clear(gl.COLOR_BUFFER_BIT)

		for y := 0; y < console.Height(); y++ {
			for x := 0; x < console.Width(); x++ {
				fg, bg, ch := console.Get(x, y)

	            gl.Color3ub(bg.R, bg.G, bg.B)
				w.letter(x, y, 0)

				if ch != 0 && ch != 32 {
	                gl.Color3ub(fg.R, fg.G, fg.B)
					w.letter(x, y, ch)
				}
			}
		}

		glfw.SwapBuffers()
	}
}

func (w *glfwBackend) Mouse() *rog.MouseData {
	return w.mouse
}

func (w *glfwBackend) mouseMove(x, y int) {
	w.mouse.Pos.X = x
	w.mouse.Pos.Y = y
	w.mouse.Cell.X = x / (16 * w.zoom)
	w.mouse.Cell.Y = y / (16 * w.zoom)
}

func (w *glfwBackend) mousePress(button, state int) {
	switch state {
	case glfw.KeyPress:
		switch button {
		case glfw.MouseLeft:
			w.mouse.Left.Pressed = true
		case glfw.MouseRight:
			w.mouse.Right.Pressed = true
		case glfw.MouseMiddle:
			w.mouse.Middle.Pressed = true
		}
	case glfw.KeyRelease:
		switch button {
		case glfw.MouseLeft:
			w.mouse.Left.Pressed = false
			w.mouse.Left.Released = true
		case glfw.MouseRight:
			w.mouse.Right.Pressed = false
			w.mouse.Right.Released = true
		case glfw.MouseMiddle:
			w.mouse.Middle.Pressed = false
			w.mouse.Middle.Released = true
		}
	}
}

func (w *glfwBackend) Key() int {
	return w.key
}

func (w *glfwBackend) setKey(key, state int) {
	if state == glfw.KeyPress {
		rogKey, exists := glfwToRogKey[key]
		if exists {
			w.key = rogKey
		}

		if key < 256 {
			w.key = key
		}
	}
}

func glInit(width, height int, font image.Image) {
	gl.Init()
	gl.Enable(gl.TEXTURE_2D)
	gl.PixelStorei(gl.UNPACK_ALIGNMENT, 1)
	gl.Viewport(0, 0, width, height)
	gl.MatrixMode(gl.PROJECTION)
	gl.LoadIdentity()
	gl.Ortho(0, float64(width), float64(height), 0, -1, 1)
	gl.MatrixMode(gl.MODELVIEW)
	gl.LoadIdentity()
	gl.Enable(gl.BLEND)
	gl.BlendFunc(gl.SRC_ALPHA, gl.ONE_MINUS_SRC_ALPHA)
	textures := make([]gl.Texture, 1)
	gl.GenTextures(textures)
	textures[0].Bind(gl.TEXTURE_2D)
	gl.TexParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST)
	gl.TexParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST)

	b := font.Bounds()
	m := image.NewRGBA(image.Rect(0, 0, b.Dx(), b.Dy()))
	draw.Draw(m, m.Bounds(), font, b.Min, draw.Src)

	gl.TexImage2D(gl.TEXTURE_2D, 0, gl.RGBA, m.Bounds().Max.X, m.Bounds().Max.Y, 0, gl.RGBA, gl.UNSIGNED_BYTE, m.Pix)
	gl.EnableClientState(gl.VERTEX_ARRAY)
	gl.EnableClientState(gl.TEXTURE_COORD_ARRAY)
}

// Draw a letter at a certain coordinate
func (w *glfwBackend) letter(lx, ly int, c rune) {
	b := w.font.Bounds()
	fc := float32(16 * w.zoom)
	cx := float32(lx) * fc
	cy := float32(ly) * fc
	verts := []float32{cx, cy, cx, cy + fc, cx + fc, cy + fc, cx + fc, cy, cx, cy}
	x := float32(c % 256)
	y := float32(c / 256)
	s := float32(16) / float32(b.Max.X)
	t := float32(16) / float32(b.Max.Y)
	u := x * s
	v := y * t
	gl.VertexPointer(2, 0, verts)
	gl.TexCoordPointer(2, 0, []float32{u, v, u, v + t, u + s, v + t, u + s, v, u, v})
	gl.DrawArrays(gl.POLYGON, 0, len(verts)/2-1)
}

var glfwToRogKey = map[int]int{
	glfw.KeyBackspace:  rog.Backspace,
	glfw.KeyTab:        rog.Tab,
	glfw.KeyEsc:        rog.Escape,
	glfw.KeySpace:      rog.Space,
	glfw.KeyDel:        rog.Delete,
	glfw.KeyLsuper:     rog.LSuper,
	glfw.KeyRsuper:     rog.RSuper,
	glfw.KeyLshift:     rog.LShift,
	glfw.KeyRshift:     rog.RShift,
	glfw.KeyLctrl:      rog.LControl,
	glfw.KeyRctrl:      rog.RControl,
	glfw.KeyLalt:       rog.LAlt,
	glfw.KeyRalt:       rog.RAlt,
	glfw.KeyF1:         rog.F1,
	glfw.KeyF2:         rog.F2,
	glfw.KeyF3:         rog.F3,
	glfw.KeyF4:         rog.F4,
	glfw.KeyF5:         rog.F5,
	glfw.KeyF6:         rog.F6,
	glfw.KeyF7:         rog.F7,
	glfw.KeyF8:         rog.F8,
	glfw.KeyF9:         rog.F9,
	glfw.KeyF10:        rog.F10,
	glfw.KeyF11:        rog.F11,
	glfw.KeyF12:        rog.F12,
	glfw.KeyF13:        rog.F13,
	glfw.KeyF14:        rog.F14,
	glfw.KeyF15:        rog.F15,
	glfw.KeyF16:        rog.F16,
	glfw.KeyUp:         rog.Up,
	glfw.KeyDown:       rog.Down,
	glfw.KeyLeft:       rog.Left,
	glfw.KeyRight:      rog.Right,
	glfw.KeyEnter:      rog.Return,
	glfw.KeyInsert:     rog.Insert,
	glfw.KeyHome:       rog.Home,
	glfw.KeyEnd:        rog.End,
	glfw.KeyCapslock:   rog.Capslock,
	glfw.KeyKPDivide:  rog.KPDivide,
	glfw.KeyKPMultiply: rog.KPMultiply,
	glfw.KeyKPSubtract: rog.KPSubtract,
	glfw.KeyKPAdd:      rog.KPAdd,
	glfw.KeyKPDecimal:  rog.KPDecimal,
	glfw.KeyKPEqual:    rog.KPEqual,
	glfw.KeyKPEnter:    rog.KPEnter,
	glfw.KeyKPNumlock:  rog.KPNumlock,
	glfw.KeyKP0:        rog.KP0,
	glfw.KeyKP1:        rog.KP1,
	glfw.KeyKP2:        rog.KP2,
	glfw.KeyKP3:        rog.KP3,
	glfw.KeyKP4:        rog.KP4,
	glfw.KeyKP5:        rog.KP5,
	glfw.KeyKP6:        rog.KP6,
	glfw.KeyKP7:        rog.KP7,
	glfw.KeyKP8:        rog.KP8,
	glfw.KeyKP9:        rog.KP9}
<|endoftext|>


<commit_before>package stormstack

import (
	"fmt"
	log "github.com/cihub/seelog"
	"launchpad.net/goose/client"
	goosehttp "launchpad.net/goose/http"
	"net/http"
	"net/url"
	"stormstack.org/stormio/persistence"
)

var (
	nclient = client.NewPublicClient("")
)

func BuildStormData(arq *persistence.AssetRequest) (stormdata string) {
	log.Debugf("stormtracker URL is %#v", arq)
	if arq.ControlProvider.StormtrackerURL == "" {
		log.Debugf("[areq %s] stormtracker URL is nil", arq.Id)
		return ""
	}
	u, err := url.Parse(arq.ControlProvider.StormtrackerURL)
	if err != nil {
		log.Debugf("[areq %s] Failed to parse the stormtracker URL %v", arq.Id, arq.ControlProvider.StormtrackerURL)
		return ""
	}
	stormdata = u.Scheme + "://" + arq.ControlTokenId + "@" + u.Host + "/" + u.Path
	log.Debugf("[areq %s] stormdata is %v", arq.Id, stormdata)
	return stormdata
}

func DeRegisterStormAgent(dr *persistence.AssetRequest) (err error) {
	if dr.AgentId != "" {
		var resp persistence.StormAgent
		headers := make(http.Header)
		headers.Add("V-Auth-Token", dr.ControlTokenId)
		u := fmt.Sprintf("%s/agents/%s", dr.ControlProvider.StormtrackerURL, dr.AgentId)
		requestData := &goosehttp.RequestData{ReqHeaders: headers, RespValue: &resp, ExpectedStatus: []int{http.StatusNoContent}}
		err = nclient.SendRequest(client.DELETE, "", u, requestData)
		if err != nil {
			log.Errorf("[areq %s][res %s] Error in deregistering storm agent %v", dr.Id, dr.ResourceId, err)
		}
	}
	//dr.AgentId = ""
	return
}

type DomainAgent struct {
	AgentId string `json:"agentId"`
}

func DomainDeleteAgent(arq *persistence.AssetRequest) (err error) {
	if arq.AgentId == "" {
		log.Debugf("[areq %s][res %s] AgentID not present in assetRequest. Hence skipping deleting the agent in StormLight", arq.Id, arq.ResourceId)
		return nil
	}
	if arq.ControlProvider.StormlightURL == "" {
		log.Debugf("[areq %s][res %s] Stormlight URL missing in the assetRequest", arq.Id, arq.ResourceId)
		return fmt.Errorf("DomainDeleteAgent: missing stormlight URL in the ControlProvider")
	}
	if arq.ControlProvider.DefaultDomainId == "" {
		log.Debugf("[areq %s][res %s] Default DomainID missing in the assetRequest", arq.Id, arq.ResourceId)
		return fmt.Errorf("DomainDeleteAgent: missing domainID in the ControlProvider")
	}
	var resp DomainAgent
	headers := make(http.Header)
	headers.Add("V-Auth-Token", arq.ControlTokenId)
	requestData := &goosehttp.RequestData{ReqHeaders: headers, RespValue: &resp, ExpectedStatus: []int{http.StatusAccepted, http.StatusNoContent}}
	u := fmt.Sprintf("%s/domains/%s/agents/%s", arq.ControlProvider.StormlightURL, arq.ControlProvider.DefaultDomainId, arq.AgentId)
	err = nclient.SendRequest(client.DELETE, "", u, requestData)
	if err != nil {
		log.Errorf("[areq %s][res %s] Error in deleting storm agent %v with StormLight", arq.Id, arq.ResourceId, err)
		return err
	}
	log.Debugf("[areq %s][res %s] Deleted Agent ID %s with stormlight", arq.Id, arq.ResourceId, arq.AgentId)
	return nil
}
func DomainAddAgent(arq *persistence.AssetRequest) (err error) {
	var req DomainAgent
	req.AgentId = arq.AgentId
	if arq.ControlProvider.StormlightURL == "" {
		log.Debugf("[areq %s][res %s] Stormlight URL missing in the assetRequest", arq.Id, arq.ResourceId)
		return fmt.Errorf("DomainAddAgent: missing stormlight URL in the ControlProvider")
	}
	if arq.ControlProvider.DefaultDomainId == "" {
		log.Debugf("[areq %s][res %s] Default DomainID missing in the assetRequest", arq.Id, arq.ResourceId)
		return fmt.Errorf("DomainAddAgent: missing domainID in the ControlProvider")
	}

	log.Debugf("Registering storm agent with Stormlight %#v", req)
	var resp DomainAgent
	headers := make(http.Header)
	headers.Add("V-Auth-Token", arq.ControlTokenId)
	requestData := &goosehttp.RequestData{ReqHeaders: headers, ReqValue: req, RespValue: &resp, ExpectedStatus: []int{http.StatusAccepted, http.StatusOK}}
	u := fmt.Sprintf("%s/domains/%s/agents", arq.ControlProvider.StormlightURL, arq.ControlProvider.DefaultDomainId)
	err = nclient.SendRequest(client.POST, "", u, requestData)
	if err != nil {
		log.Debugf("[areq %s][res %s] Error in registering storm agent %v", arq.Id, arq.ResourceId, err)
		return err
	}
	log.Debugf("[areq %s][res %s] Registered Agent with stormlight %v. Response is %#v", arq.Id, arq.ResourceId, arq.AgentId, resp)
	return nil
}

func RegisterStormAgent(arq *persistence.AssetRequest, entityId string) (err error) {
	if arq.AgentId != "" {
		log.Debugf("[areq %s][res %s] Agent ID already exists agent Id is %v", arq.Id, arq.ResourceId, arq.AgentId)
		return nil
	}
	arq.ServerId = entityId
	var req persistence.StormAgent
	req.SerialKey = entityId
	req.Stoken = arq.ControlTokenId
	req.StormBolt = arq.ControlProvider.Bolt
	req.Id = arq.AgentId

	log.Debugf("Registering storm agent with tracker %#v", req)
	var resp persistence.StormAgent
	requestData := &goosehttp.RequestData{ReqValue: req, RespValue: &resp, ExpectedStatus: []int{http.StatusAccepted, http.StatusOK}}
	u := fmt.Sprintf("%s/agents", arq.ControlProvider.StormtrackerURL)
	err = nclient.SendRequest(client.POST, "", u, requestData)
	if err != nil {
		log.Debugf("[areq %s][res %s] Error in registering storm agent %v", arq.Id, arq.ResourceId, err)
		return err
	}
	arq.AgentId = resp.Id
	arq.SerialKey = entityId
	log.Debugf("[areq %s][res %s] Registered Agent with stormtracker %v", arq.Id, arq.ResourceId, arq.AgentId)
	// Register the agent in Stormlight default domain
	err = DomainAddAgent(arq)
	if err != nil {
		log.Debugf("[areq %s][res %s] Error in Adding agent %v to default domain %v", arq.Id, arq.ResourceId, arq.AgentId, arq.ControlProvider.DefaultDomainId)
		go DeRegisterStormAgent(arq)
		return err
	}
	return nil
}
<commit_msg>Agent ID can be passed from USG as part of the asset creation request<commit_after>package stormstack

import (
	"fmt"
	log "github.com/cihub/seelog"
	"launchpad.net/goose/client"
	goosehttp "launchpad.net/goose/http"
	"net/http"
	"net/url"
	"stormstack.org/stormio/persistence"
)

var (
	nclient = client.NewPublicClient("")
)

func BuildStormData(arq *persistence.AssetRequest) (stormdata string) {
	log.Debugf("stormtracker URL is %#v", arq)
	if arq.ControlProvider.StormtrackerURL == "" {
		log.Debugf("[areq %s] stormtracker URL is nil", arq.Id)
		return ""
	}
	u, err := url.Parse(arq.ControlProvider.StormtrackerURL)
	if err != nil {
		log.Debugf("[areq %s] Failed to parse the stormtracker URL %v", arq.Id, arq.ControlProvider.StormtrackerURL)
		return ""
	}
	stormdata = u.Scheme + "://" + arq.ControlTokenId + "@" + u.Host + "/" + u.Path
	log.Debugf("[areq %s] stormdata is %v", arq.Id, stormdata)
	return stormdata
}

func DeRegisterStormAgent(dr *persistence.AssetRequest) (err error) {
	if dr.AgentId != "" {
		var resp persistence.StormAgent
		headers := make(http.Header)
		headers.Add("V-Auth-Token", dr.ControlTokenId)
		u := fmt.Sprintf("%s/agents/%s", dr.ControlProvider.StormtrackerURL, dr.AgentId)
		requestData := &goosehttp.RequestData{ReqHeaders: headers, RespValue: &resp, ExpectedStatus: []int{http.StatusNoContent}}
		err = nclient.SendRequest(client.DELETE, "", u, requestData)
		if err != nil {
			log.Errorf("[areq %s][res %s] Error in deregistering storm agent %v", dr.Id, dr.ResourceId, err)
		}
	}
	//dr.AgentId = ""
	return
}

type DomainAgent struct {
	AgentId string `json:"agentId"`
}

func DomainDeleteAgent(arq *persistence.AssetRequest) (err error) {
	if arq.AgentId == "" {
		log.Debugf("[areq %s][res %s] AgentID not present in assetRequest. Hence skipping deleting the agent in StormLight", arq.Id, arq.ResourceId)
		return nil
	}
	if arq.ControlProvider.StormlightURL == "" {
		log.Debugf("[areq %s][res %s] Stormlight URL missing in the assetRequest", arq.Id, arq.ResourceId)
		return fmt.Errorf("DomainDeleteAgent: missing stormlight URL in the ControlProvider")
	}
	if arq.ControlProvider.DefaultDomainId == "" {
		log.Debugf("[areq %s][res %s] Default DomainID missing in the assetRequest", arq.Id, arq.ResourceId)
		return fmt.Errorf("DomainDeleteAgent: missing domainID in the ControlProvider")
	}
	var resp DomainAgent
	headers := make(http.Header)
	headers.Add("V-Auth-Token", arq.ControlTokenId)
	requestData := &goosehttp.RequestData{ReqHeaders: headers, RespValue: &resp, ExpectedStatus: []int{http.StatusAccepted, http.StatusNoContent}}
	u := fmt.Sprintf("%s/domains/%s/agents/%s", arq.ControlProvider.StormlightURL, arq.ControlProvider.DefaultDomainId, arq.AgentId)
	err = nclient.SendRequest(client.DELETE, "", u, requestData)
	if err != nil {
		log.Errorf("[areq %s][res %s] Error in deleting storm agent %v with StormLight", arq.Id, arq.ResourceId, err)
		return err
	}
	log.Debugf("[areq %s][res %s] Deleted Agent ID %s with stormlight", arq.Id, arq.ResourceId, arq.AgentId)
	return nil
}
func DomainAddAgent(arq *persistence.AssetRequest) (err error) {
	var req DomainAgent
	req.AgentId = arq.AgentId
	if arq.ControlProvider.StormlightURL == "" {
		log.Debugf("[areq %s][res %s] Stormlight URL missing in the assetRequest", arq.Id, arq.ResourceId)
		return fmt.Errorf("DomainAddAgent: missing stormlight URL in the ControlProvider")
	}
	if arq.ControlProvider.DefaultDomainId == "" {
		log.Debugf("[areq %s][res %s] Default DomainID missing in the assetRequest", arq.Id, arq.ResourceId)
		return fmt.Errorf("DomainAddAgent: missing domainID in the ControlProvider")
	}

	log.Debugf("Registering storm agent with Stormlight %#v", req)
	var resp DomainAgent
	headers := make(http.Header)
	headers.Add("V-Auth-Token", arq.ControlTokenId)
	requestData := &goosehttp.RequestData{ReqHeaders: headers, ReqValue: req, RespValue: &resp, ExpectedStatus: []int{http.StatusAccepted, http.StatusOK}}
	u := fmt.Sprintf("%s/domains/%s/agents", arq.ControlProvider.StormlightURL, arq.ControlProvider.DefaultDomainId)
	err = nclient.SendRequest(client.POST, "", u, requestData)
	if err != nil {
		log.Debugf("[areq %s][res %s] Error in registering storm agent %v", arq.Id, arq.ResourceId, err)
		return err
	}
	log.Debugf("[areq %s][res %s] Registered Agent with stormlight %v. Response is %#v", arq.Id, arq.ResourceId, arq.AgentId, resp)
	return nil
}

func RegisterStormAgent(arq *persistence.AssetRequest, entityId string) (err error) {
	arq.ServerId = entityId
	var req persistence.StormAgent
	req.SerialKey = entityId
	req.Stoken = arq.ControlTokenId
	req.StormBolt = arq.ControlProvider.Bolt
	req.Id = arq.AgentId

	log.Debugf("Registering storm agent with tracker %#v", req)
	var resp persistence.StormAgent
	requestData := &goosehttp.RequestData{ReqValue: req, RespValue: &resp, ExpectedStatus: []int{http.StatusAccepted, http.StatusOK}}
	u := fmt.Sprintf("%s/agents", arq.ControlProvider.StormtrackerURL)
	err = nclient.SendRequest(client.POST, "", u, requestData)
	if err != nil {
		log.Debugf("[areq %s][res %s] Error in registering storm agent %v", arq.Id, arq.ResourceId, err)
		return err
	}
	arq.AgentId = resp.Id
	arq.SerialKey = entityId
	log.Debugf("[areq %s][res %s] Registered Agent with stormtracker %v", arq.Id, arq.ResourceId, arq.AgentId)
	// Register the agent in Stormlight default domain
	err = DomainAddAgent(arq)
	if err != nil {
		log.Debugf("[areq %s][res %s] Error in Adding agent %v to default domain %v", arq.Id, arq.ResourceId, arq.AgentId, arq.ControlProvider.DefaultDomainId)
		go DeRegisterStormAgent(arq)
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/rogeralsing/gam/actor"
    "fmt"
	"os"
    "bufio"
)

type Become struct {}
type Hello struct{ Who string }
type BecomeActor struct{}

func (state *BecomeActor) Receive(context actor.Context) {
    switch msg := context.Message().(type) {
    case Hello:
        fmt.Printf("Hello %v\n", msg.Who)
        context.Become(state.Other)
    }
}

func (state *BecomeActor) Other(context actor.Context) {
    switch msg := context.Message().(type) {
    case Hello:
        fmt.Printf("%v, ey we are now handling messages in another behavior",msg.Who)
    }
}

func NewBecomeActor() actor.Actor {
    return &BecomeActor{}
}

func main() {
    pid := actor.Spawn(actor.Props(NewBecomeActor))
    pid.Tell(Hello{Who: "Roger"})
    pid.Tell(Hello{Who: "Roger"})
    bufio.NewReader(os.Stdin).ReadString('\n')
}<commit_msg>console<commit_after>package main

import (
	"fmt"

	"github.com/rogeralsing/gam/actor"
	"github.com/rogeralsing/goconsole"
)

type Become struct{}
type Hello struct{ Who string }
type BecomeActor struct{}

func (state *BecomeActor) Receive(context actor.Context) {
	switch msg := context.Message().(type) {
	case Hello:
		fmt.Printf("Hello %v\n", msg.Who)
		context.Become(state.Other)
	}
}

func (state *BecomeActor) Other(context actor.Context) {
	switch msg := context.Message().(type) {
	case Hello:
		fmt.Printf("%v, ey we are now handling messages in another behavior", msg.Who)
	}
}

func NewBecomeActor() actor.Actor {
	return &BecomeActor{}
}

func main() {
	pid := actor.Spawn(actor.Props(NewBecomeActor))
	pid.Tell(Hello{Who: "Roger"})
	pid.Tell(Hello{Who: "Roger"})
	console.ReadLine()
}
<|endoftext|>


<commit_before>package gomruby_test

import (
	. "."
	"errors"
	"io/ioutil"
	. "launchpad.net/gocheck"
	"testing"
)

func TestGoMRuby(t *testing.T) { TestingT(t) }

type F struct {
	m *MRuby
	c *LoadContext
}

var _ = Suite(&F{})

func (f *F) SetUpTest(c *C) {
	f.m = New()
	f.c = f.m.NewLoadContext("test.rb")
}

func (f *F) TearDownTest(c *C) {
	f.c.Delete()
	f.m.Delete()
}

func (f *F) TestLoad(c *C) {
	must := func(res interface{}, err error) interface{} {
		c.Check(err, IsNil)
		return res
	}

	c.Check(must(f.c.Load(`nil`)), Equals, nil)
	c.Check(must(f.c.Load(`true`)), Equals, true)
	c.Check(must(f.c.Load(`false`)), Equals, false)
	c.Check(must(f.c.Load(`1 + 1`)), Equals, 2)
	c.Check(must(f.c.Load(`1 - 1`)), Equals, 0)
	c.Check(must(f.c.Load(`1 - 2`)), Equals, -1)
	c.Check(must(f.c.Load(`2147483647`)), Equals, 2147483647) // max int32
	c.Check(must(f.c.Load(`3.14 + 42`)), Equals, 45.14)
	c.Check(must(f.c.Load(`domain = "express" + "42" + ".com"`)), Equals, "express42.com")
	c.Check(must(f.c.Load(`domain`)), Equals, "express42.com")
	c.Check(must(f.c.Load(`""`)), Equals, "")

	slice := []interface{}{1, 3.14, "foo"}
	hash := map[interface{}]interface{}{"foo": 1, 3.14: "bar"}
	mix := []interface{}{42, map[interface{}]interface{}{3.14: []interface{}{"bar"}, "foo": 1}}
	c.Check(must(f.c.Load(`[1, 3.14, "foo"]`)), DeepEquals, slice)
	c.Check(must(f.c.Load(`{3.14=>"bar", "foo"=>1}`)), DeepEquals, hash)
	c.Check(must(f.c.Load(`[42, {3.14=>["bar"], "foo"=>1}]`)), DeepEquals, mix)

	c.Check(must(f.c.Load("ARGV.inspect")), Equals, `[]`)
	c.Check(must(f.c.Load("ARGV.inspect", nil, true, false)), Equals, `[nil, true, false]`)
	c.Check(must(f.c.Load("ARGV.inspect", 1, 3.14, "foo")), Equals, `[1, 3.14, "foo"]`)
	c.Check(must(f.c.Load("ARGV.inspect", slice)), Equals, `[[1, 3.14, "foo"]]`)
	c.Check(must(f.c.Load("ARGV.inspect", hash)), Equals, `[{3.14=>"bar", "foo"=>1}]`)
	c.Check(must(f.c.Load("ARGV.inspect", mix)), Equals, `[[42, {3.14=>["bar"], "foo"=>1}]]`)

	res, err := f.c.Load(`foo`)
	c.Check(res, Equals, nil)
	c.Check(err, DeepEquals, errors.New(`test.rb:1: undefined method 'foo' for main (NoMethodError)`))

	res, err = f.c.Load(`
begin
  foo
rescue => e
  e.inspect
  e.inspect
`)
	c.Check(res, Equals, nil)
	c.Check(err, DeepEquals, errors.New("SyntaxError: syntax error"))
}

func (f *F) TestLoadLimits(c *C) {
	c.ExpectFailure("conversion for large ints and floats is wrong")
	res, err := f.c.Load(`4294967295`) // max uint32
	c.Check(err, IsNil)
	c.Check(res, Equals, 4294967295)
	// TODO more tests for edge values
}

func (f *F) TestLoadMore(c *C) {
	res, err := f.c.Load("ARGV.map { |x| x * x }", 1, 2, 3)
	c.Check(err, IsNil)
	c.Check(res, DeepEquals, []interface{}{1, 4, 9})
}

func (f *F) TestDefineGoConst(c *C) {
	f.m.DefineGoConst("MY_CONST", 42)
	res, err := f.c.Load("Go::MY_CONST")
	c.Check(err, IsNil)
	c.Check(res, Equals, 42)
}

func (f *F) TestLoadContext(c *C) {
	res, err := f.c.Load(`$global = 1; local = 2`)
	c.Check(res, Equals, 2)
	c.Check(err, IsNil)

	// state is preserved
	res, err = f.c.Load(`$global`)
	c.Check(res, Equals, 1)
	c.Check(err, IsNil)
	res, err = f.c.Load(`local`)
	c.Check(res, Equals, 2)
	c.Check(err, IsNil)

	c2 := f.m.NewLoadContext("test2.rb")
	defer c2.Delete()

	// global variable is accessible from other context
	res, err = c2.Load(`$global`)
	c.Check(res, Equals, 1)
	c.Check(err, IsNil)

	// local is not
	res, err = c2.Load(`local`)
	c.Check(res, Equals, nil)
	c.Check(err, DeepEquals, errors.New("test2.rb:1: undefined method 'local' for main (NoMethodError)"))
}

func (f *F) TestBugRb(c *C) {
	b, err := ioutil.ReadFile("test.rb")
	c.Assert(err, IsNil)
	res, err := f.c.Load(string(b))
	c.Logf("%#v", res)
	c.Check(err, IsNil)
}
<commit_msg>Fix for Go 1.0.<commit_after>package gomruby_test

import (
	. "."
	"errors"
	"io/ioutil"
	. "launchpad.net/gocheck"
	"testing"
)

func TestGoMRuby(t *testing.T) { TestingT(t) }

type F struct {
	m *MRuby
	c *LoadContext
}

var _ = Suite(&F{})

func (f *F) SetUpTest(c *C) {
	f.m = New()
	f.c = f.m.NewLoadContext("test.rb")
}

func (f *F) TearDownTest(c *C) {
	f.c.Delete()
	f.m.Delete()
}

func (f *F) TestLoad(c *C) {
	must := func(res interface{}, err error) interface{} {
		c.Check(err, IsNil)
		return res
	}

	c.Check(must(f.c.Load(`nil`)), Equals, nil)
	c.Check(must(f.c.Load(`true`)), Equals, true)
	c.Check(must(f.c.Load(`false`)), Equals, false)
	c.Check(must(f.c.Load(`1 + 1`)), Equals, 2)
	c.Check(must(f.c.Load(`1 - 1`)), Equals, 0)
	c.Check(must(f.c.Load(`1 - 2`)), Equals, -1)
	c.Check(must(f.c.Load(`2147483647`)), Equals, 2147483647) // max int32
	c.Check(must(f.c.Load(`3.14 + 42`)), Equals, 45.14)
	c.Check(must(f.c.Load(`domain = "express" + "42" + ".com"`)), Equals, "express42.com")
	c.Check(must(f.c.Load(`domain`)), Equals, "express42.com")
	c.Check(must(f.c.Load(`""`)), Equals, "")

	slice := []interface{}{1, 3.14, "foo"}
	hash := map[interface{}]interface{}{"foo": 1, 3.14: "bar"}
	mix := []interface{}{42, map[interface{}]interface{}{3.14: []interface{}{"bar"}, "foo": 1}}
	c.Check(must(f.c.Load(`[1, 3.14, "foo"]`)), DeepEquals, slice)
	c.Check(must(f.c.Load(`{3.14=>"bar", "foo"=>1}`)), DeepEquals, hash)
	c.Check(must(f.c.Load(`[42, {3.14=>["bar"], "foo"=>1}]`)), DeepEquals, mix)

	c.Check(must(f.c.Load("ARGV.inspect")), Equals, `[]`)
	c.Check(must(f.c.Load("ARGV.inspect", nil, true, false)), Equals, `[nil, true, false]`)
	c.Check(must(f.c.Load("ARGV.inspect", 1, 3.14, "foo")), Equals, `[1, 3.14, "foo"]`)
	c.Check(must(f.c.Load("ARGV.inspect", slice)), Equals, `[[1, 3.14, "foo"]]`)
	c.Check(must(f.c.Load("ARGV.inspect", hash)), Equals, `[{3.14=>"bar", "foo"=>1}]`)
	c.Check(must(f.c.Load("ARGV.inspect", mix)), Equals, `[[42, {3.14=>["bar"], "foo"=>1}]]`)

	res, err := f.c.Load(`foo`)
	c.Check(res, Equals, nil)
	c.Check(err, DeepEquals, errors.New(`test.rb:1: undefined method 'foo' for main (NoMethodError)`))

	res, err = f.c.Load(`
begin
  foo
rescue => e
  e.inspect
  e.inspect
`)
	c.Check(res, Equals, nil)
	c.Check(err, DeepEquals, errors.New("SyntaxError: syntax error"))
}

func (f *F) TestLoadLimits(c *C) {
	c.ExpectFailure("conversion for large ints and floats is wrong")
	res, err := f.c.Load(`4294967295`) // max uint32
	c.Check(err, IsNil)
	c.Check(res, Equals, uint32(4294967295))
	// TODO more tests for edge values
}

func (f *F) TestLoadMore(c *C) {
	res, err := f.c.Load("ARGV.map { |x| x * x }", 1, 2, 3)
	c.Check(err, IsNil)
	c.Check(res, DeepEquals, []interface{}{1, 4, 9})
}

func (f *F) TestDefineGoConst(c *C) {
	f.m.DefineGoConst("MY_CONST", 42)
	res, err := f.c.Load("Go::MY_CONST")
	c.Check(err, IsNil)
	c.Check(res, Equals, 42)
}

func (f *F) TestLoadContext(c *C) {
	res, err := f.c.Load(`$global = 1; local = 2`)
	c.Check(res, Equals, 2)
	c.Check(err, IsNil)

	// state is preserved
	res, err = f.c.Load(`$global`)
	c.Check(res, Equals, 1)
	c.Check(err, IsNil)
	res, err = f.c.Load(`local`)
	c.Check(res, Equals, 2)
	c.Check(err, IsNil)

	c2 := f.m.NewLoadContext("test2.rb")
	defer c2.Delete()

	// global variable is accessible from other context
	res, err = c2.Load(`$global`)
	c.Check(res, Equals, 1)
	c.Check(err, IsNil)

	// local is not
	res, err = c2.Load(`local`)
	c.Check(res, Equals, nil)
	c.Check(err, DeepEquals, errors.New("test2.rb:1: undefined method 'local' for main (NoMethodError)"))
}

func (f *F) TestBugRb(c *C) {
	b, err := ioutil.ReadFile("test.rb")
	c.Assert(err, IsNil)
	res, err := f.c.Load(string(b))
	c.Logf("%#v", res)
	c.Check(err, IsNil)
}
<|endoftext|>


<commit_before>package gottyclient

import (
	"crypto/tls"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"regexp"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/creack/goselect"
	"github.com/gorilla/websocket"
	"golang.org/x/crypto/ssh/terminal"
)

// GetAuthTokenURL transforms a GoTTY http URL to its AuthToken file URL
func GetAuthTokenURL(httpURL string) (*url.URL, *http.Header, error) {
	header := http.Header{}
	target, err := url.Parse(httpURL)
	if err != nil {
		return nil, nil, err
	}

	target.Path = strings.TrimLeft(target.Path+"auth_token.js", "/")

	if target.User != nil {
		header.Add("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(target.User.String())))
		target.User = nil
	}

	return target, &header, nil
}

// GetURLQuery returns url.query
func GetURLQuery(rawurl string) (url.Values, error) {
	target, err := url.Parse(rawurl)
	if err != nil {
		return nil, err
	}
	return target.Query(), nil
}

// GetWebsocketURL transforms a GoTTY http URL to its WebSocket URL
func GetWebsocketURL(httpURL string) (*url.URL, *http.Header, error) {
	header := http.Header{}
	target, err := url.Parse(httpURL)
	if err != nil {
		return nil, nil, err
	}

	if target.Scheme == "https" {
		target.Scheme = "wss"
	} else {
		target.Scheme = "ws"
	}

	target.Path = strings.TrimLeft(target.Path+"ws", "/")

	if target.User != nil {
		header.Add("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(target.User.String())))
		target.User = nil
	}

	return target, &header, nil
}

type Client struct {
	Dialer          *websocket.Dialer
	Conn            *websocket.Conn
	URL             string
	WriteMutex      *sync.Mutex
	Output          io.Writer
	QuitChan        chan struct{}
	QuitChanClosed  int32 // atomic value
	SkipTLSVerify   bool
	UseProxyFromEnv bool
	Connected       bool
}

type querySingleType struct {
	AuthToken string `json:"AuthToken"`
	Arguments string `json:"Arguments"`
}

func (c *Client) write(data []byte) error {
	c.WriteMutex.Lock()
	defer c.WriteMutex.Unlock()
	return c.Conn.WriteMessage(websocket.TextMessage, data)
}

// GetAuthToken retrieves an Auth Token from dynamic auth_token.js file
func (c *Client) GetAuthToken() (string, error) {
	target, header, err := GetAuthTokenURL(c.URL)
	if err != nil {
		return "", err
	}

	logrus.Debugf("Fetching auth token auth-token: %q", target.String())
	req, err := http.NewRequest("GET", target.String(), nil)
	req.Header = *header
	tr := &http.Transport{}
	if c.SkipTLSVerify {
		conf := &tls.Config{InsecureSkipVerify: true}
		tr.TLSClientConfig = conf
	}
	if c.UseProxyFromEnv {
		tr.Proxy = http.ProxyFromEnvironment
	}
	client := &http.Client{Transport: tr}
	resp, err := client.Do(req)
	if err != nil {
		return "", err
	}

	switch resp.StatusCode {
	case 200:
		// Everything is OK
	default:
		return "", fmt.Errorf("unknown status code: %d (%s)", resp.StatusCode, http.StatusText(resp.StatusCode))
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}

	re := regexp.MustCompile("var gotty_auth_token = '(.*)'")
	output := re.FindStringSubmatch(string(body))
	if len(output) == 0 {
		return "", fmt.Errorf("Cannot fetch GoTTY auth-token, please upgrade your GoTTY server.")
	}

	return output[1], nil
}

// Connect tries to dial a websocket server
func (c *Client) Connect() error {
	// Retrieve AuthToken
	authToken, err := c.GetAuthToken()
	if err != nil {
		return err
	}
	logrus.Debugf("Auth-token: %q", authToken)

	// Open WebSocket connection
	target, header, err := GetWebsocketURL(c.URL)
	if err != nil {
		return err
	}
	logrus.Debugf("Connecting to websocket: %q", target.String())
	if c.SkipTLSVerify {
		c.Dialer.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
	}
	if c.UseProxyFromEnv {
		c.Dialer.Proxy = http.ProxyFromEnvironment
	}
	conn, _, err := c.Dialer.Dial(target.String(), *header)
	if err != nil {
		return err
	}
	c.Conn = conn
	c.Connected = true

	// Pass arguments and auth-token
	query, err := GetURLQuery(c.URL)
	if err != nil {
		return err
	}
	querySingle := querySingleType{
		Arguments: "?" + query.Encode(),
		AuthToken: authToken,
	}
	json, err := json.Marshal(querySingle)
	if err != nil {
		logrus.Errorf("Failed to parse init message %v", err)
		return err
	}
	// Send Json
	logrus.Debugf("Sending arguments and auth-token")
	err = c.write(json)
	if err != nil {
		return err
	}

	go c.pingLoop()

	return nil
}

func (c *Client) pingLoop() {
	for {
		logrus.Debugf("Sending ping")
		c.write([]byte("1"))
		time.Sleep(30 * time.Second)
	}
}

// Close will nicely close the dialer
func (c *Client) Close() {
	c.Conn.Close()
}

// ExitLoop will kill all goroutine
// ExitLoop() -> wait Loop() -> Close()
func (c *Client) ExitLoop() {
	if atomic.CompareAndSwapInt32(&c.QuitChanClosed, 0, 1) {
		close(c.QuitChan)
	}
}

// Loop will look indefinitely for new messages
func (c *Client) Loop() error {
	if !c.Connected {
		err := c.Connect()
		if err != nil {
			return err
		}
	}

	var wg sync.WaitGroup
	done := make(chan bool)

	wg.Add(1)
	go c.termsizeLoop(&wg)
	wg.Add(1)
	go c.readLoop(done, &wg)
	wg.Add(1)
	go c.writeLoop(done, &wg)
	select {
	case <-done:
		if atomic.CompareAndSwapInt32(&c.QuitChanClosed, 0, 1) {
			close(c.QuitChan)
		}
	case <-c.QuitChan:
	}
	wg.Wait()
	return nil
}

type winsize struct {
	Rows    uint16 `json:"rows"`
	Columns uint16 `json:"columns"`
	// unused
	x uint16
	y uint16
}

func (c *Client) termsizeLoop(wg *sync.WaitGroup) {
	defer wg.Done()
	ch := make(chan os.Signal, 1)
	notifySignalSIGWINCH(ch)
	defer resetSignalSIGWINCH()

	for {
		if b, err := syscallTIOCGWINSZ(); err != nil {
			logrus.Warn(err)
		} else {
			if err = c.write(append([]byte("2"), b...)); err != nil {
				logrus.Warnf("ws.WriteMessage failed: %v", err)
			}
		}
		select {
		case <-c.QuitChan:
			return
		case <-ch:
		}
	}
}

type exposeFd interface {
	Fd() uintptr
}

func (c *Client) writeLoop(done chan bool, wg *sync.WaitGroup) {
	defer wg.Done()

	buff := make([]byte, 128)
	oldState, err := terminal.MakeRaw(0)
	if err == nil {
		defer terminal.Restore(0, oldState)
	}

	rdfs := &goselect.FDSet{}
	reader := io.Reader(os.Stdin)
	for {
		select {
		case <-c.QuitChan:
			return
		default:
		}

		rdfs.Zero()
		rdfs.Set(reader.(exposeFd).Fd())
		err := goselect.Select(1, rdfs, nil, nil, 50*time.Millisecond)
		if err != nil {
			done <- true
			return
		}
		if rdfs.IsSet(reader.(exposeFd).Fd()) {
			size, err := reader.Read(buff)

			if err != nil {
				if err == io.EOF {
					// Send EOF to GoTTY

					// Send 'Input' marker, as defined in GoTTY::client_context.go,
					// followed by EOT (a translation of Ctrl-D for terminals)
					err = c.write(append([]byte("0"), byte(4)))

					if err != nil {
						done <- true
						return
					}
					continue
				} else {
					done <- true
					return
				}
			}

			if size <= 0 {
				continue
			}

			data := buff[:size]
			err = c.write(append([]byte("0"), data...))
			if err != nil {
				done <- true
				return
			}
		}
	}
}

func (c *Client) readLoop(done chan bool, wg *sync.WaitGroup) {
	defer wg.Done()

	type MessageNonBlocking struct {
		Data []byte
		Err  error
	}
	msgChan := make(chan MessageNonBlocking)

	for {
		go func() {
			_, data, err := c.Conn.ReadMessage()
			msgChan <- MessageNonBlocking{Data: data, Err: err}
		}()

		select {
		case <-c.QuitChan:
			return
		case msg := <-msgChan:
			if msg.Err != nil {
				done <- true
				if _, ok := msg.Err.(*websocket.CloseError); !ok {
					logrus.Warnf("c.Conn.ReadMessage: %v", msg.Err)
				}
				return
			}
			if len(msg.Data) == 0 {
				done <- true
				logrus.Warnf("An error has occured")
				return
			}
			switch msg.Data[0] {
			case '0': // data
				buf, err := base64.StdEncoding.DecodeString(string(msg.Data[1:]))
				if err != nil {
					logrus.Warnf("Invalid base64 content: %q", msg.Data[1:])
					break
				}
				c.Output.Write(buf)
			case '1': // pong
			case '2': // new title
				newTitle := string(msg.Data[1:])
				fmt.Fprintf(c.Output, "\033]0;%s\007", newTitle)
			case '3': // json prefs
				logrus.Debugf("Unhandled protocol message: json pref: %s", string(msg.Data[1:]))
			case '4': // autoreconnect
				logrus.Debugf("Unhandled protocol message: autoreconnect: %s", string(msg.Data))
			default:
				logrus.Warnf("Unhandled protocol message: %s", string(msg.Data))
			}
		}
	}
}

// SetOutput changes the output stream
func (c *Client) SetOutput(w io.Writer) {
	c.Output = w
}

// ParseURL parses an URL which may be incomplete and tries to standardize it
func ParseURL(input string) (string, error) {
	parsed, err := url.Parse(input)
	if err != nil {
		return "", err
	}
	switch parsed.Scheme {
	case "http", "https":
		// everything is ok
	default:
		return ParseURL(fmt.Sprintf("http://%s", input))
	}
	return parsed.String(), nil
}

// NewClient returns a GoTTY client object
func NewClient(inputURL string) (*Client, error) {
	url, err := ParseURL(inputURL)
	if err != nil {
		return nil, err
	}
	return &Client{
		Dialer:     &websocket.Dialer{},
		URL:        url,
		WriteMutex: &sync.Mutex{},
		Output:     os.Stdout,
		QuitChan:   make(chan struct{}),
	}, nil
}
<commit_msg>Implement poison-pill method to exit infinite loops<commit_after>package gottyclient

import (
	"crypto/tls"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"regexp"
	"strings"
	"sync"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/creack/goselect"
	"github.com/gorilla/websocket"
	"golang.org/x/crypto/ssh/terminal"
)

// GetAuthTokenURL transforms a GoTTY http URL to its AuthToken file URL
func GetAuthTokenURL(httpURL string) (*url.URL, *http.Header, error) {
	header := http.Header{}
	target, err := url.Parse(httpURL)
	if err != nil {
		return nil, nil, err
	}

	target.Path = strings.TrimLeft(target.Path+"auth_token.js", "/")

	if target.User != nil {
		header.Add("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(target.User.String())))
		target.User = nil
	}

	return target, &header, nil
}

// GetURLQuery returns url.query
func GetURLQuery(rawurl string) (url.Values, error) {
	target, err := url.Parse(rawurl)
	if err != nil {
		return nil, err
	}
	return target.Query(), nil
}

// GetWebsocketURL transforms a GoTTY http URL to its WebSocket URL
func GetWebsocketURL(httpURL string) (*url.URL, *http.Header, error) {
	header := http.Header{}
	target, err := url.Parse(httpURL)
	if err != nil {
		return nil, nil, err
	}

	if target.Scheme == "https" {
		target.Scheme = "wss"
	} else {
		target.Scheme = "ws"
	}

	target.Path = strings.TrimLeft(target.Path+"ws", "/")

	if target.User != nil {
		header.Add("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(target.User.String())))
		target.User = nil
	}

	return target, &header, nil
}

type Client struct {
	Dialer          *websocket.Dialer
	Conn            *websocket.Conn
	URL             string
	WriteMutex      *sync.Mutex
	Output          io.Writer
	poison          chan bool
	SkipTLSVerify   bool
	UseProxyFromEnv bool
	Connected       bool
}

type querySingleType struct {
	AuthToken string `json:"AuthToken"`
	Arguments string `json:"Arguments"`
}

func (c *Client) write(data []byte) error {
	c.WriteMutex.Lock()
	defer c.WriteMutex.Unlock()
	return c.Conn.WriteMessage(websocket.TextMessage, data)
}

// GetAuthToken retrieves an Auth Token from dynamic auth_token.js file
func (c *Client) GetAuthToken() (string, error) {
	target, header, err := GetAuthTokenURL(c.URL)
	if err != nil {
		return "", err
	}

	logrus.Debugf("Fetching auth token auth-token: %q", target.String())
	req, err := http.NewRequest("GET", target.String(), nil)
	req.Header = *header
	tr := &http.Transport{}
	if c.SkipTLSVerify {
		conf := &tls.Config{InsecureSkipVerify: true}
		tr.TLSClientConfig = conf
	}
	if c.UseProxyFromEnv {
		tr.Proxy = http.ProxyFromEnvironment
	}
	client := &http.Client{Transport: tr}
	resp, err := client.Do(req)
	if err != nil {
		return "", err
	}

	switch resp.StatusCode {
	case 200:
		// Everything is OK
	default:
		return "", fmt.Errorf("unknown status code: %d (%s)", resp.StatusCode, http.StatusText(resp.StatusCode))
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}

	re := regexp.MustCompile("var gotty_auth_token = '(.*)'")
	output := re.FindStringSubmatch(string(body))
	if len(output) == 0 {
		return "", fmt.Errorf("Cannot fetch GoTTY auth-token, please upgrade your GoTTY server.")
	}

	return output[1], nil
}

// Connect tries to dial a websocket server
func (c *Client) Connect() error {
	// Retrieve AuthToken
	authToken, err := c.GetAuthToken()
	if err != nil {
		return err
	}
	logrus.Debugf("Auth-token: %q", authToken)

	// Open WebSocket connection
	target, header, err := GetWebsocketURL(c.URL)
	if err != nil {
		return err
	}
	logrus.Debugf("Connecting to websocket: %q", target.String())
	if c.SkipTLSVerify {
		c.Dialer.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
	}
	if c.UseProxyFromEnv {
		c.Dialer.Proxy = http.ProxyFromEnvironment
	}
	conn, _, err := c.Dialer.Dial(target.String(), *header)
	if err != nil {
		return err
	}
	c.Conn = conn
	c.Connected = true

	// Pass arguments and auth-token
	query, err := GetURLQuery(c.URL)
	if err != nil {
		return err
	}
	querySingle := querySingleType{
		Arguments: "?" + query.Encode(),
		AuthToken: authToken,
	}
	json, err := json.Marshal(querySingle)
	if err != nil {
		logrus.Errorf("Failed to parse init message %v", err)
		return err
	}
	// Send Json
	logrus.Debugf("Sending arguments and auth-token")
	err = c.write(json)
	if err != nil {
		return err
	}

	go c.pingLoop()

	return nil
}

func (c *Client) pingLoop() {
	for {
		logrus.Debugf("Sending ping")
		c.write([]byte("1"))
		time.Sleep(30 * time.Second)
	}
}

// Close will nicely close the dialer
func (c *Client) Close() {
	c.Conn.Close()
}

// ExitLoop will kill all goroutines launched by c.Loop()
// ExitLoop() -> wait Loop() -> Close()
func (c *Client) ExitLoop() {
	fname := "ExitLoop"
	openPoison(fname, c.poison)
}

// Loop will look indefinitely for new messages
func (c *Client) Loop() error {
	if !c.Connected {
		err := c.Connect()
		if err != nil {
			return err
		}
	}

	wg := &sync.WaitGroup{}

	wg.Add(1)
	go c.termsizeLoop(wg)

	wg.Add(1)
	go c.readLoop(wg)

	wg.Add(1)
	go c.writeLoop(wg)

	/* Wait for all of the above goroutines to finish */
	wg.Wait()

	logrus.Debug("Client.Loop() exiting")
	return nil
}

type winsize struct {
	Rows    uint16 `json:"rows"`
	Columns uint16 `json:"columns"`
	// unused
	x uint16
	y uint16
}

type posionReason int

const (
	committedSuicide = iota
	killed
)

func openPoison(fname string, poison chan bool) posionReason {
	logrus.Debug(fname + " suicide")

	/*
	 * The close() may raise panic if multiple goroutines commit suicide at the
	 * same time. Prevent that panic from bubbling up.
	 */
	defer func() {
		if r := recover(); r != nil {
			logrus.Debug("Prevented panic() of simultaneous suicides", r)
		}
	}()

	/* Signal others to die */
	close(poison)
	return committedSuicide
}

func die(fname string, poison chan bool) posionReason {
	logrus.Debug(fname + " died")

	wasOpen := <-poison
	if wasOpen {
		logrus.Error("ERROR: The channel was open when it wasn't suppoed to be")
	}

	return killed
}

func (c *Client) termsizeLoop(wg *sync.WaitGroup) posionReason {

	defer wg.Done()
	fname := "termsizeLoop"

	ch := make(chan os.Signal, 1)
	notifySignalSIGWINCH(ch)
	defer resetSignalSIGWINCH()

	for {
		if b, err := syscallTIOCGWINSZ(); err != nil {
			logrus.Warn(err)
		} else {
			if err = c.write(append([]byte("2"), b...)); err != nil {
				logrus.Warnf("ws.WriteMessage failed: %v", err)
			}
		}
		select {
		case <-c.poison:
			/* Somebody poisoned the well; die */
			return die(fname, c.poison)
		case <-ch:
		}
	}
}

type exposeFd interface {
	Fd() uintptr
}

func (c *Client) writeLoop(wg *sync.WaitGroup) posionReason {

	defer wg.Done()
	fname := "writeLoop"

	buff := make([]byte, 128)
	oldState, err := terminal.MakeRaw(0)
	if err == nil {
		defer terminal.Restore(0, oldState)
	}

	rdfs := &goselect.FDSet{}
	reader := io.Reader(os.Stdin)
	for {
		select {
		case <-c.poison:
			/* Somebody poisoned the well; die */
			return die(fname, c.poison)
		default:
		}

		rdfs.Zero()
		rdfs.Set(reader.(exposeFd).Fd())
		err := goselect.Select(1, rdfs, nil, nil, 50*time.Millisecond)
		if err != nil {
			return openPoison(fname, c.poison)
		}
		if rdfs.IsSet(reader.(exposeFd).Fd()) {
			size, err := reader.Read(buff)

			if err != nil {
				if err == io.EOF {
					// Send EOF to GoTTY

					// Send 'Input' marker, as defined in GoTTY::client_context.go,
					// followed by EOT (a translation of Ctrl-D for terminals)
					err = c.write(append([]byte("0"), byte(4)))

					if err != nil {
						return openPoison(fname, c.poison)
					}
					continue
				} else {
					return openPoison(fname, c.poison)
				}
			}

			if size <= 0 {
				continue
			}

			data := buff[:size]
			err = c.write(append([]byte("0"), data...))
			if err != nil {
				return openPoison(fname, c.poison)
			}
		}
	}
}

func (c *Client) readLoop(wg *sync.WaitGroup) posionReason {

	defer wg.Done()
	fname := "readLoop"

	type MessageNonBlocking struct {
		Data []byte
		Err  error
	}
	msgChan := make(chan MessageNonBlocking)

	for {
		go func() {
			_, data, err := c.Conn.ReadMessage()
			msgChan <- MessageNonBlocking{Data: data, Err: err}
		}()

		select {
		case <-c.poison:
			/* Somebody poisoned the well; die */
			return die(fname, c.poison)
		case msg := <-msgChan:
			if msg.Err != nil {

				if _, ok := msg.Err.(*websocket.CloseError); !ok {
					logrus.Warnf("c.Conn.ReadMessage: %v", msg.Err)
				}
				return openPoison(fname, c.poison)
			}
			if len(msg.Data) == 0 {

				logrus.Warnf("An error has occured")
				return openPoison(fname, c.poison)
			}
			switch msg.Data[0] {
			case '0': // data
				buf, err := base64.StdEncoding.DecodeString(string(msg.Data[1:]))
				if err != nil {
					logrus.Warnf("Invalid base64 content: %q", msg.Data[1:])
					break
				}
				c.Output.Write(buf)
			case '1': // pong
			case '2': // new title
				newTitle := string(msg.Data[1:])
				fmt.Fprintf(c.Output, "\033]0;%s\007", newTitle)
			case '3': // json prefs
				logrus.Debugf("Unhandled protocol message: json pref: %s", string(msg.Data[1:]))
			case '4': // autoreconnect
				logrus.Debugf("Unhandled protocol message: autoreconnect: %s", string(msg.Data))
			default:
				logrus.Warnf("Unhandled protocol message: %s", string(msg.Data))
			}
		}
	}
}

// SetOutput changes the output stream
func (c *Client) SetOutput(w io.Writer) {
	c.Output = w
}

// ParseURL parses an URL which may be incomplete and tries to standardize it
func ParseURL(input string) (string, error) {
	parsed, err := url.Parse(input)
	if err != nil {
		return "", err
	}
	switch parsed.Scheme {
	case "http", "https":
		// everything is ok
	default:
		return ParseURL(fmt.Sprintf("http://%s", input))
	}
	return parsed.String(), nil
}

// NewClient returns a GoTTY client object
func NewClient(inputURL string) (*Client, error) {
	url, err := ParseURL(inputURL)
	if err != nil {
		return nil, err
	}
	return &Client{
		Dialer:     &websocket.Dialer{},
		URL:        url,
		WriteMutex: &sync.Mutex{},
		Output:     os.Stdout,
		poison:     make(chan bool),
	}, nil
}
<|endoftext|>


<commit_before>package rpcserver

import (
	"context"
	"fmt"
	"github.com/getlantern/bytemap"
	"github.com/getlantern/errors"
	"github.com/getlantern/golog"
	"github.com/getlantern/wal"
	"github.com/getlantern/zenodb"
	"github.com/getlantern/zenodb/common"
	"github.com/getlantern/zenodb/core"
	"github.com/getlantern/zenodb/encoding"
	"github.com/getlantern/zenodb/planner"
	"github.com/getlantern/zenodb/rpc"
	"google.golang.org/grpc"
	"google.golang.org/grpc/metadata"
	"net"
	"time"
)

var (
	log = golog.LoggerFor("zenodb.rpc")
)

type Opts struct {
	// Password, if specified, is the password that clients must present in order
	// to access the server.
	Password string
}

// DB is an interface for database-like things (implemented by common.DB).
type DB interface {
	InsertRaw(stream string, ts time.Time, dims bytemap.ByteMap, vals bytemap.ByteMap) error

	Query(sqlString string, isSubQuery bool, subQueryResults [][]interface{}, includeMemStore bool) (core.FlatRowSource, error)

	Follow(f *common.Follow, cb func([]byte, wal.Offset) error)

	RegisterQueryHandler(partition int, query planner.QueryClusterFN)
}

func Serve(db DB, l net.Listener, opts *Opts) error {
	l = &rpc.SnappyListener{l}
	gs := grpc.NewServer(grpc.CustomCodec(rpc.Codec))
	gs.RegisterService(&rpc.ServiceDesc, &server{db, opts.Password})
	return gs.Serve(l)
}

type server struct {
	db       DB
	password string
}

func (s *server) Insert(stream grpc.ServerStream) error {
	// No need to authorize, anyone can insert

	now := time.Now()
	streamName := ""

	report := &rpc.InsertReport{
		Errors: make(map[int]string),
	}

	i := -1
	for {
		i++
		insert := &rpc.Insert{}
		err := stream.RecvMsg(insert)
		if err != nil {
			return fmt.Errorf("Error reading insert: %v", err)
		}
		if insert.EndOfInserts {
			// We're done inserting
			return stream.SendMsg(report)
		}
		report.Received++

		if streamName == "" {
			streamName = insert.Stream
			if streamName == "" {
				return fmt.Errorf("Please specify a stream")
			}
		}

		if len(insert.Dims) == 0 {
			report.Errors[i] = fmt.Sprintf("Need at least one dim")
			continue
		}
		if len(insert.Vals) == 0 {
			report.Errors[i] = fmt.Sprintf("Need at least one val")
			continue
		}
		var ts time.Time
		if insert.TS == 0 {
			ts = now
		} else {
			ts = encoding.TimeFromInt(insert.TS)
		}

		// TODO: make sure we don't barf on invalid bytemaps here
		insertErr := s.db.InsertRaw(streamName, ts, bytemap.ByteMap(insert.Dims), bytemap.ByteMap(insert.Vals))
		if insertErr != nil {
			report.Errors[i] = fmt.Sprintf("Unable to insert: %v", insertErr)
			continue
		}
		report.Succeeded++
	}
}

func (s *server) Query(q *rpc.Query, stream grpc.ServerStream) error {
	authorizeErr := s.authorize(stream)
	if authorizeErr != nil {
		return authorizeErr
	}

	source, err := s.db.Query(q.SQLString, q.IsSubQuery, q.SubQueryResults, q.IncludeMemStore)
	if err != nil {
		return err
	}

	rr := &rpc.RemoteQueryResult{}
	stats, err := source.Iterate(stream.Context(), func(fields core.Fields) error {
		// Send query metadata
		md := zenodb.MetaDataFor(source, fields)
		return stream.SendMsg(md)
	}, func(row *core.FlatRow) (bool, error) {
		rr.Row = row
		return true, stream.SendMsg(rr)
	})
	if err != nil {
		return err
	}

	// Send end of results
	rr.Row = nil
	if stats != nil {
		rr.Stats = stats.(*common.QueryStats)
	}
	rr.EndOfResults = true
	return stream.SendMsg(rr)
}

func (s *server) Follow(f *common.Follow, stream grpc.ServerStream) error {
	authorizeErr := s.authorize(stream)
	if authorizeErr != nil {
		return authorizeErr
	}

	log.Debugf("Follower %d joined", f.PartitionNumber)
	defer log.Debugf("Follower %d left", f.PartitionNumber)
	s.db.Follow(f, func(data []byte, newOffset wal.Offset) error {
		return stream.SendMsg(&rpc.Point{data, newOffset})
	})
	return nil
}

func (s *server) HandleRemoteQueries(r *rpc.RegisterQueryHandler, stream grpc.ServerStream) error {
	initialResultCh := make(chan *rpc.RemoteQueryResult)
	initialErrCh := make(chan error)
	finalErrCh := make(chan error)

	finish := func(err error) {
		select {
		case finalErrCh <- err:
			// ok
		default:
			// ignore
		}
	}

	s.db.RegisterQueryHandler(r.Partition, func(ctx context.Context, sqlString string, isSubQuery bool, subQueryResults [][]interface{}, unflat bool, onFields core.OnFields, onRow core.OnRow, onFlatRow core.OnFlatRow) (interface{}, error) {
		q := &rpc.Query{
			SQLString:       sqlString,
			IsSubQuery:      isSubQuery,
			SubQueryResults: subQueryResults,
			Unflat:          unflat,
			IncludeMemStore: common.ShouldIncludeMemStore(ctx),
		}
		q.Deadline, q.HasDeadline = ctx.Deadline()
		sendErr := stream.SendMsg(q)

		m, recvErr := <-initialResultCh, <-initialErrCh

		// Check send error after reading initial result to avoid blocking
		// unnecessarily
		if sendErr != nil {
			err := errors.New("Unable to send query: %v", sendErr)
			finish(err)
			return nil, err
		}

		var finalErr error

		first := true
	receiveLoop:
		for {
			// Process current result
			if recvErr != nil {
				m.Error = recvErr.Error()
				finalErr = errors.New("Unable to receive result: %v", recvErr)
				break
			}

			log.Debugf("First? %v  Response Error?: %v", first, m.Error)

			if first {
				// First message contains only fields information
				onFields(m.Fields)
				first = false
			} else {
				// Subsequent messages contain data
				if m.EndOfResults {
					break
				}
				var more bool
				var err error
				if unflat {
					more, err = onRow(m.Key, m.Vals)
				} else {
					more, err = onFlatRow(m.Row)
				}
				if !more || err != nil {
					finalErr = err
					break receiveLoop
				}
			}

			// Read next result
			m = &rpc.RemoteQueryResult{}
			recvErr = stream.RecvMsg(m)
		}

		finish(finalErr)
		return m.Stats, finalErr
	})

	// Block on reading initial result to keep connection open
	m := &rpc.RemoteQueryResult{}
	err := stream.RecvMsg(m)
	initialResultCh <- m
	initialErrCh <- err

	if err == nil {
		// Wait for final error so we don't close the connection prematurely
		return <-finalErrCh
	}
	return err
}

func (s *server) authorize(stream grpc.ServerStream) error {
	if s.password == "" {
		log.Debug("No password specified, allowing access to world")
		return nil
	}
	md, ok := metadata.FromIncomingContext(stream.Context())
	if !ok {
		return log.Error("No metadata provided, unable to authenticate")
	}
	passwords := md[rpc.PasswordKey]
	for _, password := range passwords {
		if password == s.password {
			// authorized
			return nil
		}
	}
	return log.Error("None of the provided passwords matched, not authorized!")
}
<commit_msg>Added more logging for query error handling<commit_after>package rpcserver

import (
	"context"
	"fmt"
	"github.com/getlantern/bytemap"
	"github.com/getlantern/errors"
	"github.com/getlantern/golog"
	"github.com/getlantern/wal"
	"github.com/getlantern/zenodb"
	"github.com/getlantern/zenodb/common"
	"github.com/getlantern/zenodb/core"
	"github.com/getlantern/zenodb/encoding"
	"github.com/getlantern/zenodb/planner"
	"github.com/getlantern/zenodb/rpc"
	"google.golang.org/grpc"
	"google.golang.org/grpc/metadata"
	"net"
	"time"
)

var (
	log = golog.LoggerFor("zenodb.rpc")
)

type Opts struct {
	// Password, if specified, is the password that clients must present in order
	// to access the server.
	Password string
}

// DB is an interface for database-like things (implemented by common.DB).
type DB interface {
	InsertRaw(stream string, ts time.Time, dims bytemap.ByteMap, vals bytemap.ByteMap) error

	Query(sqlString string, isSubQuery bool, subQueryResults [][]interface{}, includeMemStore bool) (core.FlatRowSource, error)

	Follow(f *common.Follow, cb func([]byte, wal.Offset) error)

	RegisterQueryHandler(partition int, query planner.QueryClusterFN)
}

func Serve(db DB, l net.Listener, opts *Opts) error {
	l = &rpc.SnappyListener{l}
	gs := grpc.NewServer(grpc.CustomCodec(rpc.Codec))
	gs.RegisterService(&rpc.ServiceDesc, &server{db, opts.Password})
	return gs.Serve(l)
}

type server struct {
	db       DB
	password string
}

func (s *server) Insert(stream grpc.ServerStream) error {
	// No need to authorize, anyone can insert

	now := time.Now()
	streamName := ""

	report := &rpc.InsertReport{
		Errors: make(map[int]string),
	}

	i := -1
	for {
		i++
		insert := &rpc.Insert{}
		err := stream.RecvMsg(insert)
		if err != nil {
			return fmt.Errorf("Error reading insert: %v", err)
		}
		if insert.EndOfInserts {
			// We're done inserting
			return stream.SendMsg(report)
		}
		report.Received++

		if streamName == "" {
			streamName = insert.Stream
			if streamName == "" {
				return fmt.Errorf("Please specify a stream")
			}
		}

		if len(insert.Dims) == 0 {
			report.Errors[i] = fmt.Sprintf("Need at least one dim")
			continue
		}
		if len(insert.Vals) == 0 {
			report.Errors[i] = fmt.Sprintf("Need at least one val")
			continue
		}
		var ts time.Time
		if insert.TS == 0 {
			ts = now
		} else {
			ts = encoding.TimeFromInt(insert.TS)
		}

		// TODO: make sure we don't barf on invalid bytemaps here
		insertErr := s.db.InsertRaw(streamName, ts, bytemap.ByteMap(insert.Dims), bytemap.ByteMap(insert.Vals))
		if insertErr != nil {
			report.Errors[i] = fmt.Sprintf("Unable to insert: %v", insertErr)
			continue
		}
		report.Succeeded++
	}
}

func (s *server) Query(q *rpc.Query, stream grpc.ServerStream) error {
	authorizeErr := s.authorize(stream)
	if authorizeErr != nil {
		return authorizeErr
	}

	source, err := s.db.Query(q.SQLString, q.IsSubQuery, q.SubQueryResults, q.IncludeMemStore)
	if err != nil {
		return err
	}

	rr := &rpc.RemoteQueryResult{}
	stats, err := source.Iterate(stream.Context(), func(fields core.Fields) error {
		// Send query metadata
		md := zenodb.MetaDataFor(source, fields)
		return stream.SendMsg(md)
	}, func(row *core.FlatRow) (bool, error) {
		rr.Row = row
		return true, stream.SendMsg(rr)
	})
	if err != nil {
		return err
	}

	// Send end of results
	rr.Row = nil
	if stats != nil {
		rr.Stats = stats.(*common.QueryStats)
	}
	rr.EndOfResults = true
	return stream.SendMsg(rr)
}

func (s *server) Follow(f *common.Follow, stream grpc.ServerStream) error {
	authorizeErr := s.authorize(stream)
	if authorizeErr != nil {
		return authorizeErr
	}

	log.Debugf("Follower %d joined", f.PartitionNumber)
	defer log.Debugf("Follower %d left", f.PartitionNumber)
	s.db.Follow(f, func(data []byte, newOffset wal.Offset) error {
		return stream.SendMsg(&rpc.Point{data, newOffset})
	})
	return nil
}

func (s *server) HandleRemoteQueries(r *rpc.RegisterQueryHandler, stream grpc.ServerStream) error {
	initialResultCh := make(chan *rpc.RemoteQueryResult)
	initialErrCh := make(chan error)
	finalErrCh := make(chan error)

	finish := func(err error) {
		select {
		case finalErrCh <- err:
			// ok
			log.Debugf("Posted final err: %v", err)
		default:
			// ignore
			log.Debug("Already had final err!")
		}
	}

	s.db.RegisterQueryHandler(r.Partition, func(ctx context.Context, sqlString string, isSubQuery bool, subQueryResults [][]interface{}, unflat bool, onFields core.OnFields, onRow core.OnRow, onFlatRow core.OnFlatRow) (interface{}, error) {
		q := &rpc.Query{
			SQLString:       sqlString,
			IsSubQuery:      isSubQuery,
			SubQueryResults: subQueryResults,
			Unflat:          unflat,
			IncludeMemStore: common.ShouldIncludeMemStore(ctx),
		}
		q.Deadline, q.HasDeadline = ctx.Deadline()
		sendErr := stream.SendMsg(q)

		m, recvErr := <-initialResultCh, <-initialErrCh

		// Check send error after reading initial result to avoid blocking
		// unnecessarily
		if sendErr != nil {
			err := errors.New("Unable to send query: %v", sendErr)
			finish(err)
			return nil, err
		}

		var finalErr error

		first := true
	receiveLoop:
		for {
			// Process current result
			if recvErr != nil {
				m.Error = recvErr.Error()
				finalErr = errors.New("Unable to receive result: %v", recvErr)
				break
			}

			if first {
				// First message contains only fields information
				onFields(m.Fields)
				first = false
			} else {
				// Subsequent messages contain data
				if m.EndOfResults {
					break
				}
				var more bool
				var err error
				if unflat {
					more, err = onRow(m.Key, m.Vals)
				} else {
					more, err = onFlatRow(m.Row)
				}
				if !more || err != nil {
					finalErr = err
					break receiveLoop
				}
			}

			// Read next result
			m = &rpc.RemoteQueryResult{}
			recvErr = stream.RecvMsg(m)
		}

		finish(finalErr)
		return m.Stats, finalErr
	})

	// Block on reading initial result to keep connection open
	m := &rpc.RemoteQueryResult{}
	err := stream.RecvMsg(m)
	initialResultCh <- m
	initialErrCh <- err

	if err == nil {
		// Wait for final error so we don't close the connection prematurely
		err = <-finalErrCh
	}
	log.Debugf("Returning err?: %v", err)
	return err
}

func (s *server) authorize(stream grpc.ServerStream) error {
	if s.password == "" {
		log.Debug("No password specified, allowing access to world")
		return nil
	}
	md, ok := metadata.FromIncomingContext(stream.Context())
	if !ok {
		return log.Error("No metadata provided, unable to authenticate")
	}
	passwords := md[rpc.PasswordKey]
	for _, password := range passwords {
		if password == s.password {
			// authorized
			return nil
		}
	}
	return log.Error("None of the provided passwords matched, not authorized!")
}
<|endoftext|>


<commit_before>package main

import (
	"os"

	"github.com/alecthomas/kingpin"
	"github.com/couchbaselabs/logg"
)

/*
Tool for converting input data from one form to another
*/

var (
	app         = kingpin.New("data converter", "A command-line tool for converting data.")
	flat2folder = app.Command("flat2folder", "Take a flat list of files + an index and convert to folder labels")

	sourceDir = flat2folder.Flag("sourceDir", "Directory with source files").String()
	index     = flat2folder.Flag("labelIndex", "File with file -> label mapping").String()
	destDir   = flat2folder.Flag("destDir", "Destination directory for output folders").String()
)

func init() {
	logg.LogKeys["DATA_CONVERTER"] = true
}

func main() {
	switch kingpin.MustParse(app.Parse(os.Args[1:])) {
	case "flat2folder":
		logg.LogTo("DATA_CONVERTER", "do flat2folder")
	default:
		kingpin.UsageErrorf("Invalid / missing command")
	}
}
<commit_msg>loops over lines<commit_after>package main

import (
	"bufio"
	"fmt"
	"log"
	"os"

	"github.com/alecthomas/kingpin"
	"github.com/couchbaselabs/logg"
)

/*
Tool for converting input data from one form to another
*/

var (
	app         = kingpin.New("data converter", "A command-line tool for converting data.")
	flat2folder = app.Command("flat2folder", "Take a flat list of files + an index and convert to folder labels")

	sourceDir  = flat2folder.Flag("sourceDir", "Directory with source files").String()
	labelIndex = flat2folder.Flag("labelIndex", "File with file -> label mapping").String()
	destDir    = flat2folder.Flag("destDir", "Destination directory for output folders").String()
)

func init() {
	logg.LogKeys["DATA_CONVERTER"] = true
}

func main() {
	switch kingpin.MustParse(app.Parse(os.Args[1:])) {
	case "flat2folder":
		logg.LogTo("DATA_CONVERTER", "do flat2folder")
		doFlat2folder(*sourceDir, *labelIndex, *destDir)
	default:
		kingpin.UsageErrorf("Invalid / missing command")
	}
}

func processLabelIndex(labelIndex string, lineHandler func(string) error) error {

	// open labelIndex
	file, err := os.Open(labelIndex)
	if err != nil {
		return err
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		if err := lineHandler(scanner.Text()); err != nil {
			return err
		}
	}
	if err := scanner.Err(); err != nil {
		return err
	}

	return nil

}

func doFlat2folder(sourceDir, labelIndex, destDir string) {

	// validations: sourceDir and index exist
	validatePathExists(sourceDir)
	validatePathExists(labelIndex)

	labelIndexLineHandler := func(line string) error {
		fmt.Println(line)
		return nil
	}

	if err := processLabelIndex(labelIndex, labelIndexLineHandler); err != nil {
		log.Fatal(err)
	}

	// for each row in index (examples/mnist/mnist_test_files/1.png 2)

	// find the label (eg, 2)

	// create dir for label under sourceDir

	// copy file from sourceDir to destDir/label

}

func validatePathExists(path string) {
	_, err := os.Stat(path)
	if err != nil {
		logg.LogPanic("%v does not exist", path)
	}
}
<|endoftext|>