<commit_before>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package test

import (
	"fmt"
	"os"
	"path/filepath"

	"v.io/jiri/collect"
	"v.io/jiri/project"
	"v.io/jiri/tool"
	"v.io/x/devtools/internal/test"
	"v.io/x/devtools/internal/xunit"
)

var (
	mirrors = []Mirror{
		Mirror{
			name:         "browser",
			googlesource: "https://vanadium.googlesource.com/release.projects.browser",
			github:       "git@github.com:vanadium/browser.git",
		},
		Mirror{
			name:         "chat",
			googlesource: "https://vanadium.googlesource.com/release.projects.chat",
			github:       "git@github.com:vanadium/chat.git",
		},
		Mirror{
			name:         "docs",
			googlesource: "https://vanadium.googlesource.com/docs",
			github:       "git@github.com:vanadium/docs.git",
		},
		Mirror{
			name:         "go.devtools",
			googlesource: "https://vanadium.googlesource.com/release.go.x.devtools",
			github:       "git@github.com:vanadium/go.devtools.git",
		},
		Mirror{
			name:         "go.jiri",
			googlesource: "https://vanadium.googlesource.com/release.go.jiri",
			github:       "git@github.com:vanadium/go.jiri.git",
		},
		Mirror{
			name:         "go.lib",
			googlesource: "https://vanadium.googlesource.com/release.go.x.lib",
			github:       "git@github.com:vanadium/go.lib.git",
		},
		Mirror{
			name:         "go.ref",
			googlesource: "https://vanadium.googlesource.com/release.go.x.ref",
			github:       "git@github.com:vanadium/go.ref.git",
		},
		Mirror{
			name:         "go.v23",
			googlesource: "https://vanadium.googlesource.com/release.go.v23",
			github:       "git@github.com:vanadium/go.v23.git",
		},
		Mirror{
			name:         "java",
			googlesource: "https://vanadium.googlesource.com/release.java",
			github:       "git@github.com:vanadium/java",
		},
		Mirror{
			name:         "js",
			googlesource: "https://vanadium.googlesource.com/release.js.core",
			github:       "git@github.com:vanadium/js.git",
		},
		Mirror{
			name:         "js.syncbase",
			googlesource: "https://vanadium.googlesource.com/release.js.syncbase",
			github:       "git@github.com:vanadium/js.syncbase.git",
		},
		Mirror{
			name:         "mojo.syncbase",
			googlesource: "https://vanadium.googlesource.com/release.mojo.syncbase",
			github:       "git@github.com:vanadium/mojo.syncbase.git",
		},
		Mirror{
			name:         "pipe2browser",
			googlesource: "https://vanadium.googlesource.com/release.projects.pipe2browser",
			github:       "git@github.com:vanadium/pipe2browser.git",
		},
		Mirror{
			name:         "playground",
			googlesource: "https://vanadium.googlesource.com/release.projects.playground",
			github:       "git@github.com:vanadium/playground.git",
		},
		Mirror{
			name:         "reader",
			googlesource: "https://vanadium.googlesource.com/release.projects.reader",
			github:       "git@github.com:vanadium/reader.git",
		},
		Mirror{
			name:         "third_party",
			googlesource: "https://vanadium.googlesource.com/third_party",
			github:       "git@github.com:vanadium/third_party.git",
		},
		Mirror{
			name:         "todos",
			googlesource: "https://vanadium.googlesource.com/release.projects.todos",
			github:       "git@github.com:vanadium/todos.git",
		},
		Mirror{
			name:         "www",
			googlesource: "https://vanadium.googlesource.com/www",
			github:       "git@github.com:vanadium/www.git",
		},
	}
)

type Mirror struct {
	name, googlesource, github string
}

// vanadiumGitHubMirror mirrors googlesource.com vanadium projects to
// github.com.
func vanadiumGitHubMirror(ctx *tool.Context, testName string, _ ...Opt) (_ *test.Result, e error) {
	// Initialize the test/task.
	cleanup, err := initTest(ctx, testName, nil)
	if err != nil {
		return nil, internalTestError{err, "Init"}
	}
	defer collect.Error(func() error { return cleanup() }, &e)

	root, err := project.JiriRoot()
	if err != nil {
		return nil, internalTestError{err, "JiriRoot"}
	}

	projects := filepath.Join(root, "projects")
	mode := os.FileMode(0755)
	if err := ctx.Run().MkdirAll(projects, mode); err != nil {
		return nil, internalTestError{err, "MkdirAll"}
	}

	allPassed := true
	suites := []xunit.TestSuite{}
	for _, mirror := range mirrors {
		suite, err := sync(ctx, mirror, projects)
		if err != nil {
			return nil, internalTestError{err, "sync"}
		}

		allPassed = allPassed && (suite.Failures == 0)
		suites = append(suites, *suite)
	}

	if err := xunit.CreateReport(ctx, testName, suites); err != nil {
		return nil, err
	}

	if !allPassed {
		return &test.Result{Status: test.Failed}, nil
	}

	return &test.Result{Status: test.Passed}, nil
}

func sync(ctx *tool.Context, mirror Mirror, projects string) (*xunit.TestSuite, error) {
	suite := xunit.TestSuite{Name: mirror.name}
	dirname := filepath.Join(projects, mirror.name)

	// If dirname does not exist `git clone` otherwise `git pull`.
	if _, err := ctx.Run().Stat(dirname); err != nil {
		if !os.IsNotExist(err) {
			return nil, internalTestError{err, "stat"}
		}

		err := clone(ctx, mirror, projects)
		testCase := makeTestCase("clone", err)
		if err != nil {
			suite.Failures++
		}
		suite.Cases = append(suite.Cases, *testCase)
	} else {
		err := pull(ctx, mirror, projects)
		testCase := makeTestCase("pull", err)
		if err != nil {
			suite.Failures++
		}
		suite.Cases = append(suite.Cases, *testCase)
	}

	err := push(ctx, mirror, projects)
	testCase := makeTestCase("push", err)
	if err != nil {
		suite.Failures++
	}
	suite.Cases = append(suite.Cases, *testCase)

	return &suite, nil
}

func makeTestCase(action string, err error) *xunit.TestCase {
	c := xunit.TestCase{
		Classname: "git",
		Name:      action,
	}

	if err != nil {
		f := xunit.Failure{
			Message: "git error",
			Data:    fmt.Sprintf("%v", err),
		}
		c.Failures = append(c.Failures, f)
	}

	return &c
}

func clone(ctx *tool.Context, mirror Mirror, projects string) error {
	dirname := filepath.Join(projects, mirror.name)
	return ctx.Git().Clone(mirror.googlesource, dirname)
}

func pull(ctx *tool.Context, mirror Mirror, projects string) error {
	dirname := filepath.Join(projects, mirror.name)
	opts := tool.RootDirOpt(dirname)
	return ctx.Git(opts).Pull("origin", "master")
}

func push(ctx *tool.Context, mirror Mirror, projects string) error {
	dirname := filepath.Join(projects, mirror.name)
	opts := tool.RootDirOpt(dirname)
	return ctx.Git(opts).Push(mirror.github, "master")
}
<commit_msg>TBR Revert "add java mirroring"<commit_after>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package test

import (
	"fmt"
	"os"
	"path/filepath"

	"v.io/jiri/collect"
	"v.io/jiri/project"
	"v.io/jiri/tool"
	"v.io/x/devtools/internal/test"
	"v.io/x/devtools/internal/xunit"
)

var (
	mirrors = []Mirror{
		Mirror{
			name:         "browser",
			googlesource: "https://vanadium.googlesource.com/release.projects.browser",
			github:       "git@github.com:vanadium/browser.git",
		},
		Mirror{
			name:         "chat",
			googlesource: "https://vanadium.googlesource.com/release.projects.chat",
			github:       "git@github.com:vanadium/chat.git",
		},
		Mirror{
			name:         "docs",
			googlesource: "https://vanadium.googlesource.com/docs",
			github:       "git@github.com:vanadium/docs.git",
		},
		Mirror{
			name:         "go.devtools",
			googlesource: "https://vanadium.googlesource.com/release.go.x.devtools",
			github:       "git@github.com:vanadium/go.devtools.git",
		},
		Mirror{
			name:         "go.jiri",
			googlesource: "https://vanadium.googlesource.com/release.go.jiri",
			github:       "git@github.com:vanadium/go.jiri.git",
		},
		Mirror{
			name:         "go.lib",
			googlesource: "https://vanadium.googlesource.com/release.go.x.lib",
			github:       "git@github.com:vanadium/go.lib.git",
		},
		Mirror{
			name:         "go.ref",
			googlesource: "https://vanadium.googlesource.com/release.go.x.ref",
			github:       "git@github.com:vanadium/go.ref.git",
		},
		Mirror{
			name:         "go.v23",
			googlesource: "https://vanadium.googlesource.com/release.go.v23",
			github:       "git@github.com:vanadium/go.v23.git",
		},
		Mirror{
			name:         "js",
			googlesource: "https://vanadium.googlesource.com/release.js.core",
			github:       "git@github.com:vanadium/js.git",
		},
		Mirror{
			name:         "js.syncbase",
			googlesource: "https://vanadium.googlesource.com/release.js.syncbase",
			github:       "git@github.com:vanadium/js.syncbase.git",
		},
		Mirror{
			name:         "mojo.syncbase",
			googlesource: "https://vanadium.googlesource.com/release.mojo.syncbase",
			github:       "git@github.com:vanadium/mojo.syncbase.git",
		},
		Mirror{
			name:         "pipe2browser",
			googlesource: "https://vanadium.googlesource.com/release.projects.pipe2browser",
			github:       "git@github.com:vanadium/pipe2browser.git",
		},
		Mirror{
			name:         "playground",
			googlesource: "https://vanadium.googlesource.com/release.projects.playground",
			github:       "git@github.com:vanadium/playground.git",
		},
		Mirror{
			name:         "reader",
			googlesource: "https://vanadium.googlesource.com/release.projects.reader",
			github:       "git@github.com:vanadium/reader.git",
		},
		Mirror{
			name:         "third_party",
			googlesource: "https://vanadium.googlesource.com/third_party",
			github:       "git@github.com:vanadium/third_party.git",
		},
		Mirror{
			name:         "todos",
			googlesource: "https://vanadium.googlesource.com/release.projects.todos",
			github:       "git@github.com:vanadium/todos.git",
		},
		Mirror{
			name:         "www",
			googlesource: "https://vanadium.googlesource.com/www",
			github:       "git@github.com:vanadium/www.git",
		},
	}
)

type Mirror struct {
	name, googlesource, github string
}

// vanadiumGitHubMirror mirrors googlesource.com vanadium projects to
// github.com.
func vanadiumGitHubMirror(ctx *tool.Context, testName string, _ ...Opt) (_ *test.Result, e error) {
	// Initialize the test/task.
	cleanup, err := initTest(ctx, testName, nil)
	if err != nil {
		return nil, internalTestError{err, "Init"}
	}
	defer collect.Error(func() error { return cleanup() }, &e)

	root, err := project.JiriRoot()
	if err != nil {
		return nil, internalTestError{err, "JiriRoot"}
	}

	projects := filepath.Join(root, "projects")
	mode := os.FileMode(0755)
	if err := ctx.Run().MkdirAll(projects, mode); err != nil {
		return nil, internalTestError{err, "MkdirAll"}
	}

	allPassed := true
	suites := []xunit.TestSuite{}
	for _, mirror := range mirrors {
		suite, err := sync(ctx, mirror, projects)
		if err != nil {
			return nil, internalTestError{err, "sync"}
		}

		allPassed = allPassed && (suite.Failures == 0)
		suites = append(suites, *suite)
	}

	if err := xunit.CreateReport(ctx, testName, suites); err != nil {
		return nil, err
	}

	if !allPassed {
		return &test.Result{Status: test.Failed}, nil
	}

	return &test.Result{Status: test.Passed}, nil
}

func sync(ctx *tool.Context, mirror Mirror, projects string) (*xunit.TestSuite, error) {
	suite := xunit.TestSuite{Name: mirror.name}
	dirname := filepath.Join(projects, mirror.name)

	// If dirname does not exist `git clone` otherwise `git pull`.
	if _, err := ctx.Run().Stat(dirname); err != nil {
		if !os.IsNotExist(err) {
			return nil, internalTestError{err, "stat"}
		}

		err := clone(ctx, mirror, projects)
		testCase := makeTestCase("clone", err)
		if err != nil {
			suite.Failures++
		}
		suite.Cases = append(suite.Cases, *testCase)
	} else {
		err := pull(ctx, mirror, projects)
		testCase := makeTestCase("pull", err)
		if err != nil {
			suite.Failures++
		}
		suite.Cases = append(suite.Cases, *testCase)
	}

	err := push(ctx, mirror, projects)
	testCase := makeTestCase("push", err)
	if err != nil {
		suite.Failures++
	}
	suite.Cases = append(suite.Cases, *testCase)

	return &suite, nil
}

func makeTestCase(action string, err error) *xunit.TestCase {
	c := xunit.TestCase{
		Classname: "git",
		Name:      action,
	}

	if err != nil {
		f := xunit.Failure{
			Message: "git error",
			Data:    fmt.Sprintf("%v", err),
		}
		c.Failures = append(c.Failures, f)
	}

	return &c
}

func clone(ctx *tool.Context, mirror Mirror, projects string) error {
	dirname := filepath.Join(projects, mirror.name)
	return ctx.Git().Clone(mirror.googlesource, dirname)
}

func pull(ctx *tool.Context, mirror Mirror, projects string) error {
	dirname := filepath.Join(projects, mirror.name)
	opts := tool.RootDirOpt(dirname)
	return ctx.Git(opts).Pull("origin", "master")
}

func push(ctx *tool.Context, mirror Mirror, projects string) error {
	dirname := filepath.Join(projects, mirror.name)
	opts := tool.RootDirOpt(dirname)
	return ctx.Git(opts).Push(mirror.github, "master")
}
<|endoftext|>


<commit_before>/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

// Package restart enables containers to have labels added and monitored to
// keep the container's task running if it is killed.
//
// Setting the StatusLabel on a container instructs the restart monitor to keep
// that container's task in a specific status.
// Setting the LogPathLabel on a container will setup the task's IO to be redirected
// to a log file when running a task within the restart manager.
//
// The restart labels can be cleared off of a container using the WithNoRestarts Opt.
//
// The restart monitor has one option in the containerd config under the [plugins.restart]
// section.  `interval = "10s" sets the reconcile interval that the restart monitor checks
// for task state and reconciles the desired status for that task.
package restart

import (
	"context"

	"github.com/containerd/containerd"
	"github.com/containerd/containerd/cio"
	"github.com/containerd/containerd/containers"
)

const (
	// StatusLabel sets the restart status label for a container
	StatusLabel = "containerd.io/restart.status"
	// LogURILabel sets the restart log uri label for a container
	LogURILabel = "containerd.io/restart.loguri"

	// LogPathLabel sets the restart log path label for a container
	//
	// Deprecated(in release 1.5): use LogURILabel
	LogPathLabel = "containerd.io/restart.logpath"
)

// WithBinaryLogURI sets the binary-type log uri for a container.
func WithBinaryLogURI(binary string, args map[string]string) func(context.Context, *containerd.Client, *containers.Container) error {
	return func(_ context.Context, _ *containerd.Client, c *containers.Container) error {
		uri, err := cio.LogURIGenerator("binary", binary, args)
		if err != nil {
			return err
		}

		ensureLabels(c)
		c.Labels[LogURILabel] = uri.String()
		return nil
	}
}

// WithFileLogURI sets the file-type log uri for a container.
func WithFileLogURI(path string) func(context.Context, *containerd.Client, *containers.Container) error {
	return func(_ context.Context, _ *containerd.Client, c *containers.Container) error {
		uri, err := cio.LogURIGenerator("file", path, nil)
		if err != nil {
			return err
		}

		ensureLabels(c)
		c.Labels[LogURILabel] = uri.String()
		return nil
	}
}

// WithLogPath sets the log path for a container
//
// Deprecated(in release 1.5): use WithFileLogURI.
func WithLogPath(path string) func(context.Context, *containerd.Client, *containers.Container) error {
	return func(_ context.Context, _ *containerd.Client, c *containers.Container) error {
		ensureLabels(c)
		c.Labels[LogPathLabel] = path
		return nil
	}
}

// WithStatus sets the status for a container
func WithStatus(status containerd.ProcessStatus) func(context.Context, *containerd.Client, *containers.Container) error {
	return func(_ context.Context, _ *containerd.Client, c *containers.Container) error {
		ensureLabels(c)
		c.Labels[StatusLabel] = string(status)
		return nil
	}
}

// WithNoRestarts clears any restart information from the container
func WithNoRestarts(_ context.Context, _ *containerd.Client, c *containers.Container) error {
	if c.Labels == nil {
		return nil
	}
	delete(c.Labels, StatusLabel)
	delete(c.Labels, LogPathLabel)
	delete(c.Labels, LogURILabel)
	return nil
}

func ensureLabels(c *containers.Container) {
	if c.Labels == nil {
		c.Labels = make(map[string]string)
	}
}
<commit_msg>restart: allow passing existing log URI object<commit_after>/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

// Package restart enables containers to have labels added and monitored to
// keep the container's task running if it is killed.
//
// Setting the StatusLabel on a container instructs the restart monitor to keep
// that container's task in a specific status.
// Setting the LogPathLabel on a container will setup the task's IO to be redirected
// to a log file when running a task within the restart manager.
//
// The restart labels can be cleared off of a container using the WithNoRestarts Opt.
//
// The restart monitor has one option in the containerd config under the [plugins.restart]
// section.  `interval = "10s" sets the reconcile interval that the restart monitor checks
// for task state and reconciles the desired status for that task.
package restart

import (
	"context"
	"net/url"

	"github.com/containerd/containerd"
	"github.com/containerd/containerd/cio"
	"github.com/containerd/containerd/containers"
)

const (
	// StatusLabel sets the restart status label for a container
	StatusLabel = "containerd.io/restart.status"
	// LogURILabel sets the restart log uri label for a container
	LogURILabel = "containerd.io/restart.loguri"

	// LogPathLabel sets the restart log path label for a container
	//
	// Deprecated(in release 1.5): use LogURILabel
	LogPathLabel = "containerd.io/restart.logpath"
)

// WithLogURI sets the specified log uri for a container.
func WithLogURI(uri *url.URL) func(context.Context, *containerd.Client, *containers.Container) error {
	return WithLogURIString(uri.String())
}

// WithLogURIString sets the specified log uri string for a container.
func WithLogURIString(uriString string) func(context.Context, *containerd.Client, *containers.Container) error {
	return func(_ context.Context, _ *containerd.Client, c *containers.Container) error {
		ensureLabels(c)
		c.Labels[LogURILabel] = uriString
		return nil
	}
}

// WithBinaryLogURI sets the binary-type log uri for a container.
//
// Deprecated(in release 1.5): use WithLogURI
func WithBinaryLogURI(binary string, args map[string]string) func(context.Context, *containerd.Client, *containers.Container) error {
	uri, err := cio.LogURIGenerator("binary", binary, args)
	if err != nil {
		return func(context.Context, *containerd.Client, *containers.Container) error {
			return err
		}
	}
	return WithLogURI(uri)
}

// WithFileLogURI sets the file-type log uri for a container.
//
// Deprecated(in release 1.5): use WithLogURI
func WithFileLogURI(path string) func(context.Context, *containerd.Client, *containers.Container) error {
	uri, err := cio.LogURIGenerator("file", path, nil)
	if err != nil {
		return func(context.Context, *containerd.Client, *containers.Container) error {
			return err
		}
	}
	return WithLogURI(uri)
}

// WithLogPath sets the log path for a container
//
// Deprecated(in release 1.5): use WithLogURI with "file://<path>" URI.
func WithLogPath(path string) func(context.Context, *containerd.Client, *containers.Container) error {
	return func(_ context.Context, _ *containerd.Client, c *containers.Container) error {
		ensureLabels(c)
		c.Labels[LogPathLabel] = path
		return nil
	}
}

// WithStatus sets the status for a container
func WithStatus(status containerd.ProcessStatus) func(context.Context, *containerd.Client, *containers.Container) error {
	return func(_ context.Context, _ *containerd.Client, c *containers.Container) error {
		ensureLabels(c)
		c.Labels[StatusLabel] = string(status)
		return nil
	}
}

// WithNoRestarts clears any restart information from the container
func WithNoRestarts(_ context.Context, _ *containerd.Client, c *containers.Container) error {
	if c.Labels == nil {
		return nil
	}
	delete(c.Labels, StatusLabel)
	delete(c.Labels, LogPathLabel)
	delete(c.Labels, LogURILabel)
	return nil
}

func ensureLabels(c *containers.Container) {
	if c.Labels == nil {
		c.Labels = make(map[string]string)
	}
}
<|endoftext|>


<commit_before>package asock

import (
	"net"
	"testing"
//	"time"
)

// implement an echo server
func TestEOMServer(t *testing.T) {
	d := make(Dispatch) // create Dispatch
	d["echo"] = &DispatchFunc{echo, "split"} // and put a function in it
	// instantiate an asocket
	c := Config{Sockname: "/tmp/test11.sock", Msglvl: Conn}
	as, err := NewUnix(c, d)
	if err != nil {
		t.Errorf("Couldn't create socket: %v", err)
	}
	// launch echoclient. we should get a message about the
	// connection.
	go eomclient(as.s, t, "\n\n")
	msg := <-as.Msgr
	if msg.Err != nil {
		t.Errorf("connection creation returned error: %v", msg.Err)
	}
	// wait for disconnect Msg
	msg = <-as.Msgr
	// shut down asocket
	as.Quit()

	// instantiate an asocket, this time with custom EOM
	c = Config{Sockname: "/tmp/test11.sock", Msglvl: Conn, EOM: "p!p"}
	as, err = NewUnix(c, d)
	if err != nil {
		t.Errorf("Couldn't create socket: %v", err)
	}
	// launch echoclient. we should get a message about the
	// connection.
	go eomclient(as.s, t, "p!p")
	msg = <-as.Msgr
	if msg.Err != nil {
		t.Errorf("connection creation returned error: %v", msg.Err)
	}
	// wait for disconnect Msg
	msg = <-as.Msgr
	// shut down asocket
	as.Quit()
}

// this time our (less) fake client will send a string over the
// connection and (hopefully) get it echoed back.
func eomclient(sn string, t *testing.T, eom string) {
	conn, err := net.Dial("unix", sn)
	defer conn.Close()
	if err != nil {
		t.Errorf("Couldn't connect to %v: %v", sn, err)
	}
	conn.Write([]byte("echo it works!" + eom + "foo"))
	//time.Sleep(50 * time.Millisecond)
	res, err := readConn(conn)
	if err != nil {
		t.Errorf("Error on read: %v", err)
	}
	if string(res) != "it works!" + eom {
		t.Errorf("Expected 'it works!EOM' but got '%v'", string(res))
	}
	// finish the partial request sent last time
	conn.Write([]byte("foo bar" + eom))
	res, err = readConn(conn)
	if err != nil {
		t.Errorf("Error on read: %v", err)
	}
	if string(res) != "Unknown command 'foofoo'. Available commands: echo " + eom {
		t.Errorf("Expected unknown command help, but got '%v'", string(res))
	}
	// now send two requests at once
	conn.Write([]byte("echo thing one" + eom + "echo thing two" + eom))
	res, err = readConn(conn)
	if err != nil {
		t.Errorf("Error on read: %v", err)
	}
	if string(res) != "thing one" + eom + "thing two" + eom {
		t.Errorf("Expected 'thing oneEOMthing twoEOM' but got '%v'", string(res))
	}
}
<commit_msg>putting wait back in<commit_after>package asock

import (
	"net"
	"testing"
	"time"
)

// implement an echo server
func TestEOMServer(t *testing.T) {
	d := make(Dispatch) // create Dispatch
	d["echo"] = &DispatchFunc{echo, "split"} // and put a function in it
	// instantiate an asocket
	c := Config{Sockname: "/tmp/test11.sock", Msglvl: Conn}
	as, err := NewUnix(c, d)
	if err != nil {
		t.Errorf("Couldn't create socket: %v", err)
	}
	// launch echoclient. we should get a message about the
	// connection.
	go eomclient(as.s, t, "\n\n")
	msg := <-as.Msgr
	if msg.Err != nil {
		t.Errorf("connection creation returned error: %v", msg.Err)
	}
	// wait for disconnect Msg
	msg = <-as.Msgr
	// shut down asocket
	as.Quit()

	// instantiate an asocket, this time with custom EOM
	c = Config{Sockname: "/tmp/test11.sock", Msglvl: Conn, EOM: "p!p"}
	as, err = NewUnix(c, d)
	if err != nil {
		t.Errorf("Couldn't create socket: %v", err)
	}
	// launch echoclient. we should get a message about the
	// connection.
	go eomclient(as.s, t, "p!p")
	msg = <-as.Msgr
	if msg.Err != nil {
		t.Errorf("connection creation returned error: %v", msg.Err)
	}
	// wait for disconnect Msg
	msg = <-as.Msgr
	// shut down asocket
	as.Quit()
}

// test EOM conditions
func eomclient(sn string, t *testing.T, eom string) {
	conn, err := net.Dial("unix", sn)
	defer conn.Close()
	if err != nil {
		t.Errorf("Couldn't connect to %v: %v", sn, err)
	}
	// send with EOM in teh middle, but not at the end; wait to make sure it gets there
	conn.Write([]byte("echo it works!" + eom + "foo"))
	time.Sleep(25 * time.Millisecond)
	res, err := readConn(conn)
	if err != nil {
		t.Errorf("Error on read: %v", err)
	}
	if string(res) != "it works!" + eom {
		t.Errorf("Expected 'it works!EOM' but got '%v'", string(res))
	}
	// finish the partial request sent last time
	conn.Write([]byte("foo bar" + eom))
	res, err = readConn(conn)
	if err != nil {
		t.Errorf("Error on read: %v", err)
	}
	if string(res) != "Unknown command 'foofoo'. Available commands: echo " + eom {
		t.Errorf("Expected unknown command help, but got '%v'", string(res))
	}
	// now send two requests at once
	conn.Write([]byte("echo thing one" + eom + "echo thing two" + eom))
	res, err = readConn(conn)
	if err != nil {
		t.Errorf("Error on read: %v", err)
	}
	if string(res) != "thing one" + eom + "thing two" + eom {
		t.Errorf("Expected 'thing oneEOMthing twoEOM' but got '%v'", string(res))
	}
}
<|endoftext|>


<commit_before>package multiverse

import (
	"runtime"
	"strings"
	"sync"
	"unicode"

	"github.com/CasualSuperman/phonetics/metaphone"
	"github.com/CasualSuperman/phonetics/ngram"
	"github.com/CasualSuperman/phonetics/sift3"
)

func preventUnicode(name string) string {
	name = strings.ToLower(name)

	clean := ""
	for _, r := range name {
		if r > 128 {
			switch r {
			case 'á', 'à', 'â':
				clean += "a"
			case 'é':
				clean += "e"
			case 'í':
				clean += "i"
			case 'ö':
				clean += "o"
			case 'û', 'ú':
				clean += "u"

			case 'Æ', 'æ':
				clean += "ae"

			case '®':
				// We know this is an option but we're explicitly ignoring it.

			default:
			}
		} else {
			if r == ' ' || unicode.IsLetter(r) || r == '_' || r == '-' {
				clean += string(r)
			}
		}
	}

	return clean
}

type fuzzySearchList struct {
	sync.Mutex
	data []similarityItem
}

type similarityItem struct {
	index      int
	similarity int
}

func newFuzzySearchList(count int) fuzzySearchList {
	t := fuzzySearchList{}
	t.data = make([]similarityItem, 0, count)
	return t
}

func (f *fuzzySearchList) Add(index int, similarity int) {
	f.Lock()
	defer f.Unlock()
	for i, item := range f.data {
		if item.index == index {
			if f.data[i].similarity < similarity {
				f.data[i].similarity = similarity
				for i >= 1 && f.data[i].similarity > f.data[i-1].similarity {
					f.data[i-1], f.data[i] = f.data[i], f.data[i-1]
					i--
				}
			}
			return
		}
	}

	myLen := len(f.data)

	if myLen < cap(f.data) {
		f.data = f.data[:myLen+1]
		f.data[myLen] = similarityItem{index, similarity}
		myLen++
	}

	for i := myLen - 2; i >= 0; i-- {
		if f.data[i].similarity < similarity || (f.data[i].similarity == similarity && f.data[i].index > index) {
			f.data[i+1] = f.data[i]
			f.data[i].index = index
			f.data[i].similarity = similarity
		} else {
			return
		}
	}
	if f.data[0].similarity < similarity {
		f.data[0].index = index
		f.data[0].similarity = similarity
	}
}

// FuzzyNameSearch searches for a card with a similar name to the searchPhrase, and returns count or less of the most likely results.
func (m Multiverse) FuzzyNameSearch(searchPhrase string, count int) CardList {
	var done sync.WaitGroup
	aggregator := newFuzzySearchList(count)

	groups := runtime.GOMAXPROCS(-1)

	totalCards := len(m.Cards)
	groupInterval := totalCards / groups

	searchPhrase = preventUnicode(searchPhrase)
	searchGrams2 := ngram.New(searchPhrase, 2)
	searchGrams3 := ngram.New(searchPhrase, 3)

	for _, searchTerm := range split(searchPhrase) {
		if len(searchTerm) == 0 {
			continue
		}
		searchMetaphone := metaphone.Encode(searchTerm)
		done.Add(groups)

		for i := 0; i < groups; i++ {
			start := i * groupInterval
			end := start + groupInterval
			if i == groups-1 {
				end = totalCards
			}

			go func(searchTerm, searchMetaphone string, start, end int) {
				defer done.Done()
				cards := m.Cards[start:end]
				for cardIndex := range cards {
					card := cards[cardIndex]
					name := card.ascii
					metaphones := card.metaphones
					matchMod := float32(1.0)

					if name == searchPhrase {
						aggregator.Add(cardIndex+start, int(^uint(0)>>1))
						continue
					} else if strings.HasPrefix(name, searchPhrase) {
						matchMod = 1000
					}

					bestMatch := int(^uint(0) >> 1)

					for _, metaphone := range metaphones {
						if len(metaphone) == 0 {
							continue
						}

						match := int(sift3.SiftASCII(metaphone, searchMetaphone) / matchMod)

						if match < bestMatch {
							bestMatch = match
						}
					}

					similarity := float32(searchGrams2.Similarity(name))
					similarity += float32(searchGrams3.Similarity(name))
					similarity -= float32(bestMatch * 2)
					dist := sift3.SiftASCII(searchPhrase, name)
					similarity -= float32(bestMatch) * dist * dist

					if similarity > 0 {
						aggregator.Add(cardIndex+start, int(similarity))
					}
				}
			}(searchTerm, searchMetaphone, start, end)
		}
	}

	done.Wait()

	if len(aggregator.data) < count {
		count = len(aggregator.data)
	}

	results := make(CardList, count)

	for i, card := range aggregator.data {
		results[i] = &m.Cards[card.index]
	}

	return results
}
<commit_msg>preventUnicode creates less garbage now.<commit_after>package multiverse

import (
	"bytes"
	"runtime"
	"strings"
	"sync"
	"unicode"

	"github.com/CasualSuperman/phonetics/metaphone"
	"github.com/CasualSuperman/phonetics/ngram"
	"github.com/CasualSuperman/phonetics/sift3"
)

func preventUnicode(name string) string {
	var clean bytes.Buffer

	name = strings.ToLower(name)

	for _, r := range name {
		if r > 128 {
			switch r {
			case 'á', 'à', 'â':
				clean.WriteByte('a')
			case 'é':
				clean.WriteByte('e')
			case 'í':
				clean.WriteByte('i')
			case 'ö':
				clean.WriteByte('o')
			case 'û', 'ú':
				clean.WriteByte('u')

			case 'Æ', 'æ':
				clean.WriteString("ae")

			case '®':
				// We know this is an option but we're explicitly ignoring it.

			default:
			}
		} else {
			if r == ' ' || unicode.IsLetter(r) || r == '_' || r == '-' {
				clean.WriteRune(r)
			}
		}
	}

	return clean.String()
}

type fuzzySearchList struct {
	sync.Mutex
	data []similarityItem
}

type similarityItem struct {
	index      int
	similarity int
}

func newFuzzySearchList(count int) fuzzySearchList {
	t := fuzzySearchList{}
	t.data = make([]similarityItem, 0, count)
	return t
}

func (f *fuzzySearchList) Add(index int, similarity int) {
	f.Lock()
	defer f.Unlock()
	for i, item := range f.data {
		if item.index == index {
			if f.data[i].similarity < similarity {
				f.data[i].similarity = similarity
				for i >= 1 && f.data[i].similarity > f.data[i-1].similarity {
					f.data[i-1], f.data[i] = f.data[i], f.data[i-1]
					i--
				}
			}
			return
		}
	}

	myLen := len(f.data)

	if myLen < cap(f.data) {
		f.data = f.data[:myLen+1]
		f.data[myLen] = similarityItem{index, similarity}
		myLen++
	}

	for i := myLen - 2; i >= 0; i-- {
		if f.data[i].similarity < similarity || (f.data[i].similarity == similarity && f.data[i].index > index) {
			f.data[i+1] = f.data[i]
			f.data[i].index = index
			f.data[i].similarity = similarity
		} else {
			return
		}
	}
	if f.data[0].similarity < similarity {
		f.data[0].index = index
		f.data[0].similarity = similarity
	}
}

// FuzzyNameSearch searches for a card with a similar name to the searchPhrase, and returns count or less of the most likely results.
func (m Multiverse) FuzzyNameSearch(searchPhrase string, count int) CardList {
	var done sync.WaitGroup
	aggregator := newFuzzySearchList(count)

	groups := runtime.GOMAXPROCS(-1)

	totalCards := len(m.Cards)
	groupInterval := totalCards / groups

	searchPhrase = preventUnicode(searchPhrase)
	searchGrams2 := ngram.New(searchPhrase, 2)
	searchGrams3 := ngram.New(searchPhrase, 3)

	for _, searchTerm := range split(searchPhrase) {
		if len(searchTerm) == 0 {
			continue
		}
		searchMetaphone := metaphone.Encode(searchTerm)
		done.Add(groups)

		for i := 0; i < groups; i++ {
			start := i * groupInterval
			end := start + groupInterval
			if i == groups-1 {
				end = totalCards
			}

			go func(searchTerm, searchMetaphone string, start, end int) {
				defer done.Done()
				cards := m.Cards[start:end]
				for cardIndex := range cards {
					card := cards[cardIndex]
					name := card.ascii
					metaphones := card.metaphones
					matchMod := float32(1.0)

					if name == searchPhrase {
						aggregator.Add(cardIndex+start, int(^uint(0)>>1))
						continue
					} else if strings.HasPrefix(name, searchPhrase) {
						matchMod = 1000
					}

					bestMatch := int(^uint(0) >> 1)

					for _, metaphone := range metaphones {
						if len(metaphone) == 0 {
							continue
						}

						match := int(sift3.SiftASCII(metaphone, searchMetaphone) / matchMod)

						if match < bestMatch {
							bestMatch = match
						}
					}

					similarity := float32(searchGrams2.Similarity(name))
					similarity += float32(searchGrams3.Similarity(name))
					similarity -= float32(bestMatch * 2)
					dist := sift3.SiftASCII(searchPhrase, name)
					similarity -= float32(bestMatch) * dist * dist

					if similarity > 0 {
						aggregator.Add(cardIndex+start, int(similarity))
					}
				}
			}(searchTerm, searchMetaphone, start, end)
		}
	}

	done.Wait()

	if len(aggregator.data) < count {
		count = len(aggregator.data)
	}

	results := make(CardList, count)

	for i, card := range aggregator.data {
		results[i] = &m.Cards[card.index]
	}

	return results
}
<|endoftext|>


<commit_before>// Clock counts down to or up from a target time.
package main

import (
	"fmt"
	"os"
	"time"
)

// Update target and motto as desired.
var (
	target = time.Date(2019, 4, 22, 0, 0, 0, 0, time.UTC)
	motto  = "Simply Go"
)

func main() {
	printTargetTime(target, motto)
	exitOnEnterKey()

	var previous time.Time
	for {
		now := time.Now().Truncate(time.Second)
		if now != previous {
			previous = now
			countdown := now.Sub(target) // Negative times are before the target
			printCountdown(now.In(target.Location()), countdown)
		}
		time.Sleep(100 * time.Millisecond)
	}
}

func exitOnEnterKey() {
	go func() {
		buf := make([]byte, 1)
		_, _ = os.Stdin.Read(buf)
		os.Exit(0)
	}()
}

const (
	highlightStart = "\x1b[1;35m"
	highlightEnd   = "\x1b[0m"
	indent         = "\t"
)

func printTargetTime(target time.Time, motto string) {
	fmt.Print(indent, highlightStart, motto, highlightEnd, "\n")
	fmt.Print(indent, target.Format(time.UnixDate), "\n")
}

func printCountdown(now time.Time, countdown time.Duration) {
	var sign string
	if countdown >= 0 {
		sign = "+"
	} else {
		sign = "-"
		countdown = -countdown
	}

	days := int(countdown / (24 * time.Hour))
	countdown = countdown % (24 * time.Hour)

	fmt.Print(indent, now.Format(time.UnixDate), "  ", sign)
	if days > 0 {
		fmt.Print(days, "d")
	}
	fmt.Print(countdown, "          \r")
	os.Stdout.Sync()
}
<commit_msg>May Day<commit_after>// Clock counts down to or up from a target time.
package main

import (
	"fmt"
	"os"
	"time"
)

// Update target and motto as desired.
var (
	target = time.Date(2019, 5, 1, 0, 0, 0, 0, time.UTC)
	motto  = "Just Go"
)

func main() {
	printTargetTime(target, motto)
	exitOnEnterKey()

	var previous time.Time
	for {
		now := time.Now().Truncate(time.Second)
		if now != previous {
			previous = now
			countdown := now.Sub(target) // Negative times are before the target
			printCountdown(now.In(target.Location()), countdown)
		}
		time.Sleep(100 * time.Millisecond)
	}
}

func exitOnEnterKey() {
	go func() {
		buf := make([]byte, 1)
		_, _ = os.Stdin.Read(buf)
		os.Exit(0)
	}()
}

const (
	highlightStart = "\x1b[1;35m"
	highlightEnd   = "\x1b[0m"
	indent         = "\t"
)

func printTargetTime(target time.Time, motto string) {
	fmt.Print(indent, highlightStart, motto, highlightEnd, "\n")
	fmt.Print(indent, target.Format(time.UnixDate), "\n")
}

func printCountdown(now time.Time, countdown time.Duration) {
	var sign string
	if countdown >= 0 {
		sign = "+"
	} else {
		sign = "-"
		countdown = -countdown
	}

	days := int(countdown / (24 * time.Hour))
	countdown = countdown % (24 * time.Hour)

	fmt.Print(indent, now.Format(time.UnixDate), "  ", sign)
	if days > 0 {
		fmt.Print(days, "d")
	}
	fmt.Print(countdown, "          \r")
	os.Stdout.Sync()
}
<|endoftext|>


<commit_before>// Package text implements a development-friendly textual handler.
package text

import (
	"fmt"
	"io"
	"sort"
	"sync"
	"time"

	"github.com/apex/log"
)

// start time.
var start = time.Now()

// colors.
const (
	none   = 0
	red    = 31
	green  = 32
	yellow = 33
	blue   = 34
	gray   = 37
)

// colors mapping.
var colors = [...]int{
	log.DebugLevel: gray,
	log.InfoLevel:  blue,
	log.WarnLevel:  yellow,
	log.ErrorLevel: red,
	log.FatalLevel: red,
}

// strings mapping.
var strings = [...]string{
	log.DebugLevel: "DEBUG",
	log.InfoLevel:  "INFO",
	log.WarnLevel:  "WARN",
	log.ErrorLevel: "ERROR",
	log.FatalLevel: "FATAL",
}

// field used for sorting.
type field struct {
	Name  string
	Value interface{}
}

// by sorts projects by call count.
type byName []field

func (a byName) Len() int           { return len(a) }
func (a byName) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a byName) Less(i, j int) bool { return a[i].Name < a[j].Name }

// Handler implementation.
type Handler struct {
	mu     sync.Mutex
	Writer io.Writer
}

// New handler.
func New(w io.Writer) *Handler {
	return &Handler{
		Writer: w,
	}
}

// HandleLog implements log.Handler.
func (h *Handler) HandleLog(e *log.Entry) error {
	color := colors[e.Level]
	level := strings[e.Level]

	var fields []field

	for k, v := range e.Fields {
		fields = append(fields, field{k, v})
	}

	sort.Sort(byName(fields))

	h.mu.Lock()
	defer h.mu.Unlock()

	ts := time.Since(start) / time.Second
	fmt.Fprintf(h.Writer, "\033[%dm%6s\033[0m[%04d] %-25s", color, level, ts, e.Message)

	for _, f := range fields {
		fmt.Fprintf(h.Writer, " \033[%dm%s\033[0m=%v", color, f.Name, f.Value)
	}

	fmt.Fprintln(h.Writer)

	return nil
}
<commit_msg>move delta to after the fields<commit_after>// Package text implements a development-friendly textual handler.
package text

import (
	"fmt"
	"io"
	"sort"
	"sync"
	"time"

	"github.com/apex/log"
)

// start time.
var start = time.Now()

// colors.
const (
	none   = 0
	red    = 31
	green  = 32
	yellow = 33
	blue   = 34
	gray   = 37
)

// colors mapping.
var colors = [...]int{
	log.DebugLevel: gray,
	log.InfoLevel:  blue,
	log.WarnLevel:  yellow,
	log.ErrorLevel: red,
	log.FatalLevel: red,
}

// strings mapping.
var strings = [...]string{
	log.DebugLevel: "DEBUG",
	log.InfoLevel:  "INFO",
	log.WarnLevel:  "WARN",
	log.ErrorLevel: "ERROR",
	log.FatalLevel: "FATAL",
}

// field used for sorting.
type field struct {
	Name  string
	Value interface{}
}

// by sorts projects by call count.
type byName []field

func (a byName) Len() int           { return len(a) }
func (a byName) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a byName) Less(i, j int) bool { return a[i].Name < a[j].Name }

// Handler implementation.
type Handler struct {
	mu     sync.Mutex
	Writer io.Writer
}

// New handler.
func New(w io.Writer) *Handler {
	return &Handler{
		Writer: w,
	}
}

// HandleLog implements log.Handler.
func (h *Handler) HandleLog(e *log.Entry) error {
	color := colors[e.Level]
	level := strings[e.Level]

	var fields []field

	for k, v := range e.Fields {
		fields = append(fields, field{k, v})
	}

	sort.Sort(byName(fields))

	h.mu.Lock()
	defer h.mu.Unlock()

	fmt.Fprintf(h.Writer, "\033[%dm%6s\033[0m %-25s", color, level, e.Message)

	for _, f := range fields {
		fmt.Fprintf(h.Writer, " \033[%dm%s\033[0m=%v", color, f.Name, f.Value)
	}

	fmt.Fprintf(h.Writer, " [%d]\n", int(time.Since(start)/time.Millisecond))

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"log"

	"github.com/flynn/rpcplus"
	"github.com/flynn/sampi/types"
	"github.com/titanous/go-dockerclient"
)

func main() {
	scheduler, err := rpcplus.DialHTTP("tcp", "localhost:1112")
	if err != nil {
		log.Fatal(err)
	}

	var state map[string]sampi.Host
	if err := scheduler.Call("Scheduler.State", struct{}{}, &state); err != nil {
		log.Fatal(err)
	}
	log.Printf("%#v", state)

	var firstHost string
	for k := range state {
		firstHost = k
		break
	}
	if firstHost == "" {
		log.Fatal("no hosts")
	}

	var schedRes sampi.ScheduleRes
	schedReq := &sampi.ScheduleReq{
		Incremental: true,
		HostJobs:    map[string][]*sampi.Job{firstHost: {{ID: "test", Config: &docker.Config{Image: "crosbymichael/redis"}}}},
	}
	if err := scheduler.Call("Scheduler.Schedule", schedReq, &schedRes); err != nil {
		log.Fatal(err)
	}
	log.Print("scheduled container")
}
<commit_msg>host/sampi: Add lorne attach to example<commit_after>package main

import (
	"encoding/gob"
	"io"
	"log"
	"net"
	"os"
	"time"

	"github.com/flynn/lorne/types"
	"github.com/flynn/rpcplus"
	"github.com/flynn/sampi/types"
	"github.com/titanous/go-dockerclient"
)

func main() {
	scheduler, err := rpcplus.DialHTTP("tcp", "localhost:1112")
	if err != nil {
		log.Fatal(err)
	}

	var state map[string]sampi.Host
	if err := scheduler.Call("Scheduler.State", struct{}{}, &state); err != nil {
		log.Fatal(err)
	}
	log.Printf("%#v", state)

	var firstHost string
	for k := range state {
		firstHost = k
		break
	}
	if firstHost == "" {
		log.Fatal("no hosts")
	}

	var schedRes sampi.ScheduleRes
	schedReq := &sampi.ScheduleReq{
		Incremental: true,
		HostJobs:    map[string][]*sampi.Job{firstHost: {{ID: "test", Config: &docker.Config{Image: "crosbymichael/redis"}}}},
	}
	if err := scheduler.Call("Scheduler.Schedule", schedReq, &schedRes); err != nil {
		log.Fatal(err)
	}
	log.Print("scheduled container")

	// tail logs
	time.Sleep(1 * time.Second)
	conn, err := net.Dial("tcp", "localhost:1120")
	if err != nil {
		log.Fatal(err)
	}
	err = gob.NewEncoder(conn).Encode(&lorne.AttachReq{
		JobID: "test",
		Flags: lorne.AttachFlagStdout | lorne.AttachFlagStderr | lorne.AttachFlagLogs | lorne.AttachFlagStream,
	})
	if err != nil {
		log.Fatal(err)
	}
	if _, err := io.Copy(os.Stdout, conn); err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/base64"
	"encoding/json"
	"github.com/nicksnyder/go-i18n/i18n"
	"html/template"
	"net/http"
	"os"
	"path"
	"strings"
)

/**
 * JSON files containing information about articles stored in the distributed cache (Freenet)
 * are named like `json-files/<base64(feed's url)>.json`
 * @param feedUrl - The URL of the RSS/Atom feed to retrieve information about articles from
 */
func articlesFilename(feedUrl string) string {
	b64FeedUrl := base64.StdEncoding.EncodeToString([]byte(feedUrl))
	return path.Join(".", "json-files", b64FeedUrl+".json")
}

/**
 * Get information about articles from a given feed to be injected into the portal page.
 * @param {string} feedUrl - The URL of the feed to fetch articles from
 * @return a map with a "feeds" key and corresponding array of Feed structs and an optional error
 */
func initModuleWithArticles(feedUrl string) (map[string]interface{}, error) {
	articleInfoFile, openErr := os.Open(articlesFilename(feedUrl))
	if openErr != nil {
		return nil, openErr
	}
	defer articleInfoFile.Close()
	articleInfo := ArticleInfo{}
	decoder := json.NewDecoder(articleInfoFile)
	decodeErr := decoder.Decode(&articleInfo)
	if decodeErr != nil {
		return nil, decodeErr
	}
	mapping := make(map[string]interface{})
	mapping["articles"] = articleInfo.Items
	return mapping, nil
}

/**
 * Build the articles template with links to articles in a particular feed.
 */
func CreateArticlePage(w http.ResponseWriter, r *http.Request) {
	T, _ := i18n.Tfunc(os.Getenv(LANG_ENVVAR), DEFAULT_LANG)
	t, _ := template.ParseFiles(path.Join(".", "views", "articles.html"))
	// TODO - Extract feed url from URL query string
	pathComponents := strings.Split(r.URL.Path, "/")
	b64FeedUrl := pathComponents[len(pathComponents)-1]
	feedUrlBytes, _ := base64.StdEncoding.DecodeString(b64FeedUrl)
	feedUrl := string(feedUrlBytes)
	moduleData, articlesErr := initModuleWithArticles(feedUrl)
	if articlesErr != nil {
		HandleCCError(ERR_NO_ARTICLES_FILE, articlesErr.Error(), ErrorState{
			"responseWriter": w,
			"request":        r,
		})
		return
	}
	moduleData["authorWord"] = T("authors_word")
	moduleData["publishedWord"] = T("published_word")
	marshalled, err := json.Marshal(moduleData)
	var module string
	if err != nil {
		HandleCCError(ERR_CORRUPT_JSON, err.Error(), ErrorState{
			"responseWriter": w,
			"request":        r,
		})
		return
	}
	module = string(marshalled[:])
	t.Execute(w, map[string]interface{}{
		"Previous":         T("previous_word"),
		"More":             T("more_word"),
		"PortalBlurb":      T("portal_blurb"),
		"CenoPortalModule": module,
	})
}
<commit_msg>Removed unneeded TODO<commit_after>package main

import (
	"encoding/base64"
	"encoding/json"
	"github.com/nicksnyder/go-i18n/i18n"
	"html/template"
	"net/http"
	"os"
	"path"
	"strings"
)

/**
 * JSON files containing information about articles stored in the distributed cache (Freenet)
 * are named like `json-files/<base64(feed's url)>.json`
 * @param feedUrl - The URL of the RSS/Atom feed to retrieve information about articles from
 */
func articlesFilename(feedUrl string) string {
	b64FeedUrl := base64.StdEncoding.EncodeToString([]byte(feedUrl))
	return path.Join(".", "json-files", b64FeedUrl+".json")
}

/**
 * Get information about articles from a given feed to be injected into the portal page.
 * @param {string} feedUrl - The URL of the feed to fetch articles from
 * @return a map with a "feeds" key and corresponding array of Feed structs and an optional error
 */
func initModuleWithArticles(feedUrl string) (map[string]interface{}, error) {
	articleInfoFile, openErr := os.Open(articlesFilename(feedUrl))
	if openErr != nil {
		return nil, openErr
	}
	defer articleInfoFile.Close()
	articleInfo := ArticleInfo{}
	decoder := json.NewDecoder(articleInfoFile)
	decodeErr := decoder.Decode(&articleInfo)
	if decodeErr != nil {
		return nil, decodeErr
	}
	mapping := make(map[string]interface{})
	mapping["articles"] = articleInfo.Items
	return mapping, nil
}

/**
 * Build the articles template with links to articles in a particular feed.
 */
func CreateArticlePage(w http.ResponseWriter, r *http.Request) {
	T, _ := i18n.Tfunc(os.Getenv(LANG_ENVVAR), DEFAULT_LANG)
	t, _ := template.ParseFiles(path.Join(".", "views", "articles.html"))
	pathComponents := strings.Split(r.URL.Path, "/")
	b64FeedUrl := pathComponents[len(pathComponents)-1]
	feedUrlBytes, _ := base64.StdEncoding.DecodeString(b64FeedUrl)
	feedUrl := string(feedUrlBytes)
	moduleData, articlesErr := initModuleWithArticles(feedUrl)
	if articlesErr != nil {
		HandleCCError(ERR_NO_ARTICLES_FILE, articlesErr.Error(), ErrorState{
			"responseWriter": w,
			"request":        r,
		})
		return
	}
	moduleData["authorWord"] = T("authors_word")
	moduleData["publishedWord"] = T("published_word")
	marshalled, err := json.Marshal(moduleData)
	var module string
	if err != nil {
		HandleCCError(ERR_CORRUPT_JSON, err.Error(), ErrorState{
			"responseWriter": w,
			"request":        r,
		})
		return
	}
	module = string(marshalled[:])
	t.Execute(w, map[string]interface{}{
		"Previous":         T("previous_word"),
		"More":             T("more_word"),
		"PortalBlurb":      T("portal_blurb"),
		"CenoPortalModule": module,
	})
}
<|endoftext|>


<commit_before>package network

import (
	"github.com/yaricom/goNEAT/neat/genetics"
	"fmt"
	"errors"
)

// A NETWORK is a LIST of input NODEs and a LIST of output NODEs.
// The point of the network is to define a single entity which can evolve
// or learn on its own, even though it may be part of a larger framework.
type Network interface {
	// Puts the network back into an initial state
	Flush()
	// Activates the net such that all outputs are active
	Activate() (bool, error)
	// If at least one output is not active then return true
	OutputIsOff() bool

	// Prints the values of network outputs to the console
	PrintActivation()
	// Print the values of network inputs to the console
	PrintInput()
	// Verify that network was successfully flushed for debugging
	FlushCheck() error

	// Adds a new input node
	AddInputNode(node *NNode)
	// Adds a new output node
	AddOutputNode(node *NNode)

	// Takes an array of sensor values and loads it into SENSOR inputs ONLY
	LoadSensors(sensors []float64)
	// Set network name
	SetName(name string)

	// This checks a POTENTIAL link between a potential in_node
     	// and potential out_node to see if it must be recurrent.
	// Use count and thresh to jump out in the case of an infinite loop.
	IsRecurrent(potin_node, potout_node *NNode, count *int, thresh int) bool
	// Find the maximum number of neurons between an output and an input
	MaxDepth() (int32, error)

	// Counts the number of nodes in the net
	NodeCount() int
	// Counts the number of links in the net
	LinkCount() int

	// Returns all nodes in the network
	AllNodes() []*NNode
}

// Creates new network
func NewNetwork(in, out, all []*NNode, netid int32) Network {
	n := newNetwork(netid)
	n.inputs = in
	n.outputs = out
	n.all_nodes = all
	return &n
}

// The default private constructor
func newNetwork(netId int32) network {
	return network {
		numlinks:-1,
		net_id:netId,
	}
}

// The private network data holder
type network struct {
	//The number of links in the net (-1 means not yet counted)
	numlinks int

	// A list of all the nodes in the network
	all_nodes []*NNode
	// NNodes that input into the network
	inputs []*NNode
	// NNodes that output from the network
	outputs []*NNode

	// A network id
	net_id int32

	// Allows Network to be matched with its Genome
	genotype *genetics.Genome

	// Is a name of this network */
	name string
}

// The Network interface implementation
func (n *network) Flush() {
	// Flush back recursively
	for _, node := range n.all_nodes {
		node.Flushback()
	}
}
func (n *network) FlushCheck() error {
	for _, node := range n.all_nodes {
		err := node.FlushbackCheck()
		if err != nil {
			return err
		}
	}
	return nil
}
func (n *network) PrintActivation() {
	fmt.Printf("Network %s with id %d outputs: (", n.name, n.net_id)
	for i, node := range n.outputs {
		fmt.Printf("[Output #%d: %s] ", i, node)
	}
	fmt.Println(")")
}
func (n *network) PrintInput() {
	fmt.Printf("Network %s with id %d inputs: (", n.name, n.net_id)
	for i, node := range n.inputs {
		fmt.Printf("[Input #%d: %s] ", i, node)
	}
	fmt.Println(")")
}
func (n *network) OutputIsOff() bool {
	for _, node := range n.outputs {
		if node.ActivationsCount == 0 {
			return true
		}
	}
	return false
}
func (n *network) Activate() (bool, error) {
	//For adding to the activesum
	add_amount := 0.0
	//Make sure we at least activate once
	one_time := false
	//Used in case the output is somehow truncated from the network
	abort_count := 0

	// The sigmoid activator function
	sigmoid := ActivationFunc(SigmoidActivation)

	// Keep activating until all the outputs have become active
	// (This only happens on the first activation, because after that they are always active)
	for n.OutputIsOff() || !one_time {
		abort_count += 1

		if abort_count >= 20 {
			return false, errors.New("Inputs disconnected from outputs!")
		}

		// For each neuron node, compute the sum of its incoming activation
		for _, np := range n.all_nodes {
			if np.IsNeuron() {
				np.ActivationSum = 0.0 // reset activation value
				np.IsActive = false // flag node disabled

				// For each node's incoming connection, add the activity from the connection to the activesum
				for _, link := range np.Incoming {
					// Handle possible time delays
					if !link.IsTimeDelayed {
						add_amount = link.Weight * link.InNode.GetActiveOut()
						//fmt.Printf("%f -> %f\n", link.Weight, (*link.InNode).GetActiveOut())
						if link.InNode.IsActive || link.InNode.IsSensor() {
							np.IsActive = true
						}
					} else {
						add_amount = link.Weight * link.InNode.GetActiveOutTd()
					}
					np.ActivationSum += add_amount
				} // End {for} over incoming links
			} // End if != SENSOR
		}  // End {for} over all nodes

		// Now activate all the neuron nodes off their incoming activation
		for _, np := range n.all_nodes {
			if np.IsNeuron() {
				// Only activate if some active input came in
				if np.IsActive {
					// Keep a memory of activations for potential time delayed connections
					np.saveActivations()
					// Now run the net activation through an activation function
					if np.FType == SIGMOID {
						np.Activation = sigmoid.Activation(np, 4.924273, 2.4621365)
					} else {
						return false, errors.New(
							fmt.Sprintf("Unknown activation function type: %d", np.FType))
					}
					// Increment the activation_count
					// First activation cannot be from nothing!!
					np.ActivationsCount++
				}
				//fmt.Printf("Node: %s, activation sum: %f, active: %t\n", np, np.ActivationSum, np.IsActive)
			}
		}
		one_time = true
	}
	return true, nil
}
func (n *network) AddInputNode(node *NNode) {
	n.inputs = append(n.inputs, node)
}
func (n *network) AddOutputNode(node *NNode) {
	n.outputs = append(n.outputs, node)
}
func (n *network) LoadSensors(sensors []float64) {
	counter := 0
	for _, node := range n.inputs {
		if node.IsSensor() {
			node.SensorLoad(sensors[counter])
			counter += 1
		}
	}
}
func (n *network) SetName(name string) {
	n.name = name
}
func (n network) NodeCount() int {
	return len(n.all_nodes)
}
func (n network) LinkCount() int {
	n.numlinks = 0
	for _, node := range n.all_nodes {
		n.numlinks += len(node.Incoming)
	}
	return n.numlinks
}

func (n *network) IsRecurrent(in_node, out_node *NNode, count *int, thresh int) bool {
	// Count the node as visited
	*count++

	if *count > thresh {
		return false // Short out the whole thing - loop detected
	}

	if in_node == out_node {
		return true
	} else {
		// Check back on all links ...
		for _, link := range in_node.Incoming {
			// But skip links that are already recurrent -
			// We want to check back through the forward flow of signals only
			if link.IsRecurrent != true {
				if n.IsRecurrent(link.InNode, out_node, count, thresh) {
					return true
				}
			}
		}
	}
	return false
}

func (n *network) MaxDepth() (int32, error) {
	max := int32(0) // The max depth
	for _, node := range n.outputs {
		curr_depth, err := node.Depth(0)
		if err != nil {
			return curr_depth, err
		}
		if curr_depth > max {
			max = curr_depth
		}
	}
	return max, nil
}

func (n *network) AllNodes() []*NNode {
	return n.all_nodes
}
<commit_msg>Removed obsolete import producing import cycle<commit_after>package network

import (
	"fmt"
	"errors"
)

// A NETWORK is a LIST of input NODEs and a LIST of output NODEs.
// The point of the network is to define a single entity which can evolve
// or learn on its own, even though it may be part of a larger framework.
type Network interface {
	// Puts the network back into an initial state
	Flush()
	// Activates the net such that all outputs are active
	Activate() (bool, error)
	// If at least one output is not active then return true
	OutputIsOff() bool

	// Prints the values of network outputs to the console
	PrintActivation()
	// Print the values of network inputs to the console
	PrintInput()
	// Verify that network was successfully flushed for debugging
	FlushCheck() error

	// Adds a new input node
	AddInputNode(node *NNode)
	// Adds a new output node
	AddOutputNode(node *NNode)

	// Takes an array of sensor values and loads it into SENSOR inputs ONLY
	LoadSensors(sensors []float64)
	// Set network name
	SetName(name string)

	// This checks a POTENTIAL link between a potential in_node
     	// and potential out_node to see if it must be recurrent.
	// Use count and thresh to jump out in the case of an infinite loop.
	IsRecurrent(potin_node, potout_node *NNode, count *int, thresh int) bool
	// Find the maximum number of neurons between an output and an input
	MaxDepth() (int32, error)

	// Counts the number of nodes in the net
	NodeCount() int
	// Counts the number of links in the net
	LinkCount() int

	// Returns all nodes in the network
	AllNodes() []*NNode
}

// Creates new network
func NewNetwork(in, out, all []*NNode, netid int32) Network {
	n := newNetwork(netid)
	n.inputs = in
	n.outputs = out
	n.all_nodes = all
	return &n
}

// The default private constructor
func newNetwork(netId int32) network {
	return network {
		numlinks:-1,
		net_id:netId,
	}
}

// The private network data holder
type network struct {
	//The number of links in the net (-1 means not yet counted)
	numlinks int

	// A list of all the nodes in the network
	all_nodes []*NNode
	// NNodes that input into the network
	inputs []*NNode
	// NNodes that output from the network
	outputs []*NNode

	// A network id
	net_id int32

	// Is a name of this network */
	name string
}

// The Network interface implementation
func (n *network) Flush() {
	// Flush back recursively
	for _, node := range n.all_nodes {
		node.Flushback()
	}
}
func (n *network) FlushCheck() error {
	for _, node := range n.all_nodes {
		err := node.FlushbackCheck()
		if err != nil {
			return err
		}
	}
	return nil
}
func (n *network) PrintActivation() {
	fmt.Printf("Network %s with id %d outputs: (", n.name, n.net_id)
	for i, node := range n.outputs {
		fmt.Printf("[Output #%d: %s] ", i, node)
	}
	fmt.Println(")")
}
func (n *network) PrintInput() {
	fmt.Printf("Network %s with id %d inputs: (", n.name, n.net_id)
	for i, node := range n.inputs {
		fmt.Printf("[Input #%d: %s] ", i, node)
	}
	fmt.Println(")")
}
func (n *network) OutputIsOff() bool {
	for _, node := range n.outputs {
		if node.ActivationsCount == 0 {
			return true
		}
	}
	return false
}
func (n *network) Activate() (bool, error) {
	//For adding to the activesum
	add_amount := 0.0
	//Make sure we at least activate once
	one_time := false
	//Used in case the output is somehow truncated from the network
	abort_count := 0

	// The sigmoid activator function
	sigmoid := ActivationFunc(SigmoidActivation)

	// Keep activating until all the outputs have become active
	// (This only happens on the first activation, because after that they are always active)
	for n.OutputIsOff() || !one_time {
		abort_count += 1

		if abort_count >= 20 {
			return false, errors.New("Inputs disconnected from outputs!")
		}

		// For each neuron node, compute the sum of its incoming activation
		for _, np := range n.all_nodes {
			if np.IsNeuron() {
				np.ActivationSum = 0.0 // reset activation value
				np.IsActive = false // flag node disabled

				// For each node's incoming connection, add the activity from the connection to the activesum
				for _, link := range np.Incoming {
					// Handle possible time delays
					if !link.IsTimeDelayed {
						add_amount = link.Weight * link.InNode.GetActiveOut()
						//fmt.Printf("%f -> %f\n", link.Weight, (*link.InNode).GetActiveOut())
						if link.InNode.IsActive || link.InNode.IsSensor() {
							np.IsActive = true
						}
					} else {
						add_amount = link.Weight * link.InNode.GetActiveOutTd()
					}
					np.ActivationSum += add_amount
				} // End {for} over incoming links
			} // End if != SENSOR
		}  // End {for} over all nodes

		// Now activate all the neuron nodes off their incoming activation
		for _, np := range n.all_nodes {
			if np.IsNeuron() {
				// Only activate if some active input came in
				if np.IsActive {
					// Keep a memory of activations for potential time delayed connections
					np.saveActivations()
					// Now run the net activation through an activation function
					if np.FType == SIGMOID {
						np.Activation = sigmoid.Activation(np, 4.924273, 2.4621365)
					} else {
						return false, errors.New(
							fmt.Sprintf("Unknown activation function type: %d", np.FType))
					}
					// Increment the activation_count
					// First activation cannot be from nothing!!
					np.ActivationsCount++
				}
				//fmt.Printf("Node: %s, activation sum: %f, active: %t\n", np, np.ActivationSum, np.IsActive)
			}
		}
		one_time = true
	}
	return true, nil
}
func (n *network) AddInputNode(node *NNode) {
	n.inputs = append(n.inputs, node)
}
func (n *network) AddOutputNode(node *NNode) {
	n.outputs = append(n.outputs, node)
}
func (n *network) LoadSensors(sensors []float64) {
	counter := 0
	for _, node := range n.inputs {
		if node.IsSensor() {
			node.SensorLoad(sensors[counter])
			counter += 1
		}
	}
}
func (n *network) SetName(name string) {
	n.name = name
}
func (n network) NodeCount() int {
	return len(n.all_nodes)
}
func (n network) LinkCount() int {
	n.numlinks = 0
	for _, node := range n.all_nodes {
		n.numlinks += len(node.Incoming)
	}
	return n.numlinks
}

func (n *network) IsRecurrent(in_node, out_node *NNode, count *int, thresh int) bool {
	// Count the node as visited
	*count++

	if *count > thresh {
		return false // Short out the whole thing - loop detected
	}

	if in_node == out_node {
		return true
	} else {
		// Check back on all links ...
		for _, link := range in_node.Incoming {
			// But skip links that are already recurrent -
			// We want to check back through the forward flow of signals only
			if link.IsRecurrent != true {
				if n.IsRecurrent(link.InNode, out_node, count, thresh) {
					return true
				}
			}
		}
	}
	return false
}

func (n *network) MaxDepth() (int32, error) {
	max := int32(0) // The max depth
	for _, node := range n.outputs {
		curr_depth, err := node.Depth(0)
		if err != nil {
			return curr_depth, err
		}
		if curr_depth > max {
			max = curr_depth
		}
	}
	return max, nil
}

func (n *network) AllNodes() []*NNode {
	return n.all_nodes
}
<|endoftext|>


<commit_before>// rabbitmq provides a concrete client implementation using
// rabbitmq / amqp as a message bus

package client

import (
	"fmt"
	"os"
	"sync"
	"time"

	log "github.com/cihub/seelog"
	"github.com/golang/protobuf/proto"
	uuid "github.com/nu7hatch/gouuid"
	"github.com/streadway/amqp"
	"golang.org/x/net/context"

	"github.com/b2aio/typhon/errors"
	"github.com/b2aio/typhon/rabbit"

	pe "github.com/b2aio/typhon/proto/error"
)

var connectionTimeout time.Duration = 10 * time.Second

type RabbitClient struct {
	once       sync.Once
	inflight   *inflightRegistry
	replyTo    string
	connection *rabbit.RabbitConnection
}

var NewRabbitClient = func() Client {
	uuidQueue, err := uuid.NewV4()
	if err != nil {
		log.Criticalf("[Client] Failed to create UUID for reply queue")
		os.Exit(1)
	}
	return &RabbitClient{
		inflight:   newInflightRegistry(),
		connection: rabbit.NewRabbitConnection(),
		replyTo:    fmt.Sprintf("replyTo-%s", uuidQueue.String()),
	}
}

func (c *RabbitClient) Init() {
	select {
	case <-c.connection.Init():
		log.Info("[Client] Connected to RabbitMQ")
	case <-time.After(connectionTimeout):
		log.Critical("[Client] Failed to connect to RabbitMQ after %v", connectionTimeout)
		os.Exit(1)
	}
	c.initConsume()
}

func (c *RabbitClient) initConsume() {
	err := c.connection.Channel.DeclareReplyQueue(c.replyTo)
	if err != nil {
		log.Criticalf("[Client] Failed to declare reply queue: %s", err.Error())
		os.Exit(1)
	}
	deliveries, err := c.connection.Channel.ConsumeQueue(c.replyTo)
	if err != nil {
		log.Criticalf("[Client] Failed to consume from reply queue: %s", err.Error())
		os.Exit(1)
	}
	go func() {
		log.Infof("[Client] Listening for deliveries on %s", c.replyTo)
		for delivery := range deliveries {
			go c.handleDelivery(delivery)
		}
		log.Infof("[Client] Delivery channel %s closed", c.replyTo)
	}()
}

func (c *RabbitClient) handleDelivery(delivery amqp.Delivery) {
	channel := c.inflight.pop(delivery.CorrelationId)
	if channel == nil {
		log.Warnf("[Client] CorrelationID '%s' does not exist in inflight registry", delivery.CorrelationId)
		return
	}
	select {
	case channel <- delivery:
		log.Tracef("[Client] Dispatched delivery to response channel for %s", delivery.CorrelationId)
	default:
		log.Warnf("[Client] Error in delivery for message %s", delivery.CorrelationId)
	}
}

func (c *RabbitClient) Req(ctx context.Context, service, endpoint string, req proto.Message, resp proto.Message) error {

	// Build request
	payload, err := proto.Marshal(req)
	if err != nil {
		log.Errorf("[Client] Failed to marshal request: %v", err)
		return errors.Wrap(err) // @todo custom error code
	}
	protoReq, err := NewProtoRequest(service, endpoint, payload)
	if err != nil {
		return errors.Wrap(err)
	}

	// Execute
	rsp, err := c.do(protoReq)
	if err != nil {
		return err
	}

	// Unmarshal response into the provided pointer
	if err := unmarshalResponse(rsp, resp); err != nil {
		return err
	}

	return nil
}

// CustomReq makes a sends a request to a service and returns a
// response without the usual marshaling helpers
func (c *RabbitClient) CustomReq(req Request) (Response, error) {
	return c.do(req)
}

// do sends a request and returns a response, following policies
// (e.g. redirects, cookies, auth) as configured on the client.
func (c *RabbitClient) do(req Request) (Response, error) {

	// Ensure we're initialised, but only do this once
	//
	// @todo we need a connection loop here where we check if we're connected,
	// and if not, block for a short period of time while attempting to reconnect
	c.once.Do(c.Init)

	// Don't even try to send if not connected
	if !c.connection.IsConnected() {
		return nil, errors.Wrap(fmt.Errorf("Not connected to AMQP"))
	}

	replyChannel := c.inflight.push(req.Id())

	routingKey := buildRoutingKey(req.Service(), req.Endpoint())
	log.Debugf("[Client] Dispatching request to %s with correlation ID %s", routingKey, req.Id())

	// Build message from request
	message := amqp.Publishing{
		CorrelationId: req.Id(),
		Timestamp:     time.Now().UTC(),
		Body:          req.Payload(),
		ReplyTo:       c.replyTo,
		Headers: amqp.Table{
			"Content-Type":     req.ContentType(),
			"Content-Encoding": "request",
			"Service":          req.Service(),
			"Endpoint":         req.Endpoint(),
		},
	}

	err := c.connection.Publish(rabbit.Exchange, routingKey, message)
	if err != nil {
		log.Errorf("[Client] Failed to publish %s to '%s': %v", req.Id(), routingKey, err)
		return nil, errors.Wrap(err) // @todo custom error code
	}

	select {
	case delivery := <-replyChannel:
		log.Debugf("[Client] Response received for %s from %s", req.Id(), routingKey)
		rsp := deliveryToResponse(delivery)
		if rsp.IsError() {
			return nil, unmarshalErrorResponse(rsp)
		}
		return rsp, nil
	case <-time.After(defaultTimeout):
		log.Errorf("[Client] Request %s timed out calling %s", req.Id(), routingKey)

		return nil, errors.Timeout(fmt.Sprintf("%s timed out", routingKey), nil, map[string]string{
			"called_service":  req.Service(),
			"called_endpoint": req.Endpoint(),
		})
	}

}

// buildRoutingKey to send the request via AMQP
func buildRoutingKey(serviceName, endpoint string) string {
	return fmt.Sprintf("%s.%s", serviceName, endpoint)
}

// unmarshalResponse returned from a service into the response type
func unmarshalResponse(resp Response, respProto proto.Message) error {
	if err := proto.Unmarshal(resp.Payload(), respProto); err != nil {
		return errors.BadResponse(err.Error())
	}

	return nil
}

// deliveryToResponse converts our AMQP response to a client Response
func deliveryToResponse(delivery amqp.Delivery) Response {

	contentType, _ := delivery.Headers["Content-Type"].(string)
	contentEncoding, _ := delivery.Headers["Content-Encoding"].(string)
	service, _ := delivery.Headers["Service"].(string)
	endpoint, _ := delivery.Headers["Endpoint"].(string)

	return &response{
		contentType:     contentType,
		contentEncoding: contentEncoding,
		service:         service,
		endpoint:        endpoint,
		payload:         delivery.Body,
	}
}

// unmarshalErrorResponse from our wire format to a typhon error
func unmarshalErrorResponse(resp Response) *errors.Error {
	p := &pe.Error{}
	if err := proto.Unmarshal(resp.Payload(), p); err != nil {
		return errors.BadResponse(err.Error())
	}

	return errors.Unmarshal(p)
}
<commit_msg>Fix error response when unmarshaling transmitted errors<commit_after>// rabbitmq provides a concrete client implementation using
// rabbitmq / amqp as a message bus

package client

import (
	"fmt"
	"os"
	"sync"
	"time"

	log "github.com/cihub/seelog"
	"github.com/golang/protobuf/proto"
	uuid "github.com/nu7hatch/gouuid"
	"github.com/streadway/amqp"
	"golang.org/x/net/context"

	"github.com/b2aio/typhon/errors"
	"github.com/b2aio/typhon/rabbit"

	pe "github.com/b2aio/typhon/proto/error"
)

var connectionTimeout time.Duration = 10 * time.Second

type RabbitClient struct {
	once       sync.Once
	inflight   *inflightRegistry
	replyTo    string
	connection *rabbit.RabbitConnection
}

var NewRabbitClient = func() Client {
	uuidQueue, err := uuid.NewV4()
	if err != nil {
		log.Criticalf("[Client] Failed to create UUID for reply queue")
		os.Exit(1)
	}
	return &RabbitClient{
		inflight:   newInflightRegistry(),
		connection: rabbit.NewRabbitConnection(),
		replyTo:    fmt.Sprintf("replyTo-%s", uuidQueue.String()),
	}
}

func (c *RabbitClient) Init() {
	select {
	case <-c.connection.Init():
		log.Info("[Client] Connected to RabbitMQ")
	case <-time.After(connectionTimeout):
		log.Critical("[Client] Failed to connect to RabbitMQ after %v", connectionTimeout)
		os.Exit(1)
	}
	c.initConsume()
}

func (c *RabbitClient) initConsume() {
	err := c.connection.Channel.DeclareReplyQueue(c.replyTo)
	if err != nil {
		log.Criticalf("[Client] Failed to declare reply queue: %s", err.Error())
		os.Exit(1)
	}
	deliveries, err := c.connection.Channel.ConsumeQueue(c.replyTo)
	if err != nil {
		log.Criticalf("[Client] Failed to consume from reply queue: %s", err.Error())
		os.Exit(1)
	}
	go func() {
		log.Infof("[Client] Listening for deliveries on %s", c.replyTo)
		for delivery := range deliveries {
			go c.handleDelivery(delivery)
		}
		log.Infof("[Client] Delivery channel %s closed", c.replyTo)
	}()
}

func (c *RabbitClient) handleDelivery(delivery amqp.Delivery) {
	channel := c.inflight.pop(delivery.CorrelationId)
	if channel == nil {
		log.Warnf("[Client] CorrelationID '%s' does not exist in inflight registry", delivery.CorrelationId)
		return
	}
	select {
	case channel <- delivery:
		log.Tracef("[Client] Dispatched delivery to response channel for %s", delivery.CorrelationId)
	default:
		log.Warnf("[Client] Error in delivery for message %s", delivery.CorrelationId)
	}
}

func (c *RabbitClient) Req(ctx context.Context, service, endpoint string, req proto.Message, resp proto.Message) error {

	// Build request
	payload, err := proto.Marshal(req)
	if err != nil {
		log.Errorf("[Client] Failed to marshal request: %v", err)
		return errors.Wrap(err) // @todo custom error code
	}
	protoReq, err := NewProtoRequest(service, endpoint, payload)
	if err != nil {
		return errors.Wrap(err)
	}

	// Execute
	rsp, err := c.do(protoReq)
	if err != nil {
		return err
	}

	// Unmarshal response into the provided pointer
	if err := unmarshalResponse(rsp, resp); err != nil {
		return err
	}

	return nil
}

// CustomReq makes a sends a request to a service and returns a
// response without the usual marshaling helpers
func (c *RabbitClient) CustomReq(req Request) (Response, error) {
	return c.do(req)
}

// do sends a request and returns a response, following policies
// (e.g. redirects, cookies, auth) as configured on the client.
func (c *RabbitClient) do(req Request) (Response, error) {

	// Ensure we're initialised, but only do this once
	//
	// @todo we need a connection loop here where we check if we're connected,
	// and if not, block for a short period of time while attempting to reconnect
	c.once.Do(c.Init)

	// Don't even try to send if not connected
	if !c.connection.IsConnected() {
		return nil, errors.Wrap(fmt.Errorf("Not connected to AMQP"))
	}

	replyChannel := c.inflight.push(req.Id())

	routingKey := buildRoutingKey(req.Service(), req.Endpoint())
	log.Debugf("[Client] Dispatching request to %s with correlation ID %s", routingKey, req.Id())

	// Build message from request
	message := amqp.Publishing{
		CorrelationId: req.Id(),
		Timestamp:     time.Now().UTC(),
		Body:          req.Payload(),
		ReplyTo:       c.replyTo,
		Headers: amqp.Table{
			"Content-Type":     req.ContentType(),
			"Content-Encoding": "request",
			"Service":          req.Service(),
			"Endpoint":         req.Endpoint(),
		},
	}

	err := c.connection.Publish(rabbit.Exchange, routingKey, message)
	if err != nil {
		log.Errorf("[Client] Failed to publish %s to '%s': %v", req.Id(), routingKey, err)
		return nil, errors.Wrap(err) // @todo custom error code
	}

	select {
	case delivery := <-replyChannel:
		log.Debugf("[Client] Response received for %s from %s", req.Id(), routingKey)
		rsp := deliveryToResponse(delivery)
		if rsp.IsError() {
			return nil, unmarshalErrorResponse(rsp)
		}
		return rsp, nil
	case <-time.After(defaultTimeout):
		log.Errorf("[Client] Request %s timed out calling %s", req.Id(), routingKey)

		return nil, errors.Timeout(fmt.Sprintf("%s timed out", routingKey), nil, map[string]string{
			"called_service":  req.Service(),
			"called_endpoint": req.Endpoint(),
		})
	}

}

// buildRoutingKey to send the request via AMQP
func buildRoutingKey(serviceName, endpoint string) string {
	return fmt.Sprintf("%s.%s", serviceName, endpoint)
}

// unmarshalResponse returned from a service into the response type
func unmarshalResponse(resp Response, respProto proto.Message) error {
	if err := proto.Unmarshal(resp.Payload(), respProto); err != nil {
		return errors.BadResponse(err.Error())
	}

	return nil
}

// deliveryToResponse converts our AMQP response to a client Response
func deliveryToResponse(delivery amqp.Delivery) Response {

	contentType, _ := delivery.Headers["Content-Type"].(string)
	contentEncoding, _ := delivery.Headers["Content-Encoding"].(string)
	service, _ := delivery.Headers["Service"].(string)
	endpoint, _ := delivery.Headers["Endpoint"].(string)

	return &response{
		contentType:     contentType,
		contentEncoding: contentEncoding,
		service:         service,
		endpoint:        endpoint,
		payload:         delivery.Body,
	}
}

// unmarshalErrorResponse from our wire format to a typhon error
func unmarshalErrorResponse(resp Response) error {
	p := &pe.Error{}
	if err := proto.Unmarshal(resp.Payload(), p); err != nil {
		return errors.BadResponse(err.Error())
	}

	return errors.Unmarshal(p)
}
<|endoftext|>


<commit_before>// Take well-formed json from either stdin or an input file and create an elasticsearch document to be used to
// generate user specific dashboards or highly contextual alerts.
//
// LICENSE:
//   Copyright 2015 Yieldbot. <devops@yieldbot.com>
//   Released under the MIT License; see LICENSE
//   for details.

package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"github.com/yieldbot/dhuran"
	"github.com/yieldbot/dracky"
	"github.com/olivere/elastic"
	"io/ioutil"
	"os"
	"time"
)

func main() {

	// set commandline flags
	es_indexPtr := flag.String("index", dracky.STATUS_ES_INDEX, "the elasticsearch index to use")
	es_hostPtr := flag.String("host", dracky.DEFAULT_ES_HOST, "the elasticsearch host")
	es_portPtr := flag.String("port", dracky.DEFAULT_ES_PORT, "the elasticsearch port")
	stdinPtr := flag.Bool("read-stdin", true, "read input from stdin")
	//timePtr := flag.string("t-format", "", "time format to suffix on the index name")
	input_filePtr := flag.String("input-file", "", "file to read json in from, check docs for proper format")

	flag.Parse()
	es_index := *es_indexPtr
	es_type := dracky.DEFAULT_ES_TYPE
	es_host := *es_hostPtr
	es_port := *es_portPtr
	rd_stdin := *stdinPtr
	input_file := *input_filePtr

	// I don't want to call these if they are not needed
	sensu_event := new(dracky.Sensu_Event)
	user_event := new(dracky.User_Event)
	//t_format := *timePtr

	sensu_env := dracky.Set_sensu_env()

	//   if t_format != "" {
	//     // get the format of the time
	//     es_index = es_index + t_format
	//   }

	if (rd_stdin == false) && (input_file != "") {
		user_input, err := ioutil.ReadFile(input_file)
		if err != nil {
			dhuran.Check(err)
		}
		err = json.Unmarshal(user_input, &user_event)
		if err != nil {
			dhuran.Check(err)
		}
		es_type = "user"
	} else if (rd_stdin == false) && (input_file == "") {
		fmt.Printf("Please enter a file to read from")
		os.Exit(1)
	} else {
		sensu_event = sensu_event.Acquire_sensu_event()
	}

	// Create a client
	client, err := elastic.NewClient(
		elastic.SetURL("http://" + es_host + ":" + es_port),
	)
	if err != nil {
		dhuran.Check(err)
	}

	// Check to see if the index exists and if not create it
	if client.IndexExists == nil { // need to test to make sure this does what I want
		_, err = client.CreateIndex(es_index).Do()
		if err != nil {
			dhuran.Check(err)
		}
	}

	// Create an Elasticsearch document. The document type will define the mapping used for the document.
	doc := make(map[string]string)
	var doc_id string
	switch es_type {
	case "sensu":
		doc_id = dracky.Event_name(sensu_event.Client.Name, sensu_event.Check.Name)
		doc["monitored_instance"] = sensu_event.Acquire_monitored_instance()
		doc["sensu_client"] = sensu_event.Client.Name
		doc["incident_timestamp"] = time.Unix(sensu_event.Check.Issued, 0).Format(time.RFC822Z)
		doc["check_name"] = dracky.Create_check_name(sensu_event.Check.Name)
		doc["check_state"] = dracky.Define_status(sensu_event.Check.Status)
		doc["sensu_env"] = dracky.Define_sensu_env(sensu_env.Sensu.Environment)
		doc["instance_address"] = sensu_event.Client.Address
		doc["check_state_duration"] = dracky.Define_check_state_duration()
	case "user":
		doc["product"] = user_event.Product
		doc["data"] = user_event.Data
		doc["timestamp"] = time.Unix(sensu_event.Check.Issued, 0).Format(time.RFC822Z) // dracky.Set_time(user_event.Timestamp)
	default:
		fmt.Printf("Type is not correctly set")
		os.Exit(2)
	}

	// Add a document to the Elasticsearch index
	_, err = client.Index().
		Index(es_index).
		Type(es_type).
		Id(doc_id).
		BodyJson(doc).
		Do()
	if err != nil {
		dhuran.Check(err)
	}

	// Log a successful document push to stdout. I don't add the id here as some id's are fixed but
	// the user has the ability to autogenerate an id if they don't want to provide one.
	fmt.Printf("Record added to ES\n")
}
<commit_msg>intital commit<commit_after>// Get the number of open files for a process and compare that against /proc/<pid>/limits and alert if
// over the given threshold.
//
//
// LICENSE:
//   Copyright 2015 Yieldbot. <devops@yieldbot.com>
//   Released under the MIT License; see LICENSE
//   for details.

package main

import (
//	"encoding/json"
	"flag"
	"fmt"
	"github.com/yieldbot/dhuran"
//	"github.com/yieldbot/dracky"
//	"github.com/olivere/elastic"
	// "io/ioutil"
  "strconv"
	"os"
  "os/exec"
//	"time"
)

// Get the pid for the supplied process
func get_pid(app string) string {
  go_pid := strconv.Itoa(os.Getpid())
  fmt.Printf(go_pid)
  ps_aef := exec.Command("ps", "-aef")
  grep_find := exec.Command("grep", app)
  grep_exclude := exec.Command("grep", "-v", go_pid)

  outPipe, err := ps_aef.StdoutPipe()
  if err != nil {
    dhuran.Check(err)
  }

  ps_aef.Start()
  grep_find.Stdin = outPipe

  outGrep, err := grep_find.StdoutPipe()
  if err != nil {
    dhuran.Check(err)
  }
fmt.Printf("one\n")
  // fmt.Printf("%v",outGrep)
  grep_exclude.Stdin = outGrep
fmt.Printf("two\n")
  out, err := grep_exclude.Output()
  fmt.Printf("three\n")
  // fmt.Printf(out)
  if err != nil {

    fmt.Printf("%v\n", err)
    // dhuran.Check(err)
  }
  fmt.Printf("four\n")
  defer outPipe.Close()
  defer outGrep.Close()

fmt.Printf(go_pid)

  return string(out)
}

// Calculate if the value is over a threshold
// func determine_threshold(val int, threshold int) bool {
//
// }
//
// // Get the current number of open file handles for the process
// func get_file_handles(pid string) int {
//
// }

func main() {

	// set commandline flags
	// PidPtr := flag.String("pid", "1", "the pid for the process you wish to check")
	AppPtr := flag.String("app", "sbin/init", "the process name")
	WarnPtr := flag.Int("warn", 75, "the alert warning threshold percentage")
	CritPtr := flag.Int("crit", 75, "the alert critical threshold percentage")

	flag.Parse()
	// PidPtr := *PidPtr
	app := *AppPtr
	warn_threshold := *WarnPtr
	crit_threshold := *CritPtr

	// I don't want to call these if they are not needed
	// sensu_event := new(dracky.Sensu_Event)
	// user_event := new(dracky.User_Event)
	//t_format := *timePtr

	// sensu_env := dracky.Set_sensu_env()

	//   if t_format != "" {
	//     // get the format of the time
	//     es_index = es_index + t_format
	//   }

	if app != "" {
    pid := get_pid(app)
    fmt.Printf(pid)
  } else {
    fmt.Printf("Please enter a process name to check")
		os.Exit(100)
    }

    fmt.Printf(out)
    fmt.Printf("%v , %v\n", warn_threshold, crit_threshold)

	// 	if err != nil {
	// 		dhuran.Check(err)
	// 	}
	// 	err = json.Unmarshal(user_input, &user_event)
	// 	if err != nil {
	// 		dhuran.Check(err)
	// 	}
	// 	es_type = "user"
	// } else if (rd_stdin == false) && (input_file == "") {
	// 	fmt.Printf("Please enter a file to read from")
	// 	os.Exit(1)
	// } else {
	// 	sensu_event = sensu_event.Acquire_sensu_event()
	// }
  //
	// // Create a client
	// client, err := elastic.NewClient(
	// 	elastic.SetURL("http://" + es_host + ":" + es_port),
	// )
	// if err != nil {
	// 	dhuran.Check(err)
	// }
  //
	// // Check to see if the index exists and if not create it
	// if client.IndexExists == nil { // need to test to make sure this does what I want
	// 	_, err = client.CreateIndex(es_index).Do()
	// 	if err != nil {
	// 		dhuran.Check(err)
	// 	}
	// }
  //
	// // Create an Elasticsearch document. The document type will define the mapping used for the document.
	// doc := make(map[string]string)
	// var doc_id string
	// switch es_type {
	// case "sensu":
	// 	doc_id = dracky.Event_name(sensu_event.Client.Name, sensu_event.Check.Name)
	// 	doc["monitored_instance"] = sensu_event.Acquire_monitored_instance()
	// 	doc["sensu_client"] = sensu_event.Client.Name
	// 	doc["incident_timestamp"] = time.Unix(sensu_event.Check.Issued, 0).Format(time.RFC822Z)
	// 	doc["check_name"] = dracky.Create_check_name(sensu_event.Check.Name)
	// 	doc["check_state"] = dracky.Define_status(sensu_event.Check.Status)
	// 	doc["sensu_env"] = dracky.Define_sensu_env(sensu_env.Sensu.Environment)
	// 	doc["instance_address"] = sensu_event.Client.Address
	// 	doc["check_state_duration"] = dracky.Define_check_state_duration()
	// case "user":
	// 	doc["product"] = user_event.Product
	// 	doc["data"] = user_event.Data
	// 	doc["timestamp"] = time.Unix(sensu_event.Check.Issued, 0).Format(time.RFC822Z) // dracky.Set_time(user_event.Timestamp)
	// default:
	// 	fmt.Printf("Type is not correctly set")
	// 	os.Exit(2)
	// }
  //
	// // Add a document to the Elasticsearch index
	// _, err = client.Index().
	// 	Index(es_index).
	// 	Type(es_type).
	// 	Id(doc_id).
	// 	BodyJson(doc).
	// 	Do()
	// if err != nil {
	// 	dhuran.Check(err)
	// }
  //
	// // Log a successful document push to stdout. I don't add the id here as some id's are fixed but
	// // the user has the ability to autogenerate an id if they don't want to provide one.
	// fmt.Printf("Record added to ES\n")
}
<|endoftext|>


<commit_before>package sandbox

import (
	"os"
	"syscall"
	"golang.org/x/sys/unix"
	"testing"
	"runtime"
)

// Test ptraceGetSyscall
func Test_ptraceGetSyscall(t *testing.T) {
	// Disable Go runtime preempt, or I may be switched
	// to another thread, who is not the tracer. Then we'll
	// receive ESRCH (no such tracee).
	runtime.LockOSThread()

	// Create a process to trace.
	// This is a small C program, call nanosleep(1000000) 100 times.
	sysAttr := &syscall.SysProcAttr{Ptrace:true}
	attr := &os.ProcAttr{Sys:sysAttr}
	process, err := os.StartProcess("./nanosleep100.exe",
	  []string{"nanosleep100.exe"}, attr)

	if err != nil {
		t.Fatal("Cannot start process to trace. Error:", err.Error());
	}

	pid := process.Pid
	cnt := 0

	for {
		var wstat unix.WaitStatus
		_, err := unix.Wait4(pid, &wstat, 0, nil)
		if err != nil {
			t.Fatal("Wait4 failed. Error:", err.Error());
		}

		if wstat.Stopped() && wstat.StopSignal() == unix.SIGTRAP {
			// Try to get syscall ID
			syscall, err := ptraceGetSyscall(pid)
			if err != nil {
				t.Error("ptraceGetSyscall failed. Error:", err.Error());
			} else {
				t.Logf("Get syscall with ID %d\n", syscall)
			}
			if syscall == unix.SYS_NANOSLEEP {
				cnt++
			}
		}

		if wstat.Signaled() {
			t.Log("Tracee killed by", wstat.Signal().String())
			break
		}
		if wstat.Exited() {
			t.Logf("Tracee exited with status %d", wstat.ExitStatus())
			break
		}

		err = unix.PtraceSyscall(pid, 0)
		if err != nil {
			t.Fatalf("PtraceSyscall failed. Error:", err)
		}
	}

	// enable preempt
	runtime.UnlockOSThread()

	// check the result
	if cnt != 200 {
		t.Errorf("We have only traced %d syscall enter and leave, " +
		  "expect 200.", cnt);
	}
}
<commit_msg>[sandbox] Put testing tracee into a subdirectory<commit_after>package sandbox

import (
	"os"
	"syscall"
	"golang.org/x/sys/unix"
	"testing"
	"runtime"
)

// Test ptraceGetSyscall
func Test_ptraceGetSyscall(t *testing.T) {
	// Disable Go runtime preempt, or I may be switched
	// to another thread, who is not the tracer. Then we'll
	// receive ESRCH (no such tracee).
	runtime.LockOSThread()

	// Create a process to trace.
	// This is a small C program, call nanosleep(1000000) 100 times.
	sysAttr := &syscall.SysProcAttr{Ptrace:true}
	attr := &os.ProcAttr{Sys:sysAttr}
	process, err := os.StartProcess("./tracee/nsleep100.exe",
	  []string{"nsleep100.exe"}, attr)

	if err != nil {
		t.Fatal("Cannot start process to trace. Error:", err.Error());
	}

	pid := process.Pid
	cnt := 0

	for {
		var wstat unix.WaitStatus
		_, err := unix.Wait4(pid, &wstat, 0, nil)
		if err != nil {
			t.Fatal("Wait4 failed. Error:", err.Error());
		}

		if wstat.Stopped() && wstat.StopSignal() == unix.SIGTRAP {
			// Try to get syscall ID
			syscall, err := ptraceGetSyscall(pid)
			if err != nil {
				t.Error("ptraceGetSyscall failed. Error:", err.Error());
			} else {
				t.Logf("Get syscall with ID %d\n", syscall)
			}
			if syscall == unix.SYS_NANOSLEEP {
				cnt++
			}
		}

		if wstat.Signaled() {
			t.Log("Tracee killed by", wstat.Signal().String())
			break
		}
		if wstat.Exited() {
			t.Logf("Tracee exited with status %d", wstat.ExitStatus())
			break
		}

		err = unix.PtraceSyscall(pid, 0)
		if err != nil {
			t.Fatalf("PtraceSyscall failed. Error:", err)
		}
	}

	// enable preempt
	runtime.UnlockOSThread()

	// check the result
	if cnt != 200 {
		t.Errorf("We have only traced %d syscall enter and leave, " +
		  "expect 200.", cnt);
	}
}
<|endoftext|>