<commit_before>package skiplist

import (
	"bytes"
	"testing"
)

type testStruct struct {
}

func TestNext(t *testing.T) {
	l := New()
	i := l.Iterator()
	if i.Next() {
		t.Errorf("Should not have nodes to iterate for new list")
	}
	l.Insert(2, []byte("Node two"))
	i.Next()
	l.Insert(3, []byte("Node three"))
	i.Next()
	if !bytes.Equal(i.Val(), []byte("Node three")) {
		t.Errorf("Didn't iterate to next node")
	}
	if i.Key() != 3 {
		t.Errorf("Got wrong next node")
	}
}

func TestPrevious(t *testing.T) {
	l := New()
	i := l.Iterator()
	if i.Prev() {
		t.Errorf("Should not find a val if no node")
	}
	l.Insert(2, []byte("Node two"))
	i.Next()

	l.Insert(3, []byte("Node three"))
	i.Next()

	if !i.Prev() {
		t.Errorf("Should have a previous node")
	}
}

func TestValueNoCurrent(t *testing.T) {
	i := &iterable{}
	if i.Val() != nil {
		t.Errorf("Should not find a val if no node")
	}
}

func TestKeyNoCurrent(t *testing.T) {
	i := &iterable{}
	if i.Key() != 0 {
		t.Errorf("Should not find a key if no node")
	}
}
<commit_msg>Remove unused code<commit_after>package skiplist

import (
	"bytes"
	"testing"
)

func TestNext(t *testing.T) {
	l := New()
	i := l.Iterator()
	if i.Next() {
		t.Errorf("Should not have nodes to iterate for new list")
	}
	l.Insert(2, []byte("Node two"))
	i.Next()
	l.Insert(3, []byte("Node three"))
	i.Next()
	if !bytes.Equal(i.Val(), []byte("Node three")) {
		t.Errorf("Didn't iterate to next node")
	}
	if i.Key() != 3 {
		t.Errorf("Got wrong next node")
	}
}

func TestPrevious(t *testing.T) {
	l := New()
	i := l.Iterator()
	if i.Prev() {
		t.Errorf("Should not find a val if no node")
	}
	l.Insert(2, []byte("Node two"))
	i.Next()

	l.Insert(3, []byte("Node three"))
	i.Next()

	if !i.Prev() {
		t.Errorf("Should have a previous node")
	}
}

func TestValueNoCurrent(t *testing.T) {
	i := &iterable{}
	if i.Val() != nil {
		t.Errorf("Should not find a val if no node")
	}
}

func TestKeyNoCurrent(t *testing.T) {
	i := &iterable{}
	if i.Key() != 0 {
		t.Errorf("Should not find a key if no node")
	}
}
<|endoftext|>


<commit_before>package gutscript

// vim:list:

import "testing"

var lextests = []struct {
	input    string
	expected TokenType
	cnt      int
}{
	{"a = 102", T_IDENTIFIER, 1},
	{"a = 102\n", T_NEWLINE, 1},
	{"a = 102\nb = 333\n", T_NEWLINE, 2},
	{"abc123 = 100", T_IDENTIFIER, 1},
	{"af = 102", T_NUMBER, 1},
	{"af = 102", '=', 1},
	{"af = 3.1415926", T_FLOATING, 1},
	{"af = bk", T_IDENTIFIER, 2},
	{"b = 3 * -10", T_IDENTIFIER, 1},
	{"// oneline comment", T_ONELINE_COMMENT, 1},
	{"/* comment */", T_COMMENT, 1},
	{`a = "string content"`, T_STRING, 1},
	{`a = "string content contains quote \""`, T_STRING, 1},
	{`if a > 3
    a = 10
    b = 20
`, T_IF, 1},
	{`if a > 3
    a = 10
    b = 20
else
    a = 1
`, T_ELSE, 1},
	{`if a > 3
    a = 10
    b = 20
elseif a < 10
    b = 2
else
    a = 1
`, T_ELSEIF, 1},
	{`foo :: ->
    a = 10
    b = 20
`, T_FUNCTION_PROTOTYPE, 1},
}

func expectLexItems(items chan *GutsSymType, expectedItems []TokenType) {
	for {
		item := <-items
		if item == nil {
			break
		}
		fmt.Printf("Got token %s: '%s'\n", GetTokenName(int(item.typ)), item.val)
		if item.typ == T_EOF || item.typ == eof {
			break
		}
	}

}

func expectLexInput(t *testing.T, input string, typ TokenType, cnt int) {
	lexer := GutsLex{
		input: input,
		start: 0,
		pos:   0,
		items: make(chan *GutsSymType, 100),
	}
	go lexer.run()

	var found int = 0
	var item *GutsSymType
	for {
		item = <-lexer.items
		if item == nil {
			break
		}
		t.Logf("Got token %s: %s", GetTokenName(int(item.typ)), item.val)
		if item.typ == typ {
			found++
		}
		if item.typ == T_EOF || item.typ == eof {
			break
		}
	}
	if found != cnt {
		t.Fatalf("Expecting token %s x %d", GetTokenName(int(typ)), cnt)
	}
	lexer.close()
}

func BenchmarkLexer(b *testing.B) {
	input := `
	a = 102
	foo = 200
	pi = 3.1415926
	// oneline comment
	`
	var item *GutsSymType
	for i := 0; i < b.N; i++ {
		lexer := GutsLex{
			input: input,
			start: 0,
			pos:   0,
			items: make(chan *GutsSymType, 100),
		}
		go lexer.run()
		for {
			item = <-lexer.items
			if item == nil || item.typ == T_EOF {
				break
			}
		}
		lexer.close()
	}
}

func TestLexer(t *testing.T) {
	for _, test := range lextests {
		t.Log("Testing lex from ", test.input)
		expectLexInput(t, test.input, test.expected, test.cnt)
	}
}
<commit_msg>Fix lexer<commit_after>package gutscript

// vim:list:

import "testing"
import "fmt"

var lextests = []struct {
	input    string
	expected TokenType
	cnt      int
}{
	{"a = 102", T_IDENTIFIER, 1},
	{"a = 102\n", T_NEWLINE, 0},
	{"a = 102\nb = 333\n", T_NEWLINE, 1},
	{"abc123 = 100", T_IDENTIFIER, 1},
	{"af = 102", T_NUMBER, 1},
	{"af = 102", '=', 1},
	{"af = 3.1415926", T_FLOATING, 1},
	{"af = bk", T_IDENTIFIER, 2},
	{"b = 3 * -10", T_IDENTIFIER, 1},
	{"// oneline comment", T_ONELINE_COMMENT, 1},
	{"/* comment */", T_COMMENT, 1},
	{`a = "string content"`, T_STRING, 1},
	{`a = "string content contains quote \""`, T_STRING, 1},
	{`if a > 3
    a = 10
    b = 20
`, T_IF, 1},
	{`if a > 3
    a = 10
    b = 20
else
    a = 1
`, T_ELSE, 1},
	{`if a > 3
    a = 10
    b = 20
elseif a < 10
    b = 2
else
    a = 1
`, T_ELSEIF, 1},
	{`foo :: ->
    a = 10
    b = 20
`, T_FUNCTION_PROTOTYPE, 1},
}

// Given a token type list, check the returned token items
func expectLexItems(items chan *GutsSymType, expectedItems []TokenType) {
	for {
		item := <-items
		if item == nil {
			break
		}
		fmt.Printf("Got token %s: '%s'\n", GetTokenName(int(item.typ)), item.val)
		if item.typ == T_EOF || item.typ == eof {
			break
		}
	}
}

func expectLexInput(t *testing.T, input string, typ TokenType, cnt int) {
	lexer := GutsLex{
		input: input,
		start: 0,
		pos:   0,
		items: make(chan *GutsSymType, 100),
	}
	go lexer.run()

	var found int = 0
	var item *GutsSymType
	for {
		item = <-lexer.items
		if item == nil {
			break
		}
		t.Logf("Got token %s: %s", GetTokenName(int(item.typ)), item.val)
		if item.typ == typ {
			found++
		}
		if item.typ == T_EOF || item.typ == eof {
			break
		}
	}
	if found != cnt {
		t.Fatalf("Expecting token %s x %d", GetTokenName(int(typ)), cnt)
	}
	lexer.close()
}

func BenchmarkLexer(b *testing.B) {
	input := `
	a = 102
	foo = 200
	pi = 3.1415926
	// oneline comment
	`
	var item *GutsSymType
	for i := 0; i < b.N; i++ {
		lexer := GutsLex{
			input: input,
			start: 0,
			pos:   0,
			items: make(chan *GutsSymType, 100),
		}
		go lexer.run()
		for {
			item = <-lexer.items
			if item == nil || item.typ == T_EOF {
				break
			}
		}
		lexer.close()
	}
}

func TestLexer(t *testing.T) {
	for _, test := range lextests {
		t.Log("Testing lex from ", test.input)
		expectLexInput(t, test.input, test.expected, test.cnt)
	}
}
<|endoftext|>


<commit_before>package pipeline

import (
	"errors"
	"fmt"
	log "github.com/Sirupsen/logrus"
	"github.com/boyvanduuren/octorunner/lib/common"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/container"
	"github.com/docker/docker/api/types/network"
	"github.com/docker/docker/api/types/strslice"
	"golang.org/x/net/context"
	"gopkg.in/yaml.v2"
	"io"
	"io/ioutil"
	"regexp"
	"strings"
)

/*
ImageLister implementations can be used to list available images on a Docker host.
*/
type ImageLister interface {
	ImageList(ctx context.Context, options types.ImageListOptions) ([]types.ImageSummary, error)
}

/*
ImagePuller implementations are used to pull Docker images to a Docker host.
*/
type ImagePuller interface {
	ImagePull(ctx context.Context, imageName string, options types.ImagePullOptions) (io.ReadCloser, error)
}

/*
ContainerCreater implementations are used to create containers on a Docker host.
*/
type ContainerCreater interface {
	ContainerCreate(ctx context.Context, config *container.Config, hostConfig *container.HostConfig,
		networkingConfig *network.NetworkingConfig, containerName string) (container.ContainerCreateCreatedBody, error)
}

/*
ExecutionClient implementations are used to execute pipelines.
*/
type ExecutionClient interface {
	ImageLister
	ImagePuller
	ContainerCreater
	ContainerStart(ctx context.Context, containerID string, options types.ContainerStartOptions) error
	ContainerWait(ctx context.Context, containerID string) (int64, error)
	ContainerInspect(ctx context.Context, containerID string) (types.ContainerJSON, error)
	ContainerRemove(ctx context.Context, containerID string, options types.ContainerRemoveOptions) error
	CopyToContainer(ctx context.Context, container, path string, content io.Reader, options types.CopyToContainerOptions) error
}

/*
Pipeline contains an image name, and an array containing commands that are executed when
the pipeline is executed.
When the pipeline is executed, the script array will be concatenated as a single script, of which every
command needs to return 0 for the script to pass as successful.
*/
type Pipeline struct {
	Script []string `yaml:"script"`
	Image  string   `yaml:"image"`
}

const repositoryData string = "repositoryData"

/*
ParseConfig deserializes .octorunner.y[a]ml files contained in code repositories.
See https://github.com/boyvanduuren/octorunner#adding-a-test-to-your-repository.
*/
func ParseConfig(file []byte) (Pipeline, error) {
	var pipelineConfig Pipeline
	err := yaml.Unmarshal(file, &pipelineConfig)
	if err != nil {
		return pipelineConfig, err
	}

	return pipelineConfig, nil
}

// Extracted repositories are mounted as volumes on containers to WORKDIR.
const workDir = "/var/run/octorunner"

/*
Execute a pipeline, and return the exit code of its script.
*/
func (c Pipeline) Execute(ctx context.Context, cli ExecutionClient) (int, error) {
	log.Info("Starting execution of pipeline")

	repoData, ok := ctx.Value(repositoryData).(map[string]string)
	if !ok {
		return -1, errors.New("Error while reading context")
	}

	// look for image on Docker host, if we don't have it we'll pull it
	imageFound, err := imageExists(ctx, cli, c.Image)

	if !imageFound {
		log.Infof("Pulling image \"%s\"", c.Image)
		err := imagePull(ctx, cli, c.Image)
		if err != nil {
			return -1, err
		}
	} else {
		log.Debugf("Image \"%s\" is present", c.Image)
	}

	// create the container
	containerName := containerName(repoData["fullName"], repoData["commitId"])
	containerID, err := containerCreate(ctx, cli, c.Script, c.Image, containerName)
	if err != nil {
		return -1, err
	}

	// copy the working data to workDir
	log.Infof("Copying files from %q to container %q", repoData["fsLocation"], containerID)
	dst, src, out, err := common.CreateTarball(repoData["fsLocation"], workDir)
	if err != nil {
		return -1, fmt.Errorf("Error while preparing tarball: %q", err)
	}
	defer src.Close()
	defer out.Close()
	err = cli.CopyToContainer(ctx, containerID, dst, out, types.CopyToContainerOptions{AllowOverwriteDirWithFile: false})
	if err != nil {
		return -1, fmt.Errorf("Error while coping file(s): %q", err)
	}

	// start the container
	log.Infof("Starting container %q", containerID)
	err = cli.ContainerStart(ctx, containerID, types.ContainerStartOptions{})
	if err != nil {
		return -1, fmt.Errorf("Error while starting container: %q", err)
	}

	// wait until the container is done
	cli.ContainerWait(ctx, containerID)

	// inspect the finished container so we can get the exitcode
	inspectData, err := cli.ContainerInspect(ctx, containerID)
	if err != nil {
		return -1, fmt.Errorf("Error while inspecting container: %q", err)
	}
	log.Infof("Container \"%s\" done, exit code: %d", containerID, inspectData.State.ExitCode)

	log.Debugf("Removing container \"%s\"", containerID)
	err = cli.ContainerRemove(ctx, containerID, types.ContainerRemoveOptions{RemoveVolumes: true})
	if err != nil {
		return -1, fmt.Errorf("Error while removing container: %q", err)
	}

	return inspectData.State.ExitCode, nil
}

/*
Check whether a particular image is available on a Docker host. We need this information to
decide whether or not to pull the image.
*/
func imageExists(ctx context.Context, client ImageLister, imageName string) (bool, error) {
	// check if image exists
	log.Debugf("Looking if image \"%s\" is present on docker host", imageName)
	imageFound := false
	list, err := client.ImageList(ctx, types.ImageListOptions{})
	if err != nil {
		return imageFound, err
	}
	for _, summary := range list {
		for _, tag := range summary.RepoTags {
			if imageName == tag || imageName == strings.Split(tag, ":")[0] {
				imageFound = true
			}
		}
	}

	return imageFound, nil
}

/*
Pull an image to a Docker host so it can be used to create containers.
*/
func imagePull(ctx context.Context, cli ImagePuller, imageName string) error {
	reader, err := cli.ImagePull(ctx, imageName, types.ImagePullOptions{})
	if err != nil {
		return fmt.Errorf("Error while pulling %q: %q", imageName, err)
	}
	buf, err := ioutil.ReadAll(reader)
	if err != nil {
		return fmt.Errorf("Error while pulling %q: %q", imageName, err)
	}
	log.Debugf("%s", buf)

	return nil
}

/*
Create a container using imageName on a Docker host with the given commands passed to "/bin/sh" as entrypoint.
Return the ID assigned to the container by Docker, or an error if something goes wrong.
*/
func containerCreate(ctx context.Context, cli ContainerCreater, commands []string, imageName string,
	containerName string) (string, error) {
	// create the container
	script := strings.Join(commands, " && ")
	log.Debugf("Creating container with entrypoint %q", script)
	container, err := cli.ContainerCreate(ctx,
		&container.Config{
			Image:      imageName,
			Entrypoint: strslice.StrSlice{"/bin/sh", "-c", script},
			WorkingDir: workDir},
		&container.HostConfig{AutoRemove: false},
		&network.NetworkingConfig{},
		containerName)

	if err != nil {
		return "", fmt.Errorf("Error while creating container: %q", err)
	}
	// log warnings if we have some
	if len(container.Warnings) > 0 {
		warnings := make([]string, len(container.Warnings))
		for i, s := range container.Warnings {
			warnings[i] = fmt.Sprintf("%q", s)
		}
		log.Warnf("Received warnings while creating container: %v", warnings)
	}
	log.Debugf("Container with ID %q created", container.ID)

	return container.ID, nil
}

/*
Container names need to match [a-zA-Z_.-], so filter out everything that doesn't match.
Except "-", which is translated to "_".
*/
func containerName(repoFullName string, commitID string) string {
	mapping := func(r rune) rune {
		// Pattern compilation won't fail, so don't check for err
		match, _ := regexp.Match("[a-zA-Z_.-]", []byte{byte(r)})
		if match == false {
			if string(r) == "/" {
				return []rune("_")[0]
			}
			return -1
		}
		return r
	}

	return strings.Join([]string{strings.Map(mapping, repoFullName), commitID}, "-")
}
<commit_msg>WIP Log output to db<commit_after>package pipeline

import (
	"errors"
	"fmt"
	log "github.com/Sirupsen/logrus"
	"github.com/boyvanduuren/octorunner/lib/common"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/container"
	"github.com/docker/docker/api/types/network"
	"github.com/docker/docker/api/types/strslice"
	"golang.org/x/net/context"
	"gopkg.in/yaml.v2"
	"io"
	"io/ioutil"
	"regexp"
	"strings"
	"bufio"
	"github.com/boyvanduuren/octorunner/lib/persist"
)

/*
ImageLister implementations can be used to list available images on a Docker host.
*/
type ImageLister interface {
	ImageList(ctx context.Context, options types.ImageListOptions) ([]types.ImageSummary, error)
}

/*
ImagePuller implementations are used to pull Docker images to a Docker host.
*/
type ImagePuller interface {
	ImagePull(ctx context.Context, imageName string, options types.ImagePullOptions) (io.ReadCloser, error)
}

/*
ContainerCreater implementations are used to create containers on a Docker host.
*/
type ContainerCreater interface {
	ContainerCreate(ctx context.Context, config *container.Config, hostConfig *container.HostConfig,
		networkingConfig *network.NetworkingConfig, containerName string) (container.ContainerCreateCreatedBody, error)
}

/*
ExecutionClient implementations are used to execute pipelines.
*/
type ExecutionClient interface {
	ImageLister
	ImagePuller
	ContainerCreater
	ContainerStart(ctx context.Context, containerID string, options types.ContainerStartOptions) error
	ContainerWait(ctx context.Context, containerID string) (int64, error)
	ContainerInspect(ctx context.Context, containerID string) (types.ContainerJSON, error)
	ContainerRemove(ctx context.Context, containerID string, options types.ContainerRemoveOptions) error
	CopyToContainer(ctx context.Context, container, path string, content io.Reader, options types.CopyToContainerOptions) error
	ContainerLogs(ctx context.Context, container string, options types.ContainerLogsOptions) (io.ReadCloser, error)
}

/*
Pipeline contains an image name, and an array containing commands that are executed when
the pipeline is executed.
When the pipeline is executed, the script array will be concatenated as a single script, of which every
command needs to return 0 for the script to pass as successful.
*/
type Pipeline struct {
	Script []string `yaml:"script"`
	Image  string   `yaml:"image"`
}

const repositoryData string = "repositoryData"

/*
ParseConfig deserializes .octorunner.y[a]ml files contained in code repositories.
See https://github.com/boyvanduuren/octorunner#adding-a-test-to-your-repository.
*/
func ParseConfig(file []byte) (Pipeline, error) {
	var pipelineConfig Pipeline
	err := yaml.Unmarshal(file, &pipelineConfig)
	if err != nil {
		return pipelineConfig, err
	}

	return pipelineConfig, nil
}

// Extracted repositories are mounted as volumes on containers to WORKDIR.
const workDir = "/var/run/octorunner"

/*
Execute a pipeline, and return the exit code of its script.
*/
func (c Pipeline) Execute(ctx context.Context, cli ExecutionClient) (int, error) {
	log.Info("Starting execution of pipeline")

	repoData, ok := ctx.Value(repositoryData).(map[string]string)
	if !ok {
		return -1, errors.New("Error while reading context")
	}

	// look for image on Docker host, if we don't have it we'll pull it
	imageFound, err := imageExists(ctx, cli, c.Image)

	if !imageFound {
		log.Infof("Pulling image \"%s\"", c.Image)
		err := imagePull(ctx, cli, c.Image)
		if err != nil {
			return -1, err
		}
	} else {
		log.Debugf("Image \"%s\" is present", c.Image)
	}

	// create the container
	containerName := containerName(repoData["fullName"], repoData["commitId"])
	containerID, err := containerCreate(ctx, cli, c.Script, c.Image, containerName)
	if err != nil {
		return -1, err
	}

	// copy the working data to workDir
	log.Infof("Copying files from %q to container %q", repoData["fsLocation"], containerID)
	dst, src, out, err := common.CreateTarball(repoData["fsLocation"], workDir)
	if err != nil {
		return -1, fmt.Errorf("Error while preparing tarball: %q", err)
	}
	defer src.Close()
	defer out.Close()
	err = cli.CopyToContainer(ctx, containerID, dst, out, types.CopyToContainerOptions{AllowOverwriteDirWithFile: false})
	if err != nil {
		return -1, fmt.Errorf("Error while coping file(s): %q", err)
	}

	// start the container
	log.Infof("Starting container %q", containerID)
	err = cli.ContainerStart(ctx, containerID, types.ContainerStartOptions{})
	if err != nil {
		return -1, fmt.Errorf("Error while starting container: %q", err)
	}

	go logOutput(ctx, cli, containerID)

	// wait until the container is done
	cli.ContainerWait(ctx, containerID)

	// inspect the finished container so we can get the exitcode
	inspectData, err := cli.ContainerInspect(ctx, containerID)
	if err != nil {
		return -1, fmt.Errorf("Error while inspecting container: %q", err)
	}
	log.Infof("Container \"%s\" done, exit code: %d", containerID, inspectData.State.ExitCode)

	log.Debugf("Removing container \"%s\"", containerID)
	err = cli.ContainerRemove(ctx, containerID, types.ContainerRemoveOptions{RemoveVolumes: true})
	if err != nil {
		return -1, fmt.Errorf("Error while removing container: %q", err)
	}

	return inspectData.State.ExitCode, nil
}

/*
Check whether a particular image is available on a Docker host. We need this information to
decide whether or not to pull the image.
*/
func imageExists(ctx context.Context, client ImageLister, imageName string) (bool, error) {
	// check if image exists
	log.Debugf("Looking if image \"%s\" is present on docker host", imageName)
	imageFound := false
	list, err := client.ImageList(ctx, types.ImageListOptions{})
	if err != nil {
		return imageFound, err
	}
	for _, summary := range list {
		for _, tag := range summary.RepoTags {
			if imageName == tag || imageName == strings.Split(tag, ":")[0] {
				imageFound = true
			}
		}
	}

	return imageFound, nil
}

/*
Pull an image to a Docker host so it can be used to create containers.
*/
func imagePull(ctx context.Context, cli ImagePuller, imageName string) error {
	reader, err := cli.ImagePull(ctx, imageName, types.ImagePullOptions{})
	if err != nil {
		return fmt.Errorf("Error while pulling %q: %q", imageName, err)
	}
	buf, err := ioutil.ReadAll(reader)
	if err != nil {
		return fmt.Errorf("Error while pulling %q: %q", imageName, err)
	}
	log.Debugf("%s", buf)

	return nil
}

/*
Create a container using imageName on a Docker host with the given commands passed to "/bin/sh" as entrypoint.
Return the ID assigned to the container by Docker, or an error if something goes wrong.
*/
func containerCreate(ctx context.Context, cli ContainerCreater, commands []string, imageName string,
	containerName string) (string, error) {
	// create the container
	script := strings.Join(commands, " && ")
	log.Debugf("Creating container with entrypoint %q", script)
	container, err := cli.ContainerCreate(ctx,
		&container.Config{
			Image:      imageName,
			Entrypoint: strslice.StrSlice{"/bin/sh", "-c", script},
			WorkingDir: workDir},
		&container.HostConfig{AutoRemove: false},
		&network.NetworkingConfig{},
		containerName)

	if err != nil {
		return "", fmt.Errorf("Error while creating container: %q", err)
	}
	// log warnings if we have some
	if len(container.Warnings) > 0 {
		warnings := make([]string, len(container.Warnings))
		for i, s := range container.Warnings {
			warnings[i] = fmt.Sprintf("%q", s)
		}
		log.Warnf("Received warnings while creating container: %v", warnings)
	}
	log.Debugf("Container with ID %q created", container.ID)

	return container.ID, nil
}

func logOutput(ctx context.Context, cli ExecutionClient, containerID string) (error) {
	options := types.ContainerLogsOptions{
		ShowStdout: true,
		ShowStderr: true,
		Follow: true,
	}

	rc, err := cli.ContainerLogs(ctx, containerID, options)
	if err != nil {
		return err
	}

	repoData, ok := ctx.Value(repositoryData).(map[string]string)
	if !ok {
		return errors.New("Error while reading context")
	}
	//containerName := containerName(repoData["fullName"], repoData["commitId"])
	log.Debug(repoData["fullName"])
	repoName := strings.Split(repoData["fullName"], "/")[0]
	repoOwner := strings.Split(repoData["fullName"], "/")[1]
	commitID := repoData["commitId"]
	writer, err := persist.CreateOutputWriter(repoName, repoOwner, commitID, "default")
	if err != nil {
		return err
	}

	scanner := bufio.NewScanner(rc)
	for scanner.Scan() {
		writer(scanner.Text())
	}

	if err := scanner.Err(); err != nil {
		return err
	}

	return nil
}

/*
Container names need to match [a-zA-Z_.-], so filter out everything that doesn't match.
Except "-", which is translated to "_".
*/
func containerName(repoFullName string, commitID string) string {
	mapping := func(r rune) rune {
		// Pattern compilation won't fail, so don't check for err
		match, _ := regexp.Match("[a-zA-Z_.-]", []byte{byte(r)})
		if match == false {
			if string(r) == "/" {
				return []rune("_")[0]
			}
			return -1
		}
		return r
	}

	return strings.Join([]string{strings.Map(mapping, repoFullName), commitID}, "-")
}
<|endoftext|>


<commit_before>package libcentrifugo

//go:generate stringer -type=MetricID

import (
	"bytes"
	"encoding/json"
	"errors"
	"os"
	"os/exec"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"

	"github.com/centrifugal/centrifugo/Godeps/_workspace/src/github.com/FZambia/go-logger"
)

// Metrics contains various Centrifugo statistic and metric information aggregated
// once in a configurable interval.
type Metrics struct {
	// NumMsgPublished is how many messages were published into channels.
	NumMsgPublished metricCounter `json:"num_msg_published"`

	// NumMsgQueued is how many messages were put into client queues.
	NumMsgQueued metricCounter `json:"num_msg_queued"`

	// NumMsgSent is how many messages were actually sent into client connections.
	NumMsgSent metricCounter `json:"num_msg_sent"`

	// NumAPIRequests shows amount of requests to server API.
	NumAPIRequests metricCounter `json:"num_api_requests"`

	// NumClientRequests shows amount of requests to client API.
	NumClientRequests metricCounter `json:"num_client_requests"`

	// BytesClientIn shows amount of data in bytes coming into client API.
	BytesClientIn metricCounter `json:"bytes_client_in"`

	// BytesClientOut shows amount of data in bytes coming out if client API.
	BytesClientOut metricCounter `json:"bytes_client_out"`

	// TimeAPIMean shows mean response time in nanoseconds to API requests. DEPRECATED will return 0
	TimeAPIMean int64 `json:"time_api_mean"`

	// TimeClientMean shows mean response time in nanoseconds to client requests. DEPRECATED will return 0
	TimeClientMean int64 `json:"time_client_mean"`

	// TimeAPIMax shows maximum response time to API request. DEPRECATED will return 0
	TimeAPIMax int64 `json:"time_api_max"`

	// TimeClientMax shows maximum response time to client request. DEPRECATED will return 0
	TimeClientMax int64 `json:"time_client_max"`

	// MemSys shows system memory usage in bytes.
	MemSys int64 `json:"memory_sys"`

	// CPU shows cpu usage in percents.
	CPU int64 `json:"cpu_usage"`

	// mu protects from multiple processes updating snapshot values at once
	// but raw counters may still increment atomically while held so it's not a strict
	// point-in-time snapshot of all values.
	mu sync.Mutex
}

type metricCounter struct {
	value             int64
	lastIntervalValue int64
	lastIntervalDelta int64
	// marshalRaw indicates that JSON Marshalling should dump the raw counter rather than
	// the last interval delta. Note that if this is true then this struct is a read-only
	// snapshot and doesn't need to use atomic.LoadInt64 to read the raw values.
	marshalRaw bool
}

// cloneRaw creates a read-only copy of a counter by loading it's raw value
// and copying it's last interval values.
// It is only safe to do while holding counter's parent Metrics mutex to ensure that last
// interval values are not being updated.
// The cloned counter is marked to be marshaled to JSON with the raw value rather than delta.
// Any attempt to mutate the cloned counter will panic.
func (c *metricCounter) cloneRaw() *metricCounter {
	return &metricCounter{
		value:             c.LoadRaw(),
		lastIntervalValue: c.lastIntervalValue,
		lastIntervalDelta: c.lastIntervalDelta,
		marshalRaw:        true,
	}
}

// MarshalJSON converts a counter struct into a single JSON int representing
// the last interval delta since that is what we report in general.
// See marshalRaw definition for more detail
func (c metricCounter) MarshalJSON() ([]byte, error) {
	if c.marshalRaw {
		// No need for atomic load - marshalRaw should only be set
		// on a read-only copy anyway
		return json.Marshal(c.value)
	}
	return json.Marshal(c.lastIntervalDelta)
}

func (c metricCounter) UnmarshalJSON(bs []byte) error {
	// We shouldn't need to Unmarshal raw counters ever
	return json.Unmarshal(bs, &c.lastIntervalDelta)
}

func (c *metricCounter) LoadRaw() int64 {
	return atomic.LoadInt64(&c.value)
}

func (c *metricCounter) LastIn() int64 {
	return atomic.LoadInt64(&c.value)
}

// Inc is equivalent to Add(name, 1)
func (c *metricCounter) Inc() int64 {
	return c.Add(1)
}

// Add adds the given number to the counter and returns the new value.
// Note that we assume all Register calls occur during init and all
// Add calls happen strictly after init such that no lock is needed to lookup
// the counter in read-only map.
func (c *metricCounter) Add(n int64) int64 {
	if c.marshalRaw {
		panic("Attempt to modify a counter that has been cloned as a read-only raw value copy")
	}
	return atomic.AddInt64(&c.value, n)
}

// updateDelta updates the delta value for last interval based on current value and previous value.
// It is not threadsafe and should only be called by Metrics.UpdateSnapshot which is serialised by Mutex.
func (c *metricCounter) updateDelta() {
	now := atomic.LoadInt64(&c.value)
	c.lastIntervalDelta = now - c.lastIntervalValue
	c.lastIntervalValue = now
}

func (m *Metrics) UpdateSnapshot() {
	// We update under a lock to ensure that no other process is also updating
	// snapshot nor dumping the values. Other processes CAN still atomically increment raw
	// values while we go though - we don't guarantee counter values are point-in-time consistent
	// with each other
	m.mu.Lock()
	defer m.mu.Unlock()

	var mem runtime.MemStats
	runtime.ReadMemStats(&mem)

	m.MemSys = int64(mem.Sys)

	cpu, err := cpuUsage()
	if err != nil {
		logger.DEBUG.Println(err)
	}
	m.CPU = cpu

	// Would love to not have to list these explicitly but every alternative is slow
	// or hacky (code generation)
	m.NumMsgPublished.updateDelta()
	m.NumMsgQueued.updateDelta()
	m.NumMsgSent.updateDelta()
	m.NumAPIRequests.updateDelta()
	m.NumClientRequests.updateDelta()
	m.BytesClientIn.updateDelta()
	m.BytesClientOut.updateDelta()
}

// Get RawCounts returns a copy of the current raw counter values.
// The returned value is another instance of Metrics but you should treat it
// as read-only. The only valid operations are to access the raw count values,
// or more likely to marshal it to JSON
func (m *Metrics) GetRawCounts() *Metrics {
	m.mu.Lock()
	defer m.mu.Unlock()

	m2 := Metrics{
		NumMsgPublished:   *m.NumMsgPublished.cloneRaw(),
		NumMsgQueued:      *m.NumMsgQueued.cloneRaw(),
		NumMsgSent:        *m.NumMsgSent.cloneRaw(),
		NumAPIRequests:    *m.NumAPIRequests.cloneRaw(),
		NumClientRequests: *m.NumClientRequests.cloneRaw(),
		BytesClientIn:     *m.BytesClientIn.cloneRaw(),
		BytesClientOut:    *m.BytesClientOut.cloneRaw(),
		MemSys:            m.MemSys,
		CPU:               m.CPU,
	}

	return &m2
}

// cpuUsage is the simplest possible method to extract CPU usage info on most of platforms
// Centrifugo runs. I have not found a more sophisticated cross platform way to extract
// this info without using CGO.
func cpuUsage() (int64, error) {
	cmd := exec.Command("ps", "aux")
	var out bytes.Buffer
	cmd.Stdout = &out
	err := cmd.Run()
	if err != nil {
		return 0, err
	}
	currentPID := os.Getpid()
	for {
		line, err := out.ReadString('\n')
		if err != nil {
			return 0, err
		}
		tokens := strings.Split(line, " ")
		var ft []string
		for _, t := range tokens {
			if t != "" && t != "\t" {
				ft = append(ft, t)
			}
		}
		pid, err := strconv.Atoi(ft[1])
		if err != nil {
			continue
		}
		if pid != currentPID {
			continue
		}
		cpu, err := strconv.ParseFloat(ft[2], 64)
		if err != nil {
			return 0, err
		}
		return int64(cpu), nil
	}
	return 0, errors.New("no cpu info found")
}
<commit_msg>Remove cruft<commit_after>package libcentrifugo

import (
	"bytes"
	"encoding/json"
	"errors"
	"os"
	"os/exec"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"

	"github.com/centrifugal/centrifugo/Godeps/_workspace/src/github.com/FZambia/go-logger"
)

// Metrics contains various Centrifugo statistic and metric information aggregated
// once in a configurable interval.
type Metrics struct {
	// NumMsgPublished is how many messages were published into channels.
	NumMsgPublished metricCounter `json:"num_msg_published"`

	// NumMsgQueued is how many messages were put into client queues.
	NumMsgQueued metricCounter `json:"num_msg_queued"`

	// NumMsgSent is how many messages were actually sent into client connections.
	NumMsgSent metricCounter `json:"num_msg_sent"`

	// NumAPIRequests shows amount of requests to server API.
	NumAPIRequests metricCounter `json:"num_api_requests"`

	// NumClientRequests shows amount of requests to client API.
	NumClientRequests metricCounter `json:"num_client_requests"`

	// BytesClientIn shows amount of data in bytes coming into client API.
	BytesClientIn metricCounter `json:"bytes_client_in"`

	// BytesClientOut shows amount of data in bytes coming out if client API.
	BytesClientOut metricCounter `json:"bytes_client_out"`

	// TimeAPIMean shows mean response time in nanoseconds to API requests. DEPRECATED will return 0
	TimeAPIMean int64 `json:"time_api_mean"`

	// TimeClientMean shows mean response time in nanoseconds to client requests. DEPRECATED will return 0
	TimeClientMean int64 `json:"time_client_mean"`

	// TimeAPIMax shows maximum response time to API request. DEPRECATED will return 0
	TimeAPIMax int64 `json:"time_api_max"`

	// TimeClientMax shows maximum response time to client request. DEPRECATED will return 0
	TimeClientMax int64 `json:"time_client_max"`

	// MemSys shows system memory usage in bytes.
	MemSys int64 `json:"memory_sys"`

	// CPU shows cpu usage in percents.
	CPU int64 `json:"cpu_usage"`

	// mu protects from multiple processes updating snapshot values at once
	// but raw counters may still increment atomically while held so it's not a strict
	// point-in-time snapshot of all values.
	mu sync.Mutex
}

type metricCounter struct {
	value             int64
	lastIntervalValue int64
	lastIntervalDelta int64
	// marshalRaw indicates that JSON Marshalling should dump the raw counter rather than
	// the last interval delta. Note that if this is true then this struct is a read-only
	// snapshot and doesn't need to use atomic.LoadInt64 to read the raw values.
	marshalRaw bool
}

// cloneRaw creates a read-only copy of a counter by loading it's raw value
// and copying it's last interval values.
// It is only safe to do while holding counter's parent Metrics mutex to ensure that last
// interval values are not being updated.
// The cloned counter is marked to be marshaled to JSON with the raw value rather than delta.
// Any attempt to mutate the cloned counter will panic.
func (c *metricCounter) cloneRaw() *metricCounter {
	return &metricCounter{
		value:             c.LoadRaw(),
		lastIntervalValue: c.lastIntervalValue,
		lastIntervalDelta: c.lastIntervalDelta,
		marshalRaw:        true,
	}
}

// MarshalJSON converts a counter struct into a single JSON int representing
// the last interval delta since that is what we report in general.
// See marshalRaw definition for more detail
func (c metricCounter) MarshalJSON() ([]byte, error) {
	if c.marshalRaw {
		// No need for atomic load - marshalRaw should only be set
		// on a read-only copy anyway
		return json.Marshal(c.value)
	}
	return json.Marshal(c.lastIntervalDelta)
}

func (c metricCounter) UnmarshalJSON(bs []byte) error {
	// We shouldn't need to Unmarshal raw counters ever
	return json.Unmarshal(bs, &c.lastIntervalDelta)
}

func (c *metricCounter) LoadRaw() int64 {
	return atomic.LoadInt64(&c.value)
}

func (c *metricCounter) LastIn() int64 {
	return atomic.LoadInt64(&c.value)
}

// Inc is equivalent to Add(name, 1)
func (c *metricCounter) Inc() int64 {
	return c.Add(1)
}

// Add adds the given number to the counter and returns the new value.
// Note that we assume all Register calls occur during init and all
// Add calls happen strictly after init such that no lock is needed to lookup
// the counter in read-only map.
func (c *metricCounter) Add(n int64) int64 {
	if c.marshalRaw {
		panic("Attempt to modify a counter that has been cloned as a read-only raw value copy")
	}
	return atomic.AddInt64(&c.value, n)
}

// updateDelta updates the delta value for last interval based on current value and previous value.
// It is not threadsafe and should only be called by Metrics.UpdateSnapshot which is serialised by Mutex.
func (c *metricCounter) updateDelta() {
	now := atomic.LoadInt64(&c.value)
	c.lastIntervalDelta = now - c.lastIntervalValue
	c.lastIntervalValue = now
}

func (m *Metrics) UpdateSnapshot() {
	// We update under a lock to ensure that no other process is also updating
	// snapshot nor dumping the values. Other processes CAN still atomically increment raw
	// values while we go though - we don't guarantee counter values are point-in-time consistent
	// with each other
	m.mu.Lock()
	defer m.mu.Unlock()

	var mem runtime.MemStats
	runtime.ReadMemStats(&mem)

	m.MemSys = int64(mem.Sys)

	cpu, err := cpuUsage()
	if err != nil {
		logger.DEBUG.Println(err)
	}
	m.CPU = cpu

	// Would love to not have to list these explicitly but every alternative is slow
	// or hacky (code generation)
	m.NumMsgPublished.updateDelta()
	m.NumMsgQueued.updateDelta()
	m.NumMsgSent.updateDelta()
	m.NumAPIRequests.updateDelta()
	m.NumClientRequests.updateDelta()
	m.BytesClientIn.updateDelta()
	m.BytesClientOut.updateDelta()
}

// Get RawCounts returns a copy of the current raw counter values.
// The returned value is another instance of Metrics but you should treat it
// as read-only. The only valid operations are to access the raw count values,
// or more likely to marshal it to JSON
func (m *Metrics) GetRawCounts() *Metrics {
	m.mu.Lock()
	defer m.mu.Unlock()

	m2 := Metrics{
		NumMsgPublished:   *m.NumMsgPublished.cloneRaw(),
		NumMsgQueued:      *m.NumMsgQueued.cloneRaw(),
		NumMsgSent:        *m.NumMsgSent.cloneRaw(),
		NumAPIRequests:    *m.NumAPIRequests.cloneRaw(),
		NumClientRequests: *m.NumClientRequests.cloneRaw(),
		BytesClientIn:     *m.BytesClientIn.cloneRaw(),
		BytesClientOut:    *m.BytesClientOut.cloneRaw(),
		MemSys:            m.MemSys,
		CPU:               m.CPU,
	}

	return &m2
}

// cpuUsage is the simplest possible method to extract CPU usage info on most of platforms
// Centrifugo runs. I have not found a more sophisticated cross platform way to extract
// this info without using CGO.
func cpuUsage() (int64, error) {
	cmd := exec.Command("ps", "aux")
	var out bytes.Buffer
	cmd.Stdout = &out
	err := cmd.Run()
	if err != nil {
		return 0, err
	}
	currentPID := os.Getpid()
	for {
		line, err := out.ReadString('\n')
		if err != nil {
			return 0, err
		}
		tokens := strings.Split(line, " ")
		var ft []string
		for _, t := range tokens {
			if t != "" && t != "\t" {
				ft = append(ft, t)
			}
		}
		pid, err := strconv.Atoi(ft[1])
		if err != nil {
			continue
		}
		if pid != currentPID {
			continue
		}
		cpu, err := strconv.ParseFloat(ft[2], 64)
		if err != nil {
			return 0, err
		}
		return int64(cpu), nil
	}
	return 0, errors.New("no cpu info found")
}
<|endoftext|>


<commit_before>// Copyright 2010 The GoGo Authors. All rights reserved.
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

package main

import "fmt"
import "./libgogo/_obj/libgogo"

const TOKEN_IDENTIFIER uint64 = 1;
const TOKEN_STRING = 2;
const TOKEN_EOS = 3; // end of scan
const TOKEN_LBRAC = 4;
const TOKEN_RBRAC = 5;
const TOKEN_LSBRAC = 6;
const TOKEN_RSBRAC = 7;
const TOKEN_INTEGER = 8;
const TOKEN_LCBRAC = 9;
const TOKEN_RCBRAC = 10;
const TOKEN_PT = 11;
const TOKEN_NOT = 12;
const TOKEN_NOTEQUAL = 13;
const TOKEN_SEMICOLON = 14;
const TOKEN_COLON = 15;
const TOKEN_ASSIGN = 16;
const TOKEN_EQUALS = 17;

type Token struct {
    id uint64;
    value [255]byte;
    value_len uint64;

    // also need a second token, because we may scan it already when terminating the first one
    nextChar byte;
};

func tmp_error ( s string) {
    fmt.Printf("%s\n",s);
    libgogo.Exit(1);
}

//
// Function getting the next token.
//
func GetNextToken(fd uint64, oldToken Token) Token {
    var singleChar byte;
    var inComment uint64;
    var done uint64;
    var space_done uint64;
    var newToken Token;

    done = 0;
    space_done =0;
    newToken.id = 0;
    newToken.value_len = 0;
    newToken.nextChar = 0;
    inComment = 0;

    if oldToken.nextChar == 0 {
        singleChar=libgogo.GetChar(fd)
    } else {
        singleChar = oldToken.nextChar;
    }

    // check if it is a valid read
    if singleChar == 0 {
        newToken.id = TOKEN_EOS;
        done = 1;
        space_done = 1;
    }

    // skip blank, newlines and comments
    // after function singleChar contains the latest valid char
    for ;space_done != 1; {
        if singleChar == '/' {
            if inComment == 0 {
                singleChar = libgogo.GetChar(fd);
                if singleChar == '/' {
                    inComment = 1;
                } else {
                    tmp_error(">> Scanner: Unkown character combination for comments. Exiting.");
                }
            }
        } else {

            // newline
            if singleChar == 10 {
                if inComment == 1 {
                    inComment = 0;
                }
            } else {
                // skip spaces and tabs
                if singleChar != ' ' && singleChar != 9 {
                    if inComment == 0 {
                        space_done = 1;
                    }
                    if singleChar == 0 {
                        space_done = 1;
                        newToken.id = TOKEN_EOS;
                        done = 1;
                    }
                }
            }
        }
        if space_done == 0 {
            singleChar=libgogo.GetChar(fd);
        }
    }

    // get identifiers
    if (done != 1) && (singleChar > 64 && singleChar < 91) || (singleChar > 96 && singleChar < 122) || singleChar == '_' {
        newToken.id = TOKEN_IDENTIFIER;
        for ; (singleChar > 64 && singleChar < 91) || (singleChar > 96 && singleChar < 122) || singleChar == '_'  || (singleChar > 47 && singleChar < 58); singleChar = libgogo.GetChar(fd) {
            newToken.value[newToken.value_len] = singleChar;
            newToken.value_len = newToken.value_len +1;
        }
        newToken.value[newToken.value_len] = 0;
        newToken.nextChar = singleChar;
        done = 1;
    }

    // string "..."
    if (done != 1) && singleChar == '"' {
        newToken.id = TOKEN_STRING;
        for singleChar = libgogo.GetChar(fd); singleChar != '"' &&singleChar > 31 && singleChar < 127;singleChar = libgogo.GetChar(fd) {
            newToken.value[newToken.value_len] = singleChar;
            newToken.value_len = newToken.value_len +1;
        }
        newToken.value[newToken.value_len] = 0;
        if singleChar != '"' {
            tmp_error(">> Scanner: String not closing. Exiting.");
        }
        done = 1;
    }

    // left brace (
    if (done != 1) && singleChar == '(' {
        newToken.id = TOKEN_LBRAC;
        done = 1;
    }

    // right brace )
    if (done != 1) && singleChar == ')' {
        newToken.id = TOKEN_RBRAC;
        done = 1;
    }

    // left square bracket [
    if (done != 1) && singleChar == '[' {
        newToken.id = TOKEN_LSBRAC;
        done = 1;
    }

    // right square bracket ]
    if (done != 1) && singleChar == ']' {
        newToken.id = TOKEN_RSBRAC;
        done = 1;
    }

    // integer
    if (done != 1) && singleChar > 47 && singleChar < 58 {
        newToken.id = TOKEN_INTEGER;
        for ; singleChar > 47 && singleChar < 58 ; singleChar = libgogo.GetChar(fd) {
            newToken.value[newToken.value_len] = singleChar;
            newToken.value_len = newToken.value_len +1;
        }
        newToken.value[newToken.value_len] = 0
        newToken.nextChar = singleChar;
        done = 1;
    }

    if (done != 1) && singleChar == '{' {
        newToken.id = TOKEN_LCBRAC;
        done = 1;
    }

    if (done != 1) && singleChar == '}' {
        newToken.id = TOKEN_RCBRAC;
        done = 1;
    }

    if (done != 1) && singleChar == '.' {
        newToken.id = TOKEN_PT;
        done = 1;
    }

    // not or not equal
    if (done != 1) && singleChar == '!' {
        singleChar = libgogo.GetChar(fd);
        if singleChar == '=' {
            newToken.id = TOKEN_NOTEQUAL;
        } else {
            newToken.id = TOKEN_NOT;
            newToken.nextChar = singleChar;
        }
        done = 1;
    }

    if (done != 1) && singleChar == ';' {
        newToken.id = TOKEN_SEMICOLON;
        done = 1;
    }

    if (done != 1) && singleChar == ',' {
        newToken.id = TOKEN_COLON;
        done = 1;
    }

    if (done != 1) && singleChar == '=' {
        singleChar = libgogo.GetChar(fd);
        if singleChar == '=' {
            newToken.id = TOKEN_EQUALS;
        } else {
            newToken.id = TOKEN_ASSIGN;
            newToken.nextChar = singleChar;
        }
        done = 1;
    }

    if done != 1 {
        fmt.Printf("'%c'\n",singleChar);
        tmp_error(">> Scanner: Unkown char detected. Exiting");
    }

    return newToken;
}

func tmp_print(tok Token) {
    var i int;
    fmt.Printf("Token Id: %d\n",tok.id);
    if tok.id == TOKEN_IDENTIFIER || tok.id == TOKEN_STRING {
        fmt.Printf("Identifier/String value: ");
        for i=0;tok.value[i] != 0;i=i+1 {
            fmt.Printf("%c",tok.value[i]);
        }
        fmt.Printf("\n");
    }
}

func scanner_test(fd uint64) {
    var tok Token;
    tok.id = 0;
    tok.nextChar = 0;

    for tok = GetNextToken(fd,tok); tok.id != TOKEN_EOS; tok = GetNextToken(fd,tok) {
        tmp_print(tok);
    }
}
<commit_msg>scanner: Add a few comments.<commit_after>// Copyright 2010 The GoGo Authors. All rights reserved.
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

package main

import "fmt"
import "./libgogo/_obj/libgogo"

const TOKEN_IDENTIFIER uint64 = 1;
const TOKEN_STRING = 2;
const TOKEN_EOS = 3; // end of scan
const TOKEN_LBRAC = 4;
const TOKEN_RBRAC = 5;
const TOKEN_LSBRAC = 6;
const TOKEN_RSBRAC = 7;
const TOKEN_INTEGER = 8;
const TOKEN_LCBRAC = 9;
const TOKEN_RCBRAC = 10;
const TOKEN_PT = 11;
const TOKEN_NOT = 12;
const TOKEN_NOTEQUAL = 13;
const TOKEN_SEMICOLON = 14;
const TOKEN_COLON = 15;
const TOKEN_ASSIGN = 16;
const TOKEN_EQUALS = 17;

type Token struct {
    id uint64;
    value [255]byte;
    value_len uint64;

    // also need a second token, because we may scan it already when terminating the first one
    nextChar byte;
};

//
// Function getting the next token.
//
func GetNextToken(fd uint64, oldToken Token) Token {
    var singleChar byte;
    var inComment uint64;
    var done uint64;
    var space_done uint64;
    var newToken Token;

    done = 0;
    space_done =0;
    newToken.id = 0;
    newToken.value_len = 0;
    newToken.nextChar = 0;
    inComment = 0;

    if oldToken.nextChar == 0 {
        singleChar=libgogo.GetChar(fd)
    } else {
        singleChar = oldToken.nextChar;
    }

    // check if it is a valid read, or an EOF
    if singleChar == 0 {
        newToken.id = TOKEN_EOS;
        done = 1;
        space_done = 1;
    }

    // skip blank, newlines and comments
    // after function singleChar contains the latest valid char
    for ;space_done != 1; {
        if singleChar == '/' {
            if inComment == 0 {
                singleChar = libgogo.GetChar(fd);
                if singleChar == '/' {
                    inComment = 1;
                } else {
                    tmp_error(">> Scanner: Unkown character combination for comments. Exiting.");
                }
            }
        } else {
            // if character is a newline:
            // 1) if in a comment, exit the comment
            // 2) skip otherwise
            if singleChar == 10 {
                if inComment == 1 {
                    inComment = 0;
                }
            } else {
                // skip spaces and tabs and handle everything else
                if singleChar != ' ' && singleChar != 9 {
                    // if not in a comment we have our current valid char
                    if inComment == 0 {
                        space_done = 1;
                    }
                    // check if GetChar() returned EOF while skipping
                    if singleChar == 0 {
                        space_done = 1;
                        newToken.id = TOKEN_EOS;
                        done = 1;
                    }
                }
            }
        }
        if space_done == 0 {
            singleChar=libgogo.GetChar(fd);
        }
    }

    // catch identifiers.
    // identifier = letter { letter | digit }.
    if (done != 1) && (singleChar >= 'A' && singleChar <= 'Z') || (singleChar >= 'a' && singleChar <= 'z') || singleChar == '_' { // check for letter or _
        newToken.id = TOKEN_IDENTIFIER;
        // preceding characters may be letter,_, or a number
        for ; (singleChar >= 'A' && singleChar <= 'Z') || (singleChar >= 'a' && singleChar <= 'z') || singleChar == '_' || (singleChar >= '0' && singleChar <= '9'); singleChar = libgogo.GetChar(fd) {
            newToken.value[newToken.value_len] = singleChar;
            newToken.value_len = newToken.value_len +1;
        }
        newToken.value[newToken.value_len] = 0;
        // save the last read character for the next GetNextToken() cycle
        newToken.nextChar = singleChar;
        done = 1;
    }

    // string "..."
    if (done != 1) && singleChar == '"' {
        newToken.id = TOKEN_STRING;
        for singleChar = libgogo.GetChar(fd); singleChar != '"' &&singleChar > 31 && singleChar < 127;singleChar = libgogo.GetChar(fd) {
            newToken.value[newToken.value_len] = singleChar;
            newToken.value_len = newToken.value_len +1;
        }
        newToken.value[newToken.value_len] = 0;
        if singleChar != '"' {
            tmp_error(">> Scanner: String not closing. Exiting.");
        }
        done = 1;
    }

    // left brace (
    if (done != 1) && singleChar == '(' {
        newToken.id = TOKEN_LBRAC;
        done = 1;
    }

    // right brace )
    if (done != 1) && singleChar == ')' {
        newToken.id = TOKEN_RBRAC;
        done = 1;
    }

    // left square bracket [
    if (done != 1) && singleChar == '[' {
        newToken.id = TOKEN_LSBRAC;
        done = 1;
    }

    // right square bracket ]
    if (done != 1) && singleChar == ']' {
        newToken.id = TOKEN_RSBRAC;
        done = 1;
    }

    // integer
    if (done != 1) && singleChar > 47 && singleChar < 58 {
        newToken.id = TOKEN_INTEGER;
        for ; singleChar > 47 && singleChar < 58 ; singleChar = libgogo.GetChar(fd) {
            newToken.value[newToken.value_len] = singleChar;
            newToken.value_len = newToken.value_len +1;
        }
        newToken.value[newToken.value_len] = 0
        newToken.nextChar = singleChar;
        done = 1;
    }

    if (done != 1) && singleChar == '{' {
        newToken.id = TOKEN_LCBRAC;
        done = 1;
    }

    if (done != 1) && singleChar == '}' {
        newToken.id = TOKEN_RCBRAC;
        done = 1;
    }

    if (done != 1) && singleChar == '.' {
        newToken.id = TOKEN_PT;
        done = 1;
    }

    // not or not equal
    if (done != 1) && singleChar == '!' {
        singleChar = libgogo.GetChar(fd);
        if singleChar == '=' {
            newToken.id = TOKEN_NOTEQUAL;
        } else {
            newToken.id = TOKEN_NOT;
            newToken.nextChar = singleChar;
        }
        done = 1;
    }

    if (done != 1) && singleChar == ';' {
        newToken.id = TOKEN_SEMICOLON;
        done = 1;
    }

    if (done != 1) && singleChar == ',' {
        newToken.id = TOKEN_COLON;
        done = 1;
    }

    if (done != 1) && singleChar == '=' {
        singleChar = libgogo.GetChar(fd);
        if singleChar == '=' {
            newToken.id = TOKEN_EQUALS;
        } else {
            newToken.id = TOKEN_ASSIGN;
            newToken.nextChar = singleChar;
        }
        done = 1;
    }

    if done != 1 {
        fmt.Printf("'%c'\n",singleChar);
        tmp_error(">> Scanner: Unkown char detected. Exiting");
    }

    return newToken;
}

func tmp_print(tok Token) {
    var i int;
    fmt.Printf("Token Id: %d\n",tok.id);
    if tok.id == TOKEN_IDENTIFIER || tok.id == TOKEN_STRING {
        fmt.Printf("Identifier/String value: ");
        for i=0;tok.value[i] != 0;i=i+1 {
            fmt.Printf("%c",tok.value[i]);
        }
        fmt.Printf("\n");
    }
}

func tmp_error ( s string) {
    fmt.Printf("%s\n",s);
    libgogo.Exit(1);
}

func scanner_test(fd uint64) {
    var tok Token;
    tok.id = 0;
    tok.nextChar = 0;

    for tok = GetNextToken(fd,tok); tok.id != TOKEN_EOS; tok = GetNextToken(fd,tok) {
        tmp_print(tok);
    }
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 by Richard A. Wilkes. All rights reserved.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, version 2.0. If a copy of the MPL was not distributed with
// this file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// This Source Code Form is "Incompatible With Secondary Licenses", as
// defined by the Mozilla Public License, version 2.0.

package ui

import (
	"reflect"
	"time"
)

// Block is the basic graphical block in a window.
type Block struct {
	eventHandlers map[int][]EventHandler
	window        *Window
	parent        Widget
	children      []Widget
	sizer         Sizer
	layout        Layout
	border        Border
	bounds        Rect
	layoutData    interface{}
	background    Color
	needLayout    bool
	disabled      bool
	focusable     bool
	padding       bool // Just here to quiet aligncheck, since there is nothing I can do about it
}

// NewBlock creates a new, empty block.
func NewBlock() *Block {
	return &Block{}
}

// EventHandlers returns the current event handler map.
func (b *Block) EventHandlers() map[int][]EventHandler {
	return b.eventHandlers
}

// AddEventHandler adds an event handler for an event type.
func (b *Block) AddEventHandler(eventType int, handler EventHandler) {
	if b.eventHandlers == nil {
		b.eventHandlers = make(map[int][]EventHandler)
	}
	b.eventHandlers[eventType] = append(b.eventHandlers[eventType], handler)
}

// RemoveEventHandler removes an event handler for an event type.
func (b *Block) RemoveEventHandler(eventType int, handler EventHandler) {
	if b.eventHandlers != nil {
		hPtr := reflect.ValueOf(handler).Pointer()
		handlers := b.eventHandlers[eventType]
		for i, one := range handlers {
			if reflect.ValueOf(one).Pointer() == hPtr {
				if len(handlers) == 1 {
					delete(b.eventHandlers, eventType)
				} else {
					copy(handlers[i:], handlers[i+1:])
					length := len(handlers) - 1
					handlers[length] = nil
					b.eventHandlers[eventType] = handlers[:length]
				}
				break
			}
		}
	}
}

// Sizer implements the Widget interface.
func (b *Block) Sizer() Sizer {
	return b.sizer
}

// SetSizer implements the Widget interface.
func (b *Block) SetSizer(sizer Sizer) {
	b.sizer = sizer
}

// Layout implements the Widget interface.
func (b *Block) Layout() Layout {
	return b.layout
}

// SetLayout implements the Widget interface.
func (b *Block) SetLayout(layout Layout) {
	b.layout = layout
	b.SetNeedLayout(true)
}

// NeedLayout implements the Widget interface.
func (b *Block) NeedLayout() bool {
	return b.needLayout
}

// SetNeedLayout implements the Widget interface.
func (b *Block) SetNeedLayout(needLayout bool) {
	b.needLayout = needLayout
}

// LayoutData implements the Widget interface.
func (b *Block) LayoutData() interface{} {
	return b.layoutData
}

// SetLayoutData implements the Widget interface.
func (b *Block) SetLayoutData(data interface{}) {
	if b.layoutData != data {
		b.layoutData = data
		b.SetNeedLayout(true)
	}
}

// ValidateLayout implements the Widget interface.
func (b *Block) ValidateLayout() {
	if b.NeedLayout() {
		if layout := b.Layout(); layout != nil {
			layout.Layout()
			b.Repaint()
		}
		b.SetNeedLayout(false)
	}
	for _, child := range b.children {
		child.ValidateLayout()
	}
}

// Border implements the Widget interface.
func (b *Block) Border() Border {
	return b.border
}

// SetBorder implements the Widget interface.
func (b *Block) SetBorder(border Border) {
	b.border = border
}

// Repaint implements the Widget interface.
func (b *Block) Repaint() {
	b.RepaintBounds(b.LocalBounds())
}

// RepaintBounds implements the Widget interface.
func (b *Block) RepaintBounds(bounds Rect) {
	bounds.Intersect(b.LocalBounds())
	if !bounds.IsEmpty() {
		if p := b.Parent(); p != nil {
			bounds.X += b.bounds.X
			bounds.Y += b.bounds.Y
			p.RepaintBounds(bounds)
		} else if b.RootOfWindow() {
			b.Window().RepaintBounds(bounds)
		}
	}
}

// Paint implements the Widget interface.
func (b *Block) Paint(g Graphics, dirty Rect) {
	dirty.Intersect(b.LocalBounds())
	if !dirty.IsEmpty() {
		b.paintSelf(g, dirty)
		for _, child := range b.children {
			adjusted := dirty
			adjusted.Intersect(child.Bounds())
			if !adjusted.IsEmpty() {
				b.paintChild(child, g, adjusted)
			}
		}
	}
}

func (b *Block) paintSelf(g Graphics, dirty Rect) {
	g.Save()
	defer g.Restore()
	g.ClipRect(dirty)
	if b.background.Alpha() > 0 {
		g.SetFillColor(b.background)
		g.FillRect(dirty)
	}
	b.paintBorder(g)
	event := &Event{Type: PaintEvent, When: time.Now(), Target: b, GC: g, DirtyRect: dirty}
	event.Dispatch()
}

func (b *Block) paintBorder(g Graphics) {
	if border := b.Border(); border != nil {
		g.Save()
		defer g.Restore()
		border.PaintBorder(g, b.LocalBounds())
	}
}

func (b *Block) paintChild(child Widget, g Graphics, dirty Rect) {
	g.Save()
	defer g.Restore()
	bounds := child.Bounds()
	g.Translate(bounds.X, bounds.Y)
	dirty.X -= bounds.X
	dirty.Y -= bounds.Y
	child.Paint(g, dirty)
}

// Enabled implements the Widget interface.
func (b *Block) Enabled() bool {
	return !b.disabled
}

// SetEnabled implements the Widget interface.
func (b *Block) SetEnabled(enabled bool) {
	disabled := !enabled
	if b.disabled != disabled {
		b.disabled = disabled
		b.Repaint()
	}
}

// Focused implements the Widget interface.
func (b *Block) Focusable() bool {
	return b.focusable && !b.disabled
}

// SetFocused implements the Widget interface.
func (b *Block) SetFocusable(focusable bool) {
	if b.focusable != focusable {
		b.focusable = focusable
	}
}

// Focused implements the Widget interface.
func (b *Block) Focused() bool {
	if window := b.Window(); window != nil {
		return reflect.ValueOf(Widget(b)).Pointer() == reflect.ValueOf(window.Focus()).Pointer()
	}
	return false
}

// Children implements the Widget interface.
func (b *Block) Children() []Widget {
	return b.children
}

// IndexOfChild implements the Widget interface.
func (b *Block) IndexOfChild(child Widget) int {
	for i, one := range b.children {
		if one == child {
			return i
		}
	}
	return -1
}

// AddChild implements the Widget interface.
func (b *Block) AddChild(child Widget) {
	child.RemoveFromParent()
	b.children = append(b.children, child)
	child.SetParent(b)
	b.SetNeedLayout(true)
}

// AddChildAtIndex implements the Widget interface.
func (b *Block) AddChildAtIndex(child Widget, index int) {
	child.RemoveFromParent()
	if index < 0 {
		index = 0
	}
	if index >= len(b.children) {
		b.children = append(b.children, child)
	} else {
		b.children = append(b.children, nil)
		copy(b.children[index+1:], b.children[index:])
		b.children[index] = child
	}
	child.SetParent(b)
	b.SetNeedLayout(true)
}

// RemoveChild implements the Widget interface.
func (b *Block) RemoveChild(child Widget) {
	b.RemoveChildAtIndex(b.IndexOfChild(child))
}

// RemoveChildAtIndex implements the Widget interface.
func (b *Block) RemoveChildAtIndex(index int) {
	if index >= 0 && index < len(b.children) {
		child := b.children[index]
		copy(b.children[index:], b.children[index+1:])
		length := len(b.children) - 1
		b.children[length] = nil
		b.children = b.children[:length]
		b.SetNeedLayout(true)
		child.SetParent(nil)
	}
}

// RemoveFromParent implements the Widget interface.
func (b *Block) RemoveFromParent() {
	if p := b.Parent(); p != nil {
		p.RemoveChild(b)
	}
}

// Parent implements the Widget interface.
func (b *Block) Parent() Widget {
	return b.parent
}

// SetParent implements the Widget interface.
func (b *Block) SetParent(parent Widget) {
	b.parent = parent
}

// Window implements the Widget interface.
func (b *Block) Window() *Window {
	if b.window != nil {
		return b.window
	}
	if b.parent != nil {
		return b.parent.Window()
	}
	return nil
}

// RootOfWindow implements the Widget interface.
func (b *Block) RootOfWindow() bool {
	return b.window != nil
}

// Bounds implements the Widget interface.
func (b *Block) Bounds() Rect {
	return b.bounds
}

// LocalBounds implements the Widget interface.
func (b *Block) LocalBounds() Rect {
	return b.bounds.CopyAndZeroLocation()
}

// LocalInsetBounds implements the Widget interface.
func (b *Block) LocalInsetBounds() Rect {
	bounds := b.LocalBounds()
	if border := b.Border(); border != nil {
		bounds.Inset(border.Insets())
	}
	return bounds
}

// SetBounds implements the Widget interface.
func (b *Block) SetBounds(bounds Rect) {
	moved := b.bounds.X != bounds.X || b.bounds.Y != bounds.Y
	resized := b.bounds.Width != bounds.Width || b.bounds.Height != bounds.Height
	if moved || resized {
		b.Repaint()
		if moved {
			b.bounds.Point = bounds.Point
		}
		if resized {
			b.bounds.Size = bounds.Size
			b.SetNeedLayout(true)
			event := &Event{Type: ResizeEvent, When: time.Now(), Target: b}
			event.Dispatch()
		}
		b.Repaint()
	}
}

// Location implements the Widget interface.
func (b *Block) Location() Point {
	return b.bounds.Point
}

// SetLocation implements the Widget interface.
func (b *Block) SetLocation(pt Point) {
	if b.bounds.Point != pt {
		b.Repaint()
		b.bounds.Point = pt
		b.Repaint()
	}
}

// Size implements the Widget interface.
func (b *Block) Size() Size {
	return b.bounds.Size
}

// SetSize implements the Widget interface.
func (b *Block) SetSize(size Size) {
	if b.bounds.Size != size {
		b.Repaint()
		b.bounds.Size = size
		b.SetNeedLayout(true)
		event := &Event{Type: ResizeEvent, When: time.Now(), Target: b}
		event.Dispatch()
		b.Repaint()
	}
}

// WidgetAt implements the Widget interface.
func (b *Block) WidgetAt(pt Point) Widget {
	for _, child := range b.children {
		bounds := child.Bounds()
		if bounds.Contains(pt) {
			pt.Subtract(bounds.Point)
			return child.WidgetAt(pt)
		}
	}
	return b
}

// ToWindow implements the Widget interface.
func (b *Block) ToWindow(pt Point) Point {
	pt.Add(b.bounds.Point)
	parent := b.parent
	for parent != nil {
		pt.Add(parent.Bounds().Point)
		parent = parent.Parent()
	}
	return pt
}

// FromWindow implements the Widget interface.
func (b *Block) FromWindow(pt Point) Point {
	pt.Subtract(b.bounds.Point)
	parent := b.parent
	for parent != nil {
		pt.Subtract(parent.Bounds().Point)
		parent = parent.Parent()
	}
	return pt
}

// Background implements the Widget interface.
func (b *Block) Background() Color {
	return b.background
}

// SetBackground implements the Widget interface.
func (b *Block) SetBackground(color Color) {
	if color != b.background {
		b.background = color
		b.Repaint()
	}
}
<commit_msg>Fix comments for Focusable and SetFocusable<commit_after>// Copyright (c) 2016 by Richard A. Wilkes. All rights reserved.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, version 2.0. If a copy of the MPL was not distributed with
// this file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// This Source Code Form is "Incompatible With Secondary Licenses", as
// defined by the Mozilla Public License, version 2.0.

package ui

import (
	"reflect"
	"time"
)

// Block is the basic graphical block in a window.
type Block struct {
	eventHandlers map[int][]EventHandler
	window        *Window
	parent        Widget
	children      []Widget
	sizer         Sizer
	layout        Layout
	border        Border
	bounds        Rect
	layoutData    interface{}
	background    Color
	needLayout    bool
	disabled      bool
	focusable     bool
	padding       bool // Just here to quiet aligncheck, since there is nothing I can do about it
}

// NewBlock creates a new, empty block.
func NewBlock() *Block {
	return &Block{}
}

// EventHandlers returns the current event handler map.
func (b *Block) EventHandlers() map[int][]EventHandler {
	return b.eventHandlers
}

// AddEventHandler adds an event handler for an event type.
func (b *Block) AddEventHandler(eventType int, handler EventHandler) {
	if b.eventHandlers == nil {
		b.eventHandlers = make(map[int][]EventHandler)
	}
	b.eventHandlers[eventType] = append(b.eventHandlers[eventType], handler)
}

// RemoveEventHandler removes an event handler for an event type.
func (b *Block) RemoveEventHandler(eventType int, handler EventHandler) {
	if b.eventHandlers != nil {
		hPtr := reflect.ValueOf(handler).Pointer()
		handlers := b.eventHandlers[eventType]
		for i, one := range handlers {
			if reflect.ValueOf(one).Pointer() == hPtr {
				if len(handlers) == 1 {
					delete(b.eventHandlers, eventType)
				} else {
					copy(handlers[i:], handlers[i+1:])
					length := len(handlers) - 1
					handlers[length] = nil
					b.eventHandlers[eventType] = handlers[:length]
				}
				break
			}
		}
	}
}

// Sizer implements the Widget interface.
func (b *Block) Sizer() Sizer {
	return b.sizer
}

// SetSizer implements the Widget interface.
func (b *Block) SetSizer(sizer Sizer) {
	b.sizer = sizer
}

// Layout implements the Widget interface.
func (b *Block) Layout() Layout {
	return b.layout
}

// SetLayout implements the Widget interface.
func (b *Block) SetLayout(layout Layout) {
	b.layout = layout
	b.SetNeedLayout(true)
}

// NeedLayout implements the Widget interface.
func (b *Block) NeedLayout() bool {
	return b.needLayout
}

// SetNeedLayout implements the Widget interface.
func (b *Block) SetNeedLayout(needLayout bool) {
	b.needLayout = needLayout
}

// LayoutData implements the Widget interface.
func (b *Block) LayoutData() interface{} {
	return b.layoutData
}

// SetLayoutData implements the Widget interface.
func (b *Block) SetLayoutData(data interface{}) {
	if b.layoutData != data {
		b.layoutData = data
		b.SetNeedLayout(true)
	}
}

// ValidateLayout implements the Widget interface.
func (b *Block) ValidateLayout() {
	if b.NeedLayout() {
		if layout := b.Layout(); layout != nil {
			layout.Layout()
			b.Repaint()
		}
		b.SetNeedLayout(false)
	}
	for _, child := range b.children {
		child.ValidateLayout()
	}
}

// Border implements the Widget interface.
func (b *Block) Border() Border {
	return b.border
}

// SetBorder implements the Widget interface.
func (b *Block) SetBorder(border Border) {
	b.border = border
}

// Repaint implements the Widget interface.
func (b *Block) Repaint() {
	b.RepaintBounds(b.LocalBounds())
}

// RepaintBounds implements the Widget interface.
func (b *Block) RepaintBounds(bounds Rect) {
	bounds.Intersect(b.LocalBounds())
	if !bounds.IsEmpty() {
		if p := b.Parent(); p != nil {
			bounds.X += b.bounds.X
			bounds.Y += b.bounds.Y
			p.RepaintBounds(bounds)
		} else if b.RootOfWindow() {
			b.Window().RepaintBounds(bounds)
		}
	}
}

// Paint implements the Widget interface.
func (b *Block) Paint(g Graphics, dirty Rect) {
	dirty.Intersect(b.LocalBounds())
	if !dirty.IsEmpty() {
		b.paintSelf(g, dirty)
		for _, child := range b.children {
			adjusted := dirty
			adjusted.Intersect(child.Bounds())
			if !adjusted.IsEmpty() {
				b.paintChild(child, g, adjusted)
			}
		}
	}
}

func (b *Block) paintSelf(g Graphics, dirty Rect) {
	g.Save()
	defer g.Restore()
	g.ClipRect(dirty)
	if b.background.Alpha() > 0 {
		g.SetFillColor(b.background)
		g.FillRect(dirty)
	}
	b.paintBorder(g)
	event := &Event{Type: PaintEvent, When: time.Now(), Target: b, GC: g, DirtyRect: dirty}
	event.Dispatch()
}

func (b *Block) paintBorder(g Graphics) {
	if border := b.Border(); border != nil {
		g.Save()
		defer g.Restore()
		border.PaintBorder(g, b.LocalBounds())
	}
}

func (b *Block) paintChild(child Widget, g Graphics, dirty Rect) {
	g.Save()
	defer g.Restore()
	bounds := child.Bounds()
	g.Translate(bounds.X, bounds.Y)
	dirty.X -= bounds.X
	dirty.Y -= bounds.Y
	child.Paint(g, dirty)
}

// Enabled implements the Widget interface.
func (b *Block) Enabled() bool {
	return !b.disabled
}

// SetEnabled implements the Widget interface.
func (b *Block) SetEnabled(enabled bool) {
	disabled := !enabled
	if b.disabled != disabled {
		b.disabled = disabled
		b.Repaint()
	}
}

// Focusable implements the Widget interface.
func (b *Block) Focusable() bool {
	return b.focusable && !b.disabled
}

// SetFocusable implements the Widget interface.
func (b *Block) SetFocusable(focusable bool) {
	if b.focusable != focusable {
		b.focusable = focusable
	}
}

// Focused implements the Widget interface.
func (b *Block) Focused() bool {
	if window := b.Window(); window != nil {
		return reflect.ValueOf(Widget(b)).Pointer() == reflect.ValueOf(window.Focus()).Pointer()
	}
	return false
}

// Children implements the Widget interface.
func (b *Block) Children() []Widget {
	return b.children
}

// IndexOfChild implements the Widget interface.
func (b *Block) IndexOfChild(child Widget) int {
	for i, one := range b.children {
		if one == child {
			return i
		}
	}
	return -1
}

// AddChild implements the Widget interface.
func (b *Block) AddChild(child Widget) {
	child.RemoveFromParent()
	b.children = append(b.children, child)
	child.SetParent(b)
	b.SetNeedLayout(true)
}

// AddChildAtIndex implements the Widget interface.
func (b *Block) AddChildAtIndex(child Widget, index int) {
	child.RemoveFromParent()
	if index < 0 {
		index = 0
	}
	if index >= len(b.children) {
		b.children = append(b.children, child)
	} else {
		b.children = append(b.children, nil)
		copy(b.children[index+1:], b.children[index:])
		b.children[index] = child
	}
	child.SetParent(b)
	b.SetNeedLayout(true)
}

// RemoveChild implements the Widget interface.
func (b *Block) RemoveChild(child Widget) {
	b.RemoveChildAtIndex(b.IndexOfChild(child))
}

// RemoveChildAtIndex implements the Widget interface.
func (b *Block) RemoveChildAtIndex(index int) {
	if index >= 0 && index < len(b.children) {
		child := b.children[index]
		copy(b.children[index:], b.children[index+1:])
		length := len(b.children) - 1
		b.children[length] = nil
		b.children = b.children[:length]
		b.SetNeedLayout(true)
		child.SetParent(nil)
	}
}

// RemoveFromParent implements the Widget interface.
func (b *Block) RemoveFromParent() {
	if p := b.Parent(); p != nil {
		p.RemoveChild(b)
	}
}

// Parent implements the Widget interface.
func (b *Block) Parent() Widget {
	return b.parent
}

// SetParent implements the Widget interface.
func (b *Block) SetParent(parent Widget) {
	b.parent = parent
}

// Window implements the Widget interface.
func (b *Block) Window() *Window {
	if b.window != nil {
		return b.window
	}
	if b.parent != nil {
		return b.parent.Window()
	}
	return nil
}

// RootOfWindow implements the Widget interface.
func (b *Block) RootOfWindow() bool {
	return b.window != nil
}

// Bounds implements the Widget interface.
func (b *Block) Bounds() Rect {
	return b.bounds
}

// LocalBounds implements the Widget interface.
func (b *Block) LocalBounds() Rect {
	return b.bounds.CopyAndZeroLocation()
}

// LocalInsetBounds implements the Widget interface.
func (b *Block) LocalInsetBounds() Rect {
	bounds := b.LocalBounds()
	if border := b.Border(); border != nil {
		bounds.Inset(border.Insets())
	}
	return bounds
}

// SetBounds implements the Widget interface.
func (b *Block) SetBounds(bounds Rect) {
	moved := b.bounds.X != bounds.X || b.bounds.Y != bounds.Y
	resized := b.bounds.Width != bounds.Width || b.bounds.Height != bounds.Height
	if moved || resized {
		b.Repaint()
		if moved {
			b.bounds.Point = bounds.Point
		}
		if resized {
			b.bounds.Size = bounds.Size
			b.SetNeedLayout(true)
			event := &Event{Type: ResizeEvent, When: time.Now(), Target: b}
			event.Dispatch()
		}
		b.Repaint()
	}
}

// Location implements the Widget interface.
func (b *Block) Location() Point {
	return b.bounds.Point
}

// SetLocation implements the Widget interface.
func (b *Block) SetLocation(pt Point) {
	if b.bounds.Point != pt {
		b.Repaint()
		b.bounds.Point = pt
		b.Repaint()
	}
}

// Size implements the Widget interface.
func (b *Block) Size() Size {
	return b.bounds.Size
}

// SetSize implements the Widget interface.
func (b *Block) SetSize(size Size) {
	if b.bounds.Size != size {
		b.Repaint()
		b.bounds.Size = size
		b.SetNeedLayout(true)
		event := &Event{Type: ResizeEvent, When: time.Now(), Target: b}
		event.Dispatch()
		b.Repaint()
	}
}

// WidgetAt implements the Widget interface.
func (b *Block) WidgetAt(pt Point) Widget {
	for _, child := range b.children {
		bounds := child.Bounds()
		if bounds.Contains(pt) {
			pt.Subtract(bounds.Point)
			return child.WidgetAt(pt)
		}
	}
	return b
}

// ToWindow implements the Widget interface.
func (b *Block) ToWindow(pt Point) Point {
	pt.Add(b.bounds.Point)
	parent := b.parent
	for parent != nil {
		pt.Add(parent.Bounds().Point)
		parent = parent.Parent()
	}
	return pt
}

// FromWindow implements the Widget interface.
func (b *Block) FromWindow(pt Point) Point {
	pt.Subtract(b.bounds.Point)
	parent := b.parent
	for parent != nil {
		pt.Subtract(parent.Bounds().Point)
		parent = parent.Parent()
	}
	return pt
}

// Background implements the Widget interface.
func (b *Block) Background() Color {
	return b.background
}

// SetBackground implements the Widget interface.
func (b *Block) SetBackground(color Color) {
	if color != b.background {
		b.background = color
		b.Repaint()
	}
}
<|endoftext|>


<commit_before>import (
        "strconv"
        "strings"
       )

type Scanner struct {
	x, y, units, step float64
	relative          bool
}

func (context *Scanner) execute(line string) []byte {
	out := make([]byte, 2)
	tokens := strings.Split(line, " ")
	switch tokens[0] {
	case "G0":
	case "G1":
		cordinates := handleArgs(tokens[1:], "X", "Y")
		flag := 0
		if cordinates[0] != "" {
			tmp, error := strconv.ParseFloat(cordinates[0], 64)
			if error != nil {
				panic("not a number")
			}
			tmp *= context.units * context.step
			if context.relative {
				context.x += tmp
			} else {
				context.x = tmp
			}
			//checks here
			out = append(out, uint8(context.x/256), uint8(context.x))
			flag |= 1
		}
		if cordinates[1] != "" {
			tmp, error := strconv.ParseFloat(cordinates[1], 64)
			if error != nil {
				panic("not a number")
			}
			tmp *= context.step
			if context.relative {
				context.y += tmp
			} else {
				context.y = tmp
			}
			//checks here
			out = append(out, uint8(context.y/256), uint8(context.y))
			flag |= 2
		}
		out[1] = byte(flag<<4 + 1) //0yx + 0001 <- command code
		out[0] = byte(len(out) - 1)
	case "G4":
	case "G20":
		context.setIn()
		return nil
	case "G21":
		context.setMM()
		return nil
	defualt:
		panic("Invalid command")
	}
	return nil
}

func (context *Scanner) setMM() {
	context.units = 1.0
}
func (context *Scanner) setIn() {
	context.units = 2.6
}

func handleArgs(tokens []string, types ...string) []string {
	out := make([]string, len(types))
	for _, token := range tokens {
		for i, prefix := range types {
			if strings.HasPrefix(token, prefix) {
				out[i] = strings.TrimPrefix(token, prefix)
			}
		}
	}
	return out
}
<commit_msg>gcode translation working + extendable<commit_after>package main

import (
	"fmt"
	"strconv"
	"strings"
)

type Scanner struct {
	x, y, units, xStep, yStep float64
	relative                  bool
}

var filter map[string]func(string) (byte, byte, bool)
var context Scanner

func setup() {
	filter = make(map[string]func(string) (byte, byte, bool))
	filter["X"] = func(str string) (byte, byte, bool) {
		tmp, error := strconv.ParseFloat(str, 64)
		if error != nil {
			panic("not a number")
		}
		tmp *= context.units * context.xStep
		if context.relative {
			context.x += tmp
		} else {
			context.x = tmp
		}
		return uint8(context.x / 256), uint8(context.x), true
	}
	filter["Y"] = func(str string) (byte, byte, bool) {
		tmp, error := strconv.ParseFloat(str, 64)
		if error != nil {
			panic("not a number")
		}
		tmp *= context.yStep
		if context.relative {
			context.y += tmp
		} else {
			context.y = tmp
		}
		return uint8(context.y / 256), uint8(context.y), true
	}
	filter["S"] = func(str string) (byte, byte, bool) {
		tmp, error := strconv.ParseInt(str, 10, 16)
		if error != nil {
			panic("not a number")
		}
		return uint8(tmp / 256), uint8(tmp), true
	}
	filter["M"] = filter["S"]

	context.xStep = 100
	context.yStep = 10
	context.units = 1
}
func main() {
	setup()

	fmt.Println(execute("G0 X1.4 Y1.2123")) //move to 1.4 1.2123
	fmt.Println(execute("G91"))             //set relative
	fmt.Println(execute("G0 X1.4"))         //move to 2.8 1.2123
}
func execute(line string) []byte {
	tokens := strings.Split(line, " ")
	switch tokens[0] {
	case "G0", "G1":
		return compile([]string{"Y", "X"}, 1, tokens, filter)
	case "G4":
		return compile([]string{"S", "M"}, 3, tokens, filter)
	case "G20":
		setScale(2.6)
	case "G21":
		setScale(1.0)
	case "G90":
		setRelative(false)
	case "G91":
		setRelative(true)
	default:
		panic("Invalid command")
	}
	return nil
}

func setScale(scale float64) {
	context.units = scale
}
func setRelative(relative bool) {
	context.relative = relative
}

func compile(flags []string, code byte, tokens []string, filter map[string]func(string) (byte, byte, bool)) []byte {
	args := Parse(tokens, flags)
	out := make([]byte, 2)

	for i, arg := range args {
		first, second, set := filter[i](arg)
		if set {
			out = append(out, first, second)
		}
		out[1] += 1 << uint(IndexOf(i, flags))
	}

	out[0] = byte(len(out) - 1)

	out[1] = out[1]<<4 + code

	return out
}
func Parse(tokens []string, flags []string) map[string]string {
	out := make(map[string]string)
	for _, token := range tokens {
		for _, flag := range flags {
			if strings.HasPrefix(token, flag) {
				out[flag] = strings.TrimPrefix(token, flag)
			}
		}
	}
	return out
}
func IndexOf(elem string, array []string) int {
	for i, el := range array {
		if el == elem {
			return i
		}
	}
	return -1
}
<|endoftext|>


<commit_before>// Copyright 2015-2016, Cyrill @ Schumacher.fm and the CoreStore contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

/*
Package storage provides everything from MySQL, Redis, BoltDB, file, etc functions.

TODO(cs): github.com/youtube/vitess/go/mysql we should be compatible with this raw MYSQL driver.
TODO: use https://github.com/rubenv/sql-migrate for database migrations
	https://github.com/rubenv/sql-migrate is better because the API for using it within your code (FOSDEM2016).
*/
package storage
<commit_msg>storage: Update doc to add some ideas<commit_after>// Copyright 2015-2016, Cyrill @ Schumacher.fm and the CoreStore contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

/*
Package storage provides everything from MySQL, Redis, BoltDB, file, etc functions.

DB Schema Go Generation

bitbucket.org/jatone/genieql
sql query and code generation program. its purpose is to generate as much of the
boilerplate code for interacting with database/sql as possible. without putting
any runtime dependencies into your codebase. it only supports postgresql
currently. adding additional support is very straight forward, just implement
the Dialect interface. see the postgresql implementation as the example.

github.com/go-reform/reform
A better ORM for Go, based on non-empty interfaces and code generation.
https://gopkg.in/reform.v1

DB Migrations

github.com/rubenv/sql-migrate 500 Stars
SQL Schema migration tool for Go. Based on gorp and goose.
Is better because the API for using it within your code (FOSDEM2016).

https://bitbucket.org/liamstask/goose
goose is a database migration tool. You can manage your database's evolution by
creating incremental SQL or Go scripts.
Cons: https://www.reddit.com/r/golang/comments/2dlbz5/database_migration_handling_in_go/

github.com/mattes/migrate 744 Stars
A migration helper written in Go. Use it in your existing Golang code or run
commands via the CLI.

*/
package storage
<|endoftext|>


<commit_before>// Copyright 2016 The NorthShore Authors All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package store

import (
	"encoding/json"
	"errors"
	"strings"

	log "github.com/Sirupsen/logrus"

	"github.com/boltdb/bolt"
	"github.com/spf13/viper"
)

func openDBBucket(bucket []byte) *bolt.DB {
	path := viper.GetString("BoltDBPath")
	log.Debugln("Open #DB -> ", path)
	db, err := bolt.Open(path, 0600, nil)
	if err != nil {
		log.Fatal(err)
	}

	if err = db.Update(func(tx *bolt.Tx) error {
		_, err = tx.CreateBucketIfNotExists(bucket)
		if err != nil {
			return err
		}
		return nil
	}); err != nil {
		log.Fatal(err)
	}

	return db
}

// Delete deletes key from boltdb Bucket
func Delete(bucket []byte, key []byte) error {
	db := openDBBucket(bucket)
	defer db.Close()

	if err := db.Update(func(tx *bolt.Tx) error {
		return tx.Bucket(bucket).Delete(key)
	}); err != nil {
		log.Errorln("#DB,#Delete", err)
		return err
	}
	return nil
}

// Load loads item from boltdb Bucket
func Load(bucket []byte, key []byte, v interface{}) error {
	db := openDBBucket(bucket)
	defer db.Close()

	log.Debugln("#DB", "Load data from DB")
	err := db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket(bucket)
		buf := b.Get(key)
		if buf == nil {
			log.Debugln("#DB,#Load,#Nil")
			return errors.New("Key does not exist or key is a nested bucket")
		}

		if err := json.Unmarshal(buf, &v); err != nil {
			log.Errorln("#DB,#Load", err)
			return err
		}
		return nil
	})
	return err
}

// LoadBucket loads all items from boltdb Bucket
func LoadBucket(bucket []byte, buf *[]interface{}) error {
	db := openDBBucket(bucket)
	defer db.Close()

	log.Debugln("#DB", "Load bucket from DB")
	err := db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket(bucket)
		b.ForEach(func(k, v []byte) error {
			var item interface{}
			if err := json.Unmarshal(v, &item); err != nil {
				log.Errorln("#DB,#LoadBucket", err)
				return err
			}
			*buf = append(*buf, item)
			return nil
		})
		return nil
	})
	return err
}

// LoadBucketAsSlice loads all items from boltdb Bucket
func LoadBucketAsSlice(bucket []byte, v interface{}) error {
	db := openDBBucket(bucket)
	defer db.Close()

	err := db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket(bucket)

		bLenght := 0
		b.ForEach(func(_, _ []byte) error {
			bLenght++
			return nil
		})

		ss := make([]string, bLenght)
		idx := 0
		b.ForEach(func(_, v []byte) error {
			ss[idx] = string(v)
			idx++
			return nil
		})

		s := "[" + strings.Join(ss, ", ") + "]"

		if err := json.Unmarshal([]byte(s), v); err != nil {
			log.Errorln("#DB,#LoadBucketAsSlice", err)
			return err
		}

		return nil
	})
	return err
}

// Save stores item in boltdb Bucket as JSON
func Save(bucket []byte, key []byte, v interface{}) error {
	db := openDBBucket(bucket)
	defer db.Close()

	log.Debugln("#DB", "Save data to DB")
	err := db.Update(func(tx *bolt.Tx) error {
		b := tx.Bucket(bucket)
		vEncoded, err := json.Marshal(v)
		log.Debugln("#DB,#Save,#Encoded", string(vEncoded))
		if err != nil {
			log.Errorln("#DB,#Store", err)
			return err
		}
		if err := b.Put(key, vEncoded); err != nil {
			log.Errorln("#DB,#Store", err)
			return err
		}
		return nil
	})
	return err
}
<commit_msg>[store] Get bucket length from Stats<commit_after>// Copyright 2016 The NorthShore Authors All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package store

import (
	"encoding/json"
	"errors"
	"strings"

	log "github.com/Sirupsen/logrus"

	"github.com/boltdb/bolt"
	"github.com/spf13/viper"
)

func openDBBucket(bucket []byte) *bolt.DB {
	path := viper.GetString("BoltDBPath")
	log.Debugln("Open #DB -> ", path)
	db, err := bolt.Open(path, 0600, nil)
	if err != nil {
		log.Fatal(err)
	}

	if err = db.Update(func(tx *bolt.Tx) error {
		_, err = tx.CreateBucketIfNotExists(bucket)
		if err != nil {
			return err
		}
		return nil
	}); err != nil {
		log.Fatal(err)
	}

	return db
}

// Delete deletes key from boltdb Bucket
func Delete(bucket []byte, key []byte) error {
	db := openDBBucket(bucket)
	defer db.Close()

	if err := db.Update(func(tx *bolt.Tx) error {
		return tx.Bucket(bucket).Delete(key)
	}); err != nil {
		log.Errorln("#DB,#Delete", err)
		return err
	}
	return nil
}

// Load loads item from boltdb Bucket
func Load(bucket []byte, key []byte, v interface{}) error {
	db := openDBBucket(bucket)
	defer db.Close()

	log.Debugln("#DB", "Load data from DB")
	err := db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket(bucket)
		buf := b.Get(key)
		if buf == nil {
			log.Debugln("#DB,#Load,#Nil")
			return errors.New("Key does not exist or key is a nested bucket")
		}

		if err := json.Unmarshal(buf, &v); err != nil {
			log.Errorln("#DB,#Load", err)
			return err
		}
		return nil
	})
	return err
}

// LoadBucket loads all items from boltdb Bucket
func LoadBucket(bucket []byte, buf *[]interface{}) error {
	db := openDBBucket(bucket)
	defer db.Close()

	log.Debugln("#DB", "Load bucket from DB")
	err := db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket(bucket)
		b.ForEach(func(k, v []byte) error {
			var item interface{}
			if err := json.Unmarshal(v, &item); err != nil {
				log.Errorln("#DB,#LoadBucket", err)
				return err
			}
			*buf = append(*buf, item)
			return nil
		})
		return nil
	})
	return err
}

// LoadBucketAsSlice loads all items from boltdb Bucket
func LoadBucketAsSlice(bucket []byte, v interface{}) error {
	db := openDBBucket(bucket)
	defer db.Close()

	err := db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket(bucket)

		ss := make([]string, b.Stats().KeyN)
		idx := 0
		b.ForEach(func(_, v []byte) error {
			ss[idx] = string(v)
			idx++
			return nil
		})

		s := "[" + strings.Join(ss, ", ") + "]"

		if err := json.Unmarshal([]byte(s), v); err != nil {
			log.Errorln("#DB,#LoadBucketAsSlice", err)
			return err
		}

		return nil
	})
	return err
}

// Save stores item in boltdb Bucket as JSON
func Save(bucket []byte, key []byte, v interface{}) error {
	db := openDBBucket(bucket)
	defer db.Close()

	log.Debugln("#DB", "Save data to DB")
	err := db.Update(func(tx *bolt.Tx) error {
		b := tx.Bucket(bucket)
		vEncoded, err := json.Marshal(v)
		log.Debugln("#DB,#Save,#Encoded", string(vEncoded))
		if err != nil {
			log.Errorln("#DB,#Store", err)
			return err
		}
		if err := b.Put(key, vEncoded); err != nil {
			log.Errorln("#DB,#Store", err)
			return err
		}
		return nil
	})
	return err
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"github.com/kurrik/oauth1a"
	"github.com/kurrik/twittergo"
	"github.com/kurrik/json"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"strings"
	"sync"
	"time"
)

func LoadCredentials() (client *twittergo.Client, err error) {
	credentials, err := ioutil.ReadFile("CREDENTIALS")
	if err != nil {
		return
	}
	lines := strings.Split(string(credentials), "\n")
	config := &oauth1a.ClientConfig{
		ConsumerKey:    lines[0],
		ConsumerSecret: lines[1],
	}
	user := oauth1a.NewAuthorizedConfig(lines[2], lines[3])
	client = twittergo.NewClient(config, user)
	return
}

type Args struct {
	Track string
}

func parseArgs() *Args {
	a := &Args{}
	flag.StringVar(&a.Track, "track", "Data Science,Big Data", "Keyword to look up")
	flag.Parse()
	return a
}

type streamConn struct {
	client   *http.Client
	resp     *http.Response
	url      *url.URL
	stale    bool
	closed   bool
	mu       sync.Mutex
	// wait time before trying to reconnect, this will be
	// exponentially moved up until reaching maxWait, when
	// it will exit
	wait    int
	maxWait int
	connect func() (*http.Response, error)
}

func NewStreamConn(max int) streamConn {
	return streamConn{wait: 1, maxWait: max}
}

//type StreamHandler func([]byte)

func (conn *streamConn) Close() {
	// Just mark the connection as stale, and let the connect() handler close after a read
	conn.mu.Lock()
	defer conn.mu.Unlock()
	conn.stale = true
	conn.closed = true
	if conn.resp != nil {
		conn.resp.Body.Close()
	}
}

func (conn *streamConn) isStale() bool {
	conn.mu.Lock()
	r := conn.stale
	conn.mu.Unlock()
	return r
}

func readStream(client *twittergo.Client, sc streamConn, path string, query url.Values,
				resp *twittergo.APIResponse, handler func([]byte), done chan bool) {

	var reader *bufio.Reader
	reader = bufio.NewReader(resp.Body)

	for {
		//we've been closed
		if sc.isStale() {
			sc.Close()
			fmt.Println("Connection closed, shutting down ")
			break
		}

		line, err := reader.ReadBytes('\n')

		if err != nil {
			if sc.isStale() {
				fmt.Println("conn stale, continue")
				continue
			}

			time.Sleep(time.Second * time.Duration(sc.wait))
			//try reconnecting, but exponentially back off until MaxWait is reached then exit?
			resp, err := Connect(client, path, query)
			if err != nil || resp == nil {
				fmt.Println(" Could not reconnect to source? sleeping and will retry ")
				if sc.wait < sc.maxWait {
					sc.wait = sc.wait * 2
				} else {
					fmt.Println("exiting, max wait reached")
					done <- true
					return
				}
				continue
			}
			if resp.StatusCode != 200 {
				fmt.Printf("resp.StatusCode = %d", resp.StatusCode)
				if sc.wait < sc.maxWait {
					sc.wait = sc.wait * 2
				}
				continue
			}

			reader = bufio.NewReader(resp.Body)
			continue
		} else if sc.wait != 1 {
			sc.wait = 1
		}
		line = bytes.TrimSpace(line)
		fmt.Println("Received a line ")

		if len(line) == 0 {
			continue
		}
		handler(line)
	}
}

func Connect(client *twittergo.Client, path string, query url.Values) (resp *twittergo.APIResponse, err error) {
	var (
		req 	*http.Request
	)

	url := fmt.Sprintf("%v?%v", path, query.Encode())
	req, err = http.NewRequest("GET", url, nil)
	if err != nil {
		err = fmt.Errorf("Could not parse request: %v\n", err)
		return
	}
	resp, err = client.SendRequest(req)
	if err != nil {
		err = fmt.Errorf("Could not send request: %v\n", err)
		return
	}

	fmt.Printf("resp.StatusCode=%d\n", resp.StatusCode)
	return
}

func filterStream(client *twittergo.Client, path string, query url.Values) (err error) {
	var (
		resp    *twittergo.APIResponse
	)

	sc := NewStreamConn(300)

	resp, err = Connect(client, path, query)

	done := make(chan bool)
	stream := make(chan []byte, 1000)
	go func() {
		for data := range stream {
			tweet := &twittergo.Tweet{}
			err := json.Unmarshal(data, tweet)
			if (err == nil) {
					fmt.Printf("ID:                   %v\n", tweet.Id())
					fmt.Printf("User:                   %v\n", tweet.User().ScreenName())
					fmt.Printf("Tweet:                %v\n", tweet.Text())

			}
		}
	}()

	readStream(client, sc, path, query, resp, func(line []byte) {
		stream <- line}, done)

	return
}

func main() {
	var (
		err    error
		args   *Args
		client *twittergo.Client
	)
	args = parseArgs()
	if client, err = LoadCredentials(); err != nil {
		fmt.Printf("Could not parse CREDENTIALS file: %v\n", err)
		os.Exit(1)
	}
	fmt.Println(args.Track)
	query := url.Values{}
	query.Set("track", args.Track)

	fmt.Println("Printing everything about data science:")
	fmt.Printf("=========================================================\n")
	if err = filterStream(client, "/1.1/statuses/filter.json", query); err != nil {
		fmt.Println("Error: %v\n", err)
	}
	fmt.Printf("\n\n")

}
<commit_msg>Adds a streaming example by codingneo.<commit_after>package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"github.com/kurrik/json"
	"github.com/kurrik/oauth1a"
	"github.com/kurrik/twittergo"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"strings"
	"sync"
	"time"
)

func LoadCredentials() (client *twittergo.Client, err error) {
	credentials, err := ioutil.ReadFile("CREDENTIALS")
	if err != nil {
		return
	}
	lines := strings.Split(string(credentials), "\n")
	config := &oauth1a.ClientConfig{
		ConsumerKey:    lines[0],
		ConsumerSecret: lines[1],
	}
	user := oauth1a.NewAuthorizedConfig(lines[2], lines[3])
	client = twittergo.NewClient(config, user)
	return
}

type Args struct {
	Track string
}

func parseArgs() *Args {
	a := &Args{}
	flag.StringVar(&a.Track, "track", "Data Science,Big Data", "Keyword to look up")
	flag.Parse()
	return a
}

type streamConn struct {
	client *http.Client
	resp   *http.Response
	url    *url.URL
	stale  bool
	closed bool
	mu     sync.Mutex
	// wait time before trying to reconnect, this will be
	// exponentially moved up until reaching maxWait, when
	// it will exit
	wait    int
	maxWait int
	connect func() (*http.Response, error)
}

func NewStreamConn(max int) streamConn {
	return streamConn{wait: 1, maxWait: max}
}

func (conn *streamConn) Close() {
	// Just mark the connection as stale, and let the connect() handler close after a read
	conn.mu.Lock()
	defer conn.mu.Unlock()
	conn.stale = true
	conn.closed = true
	if conn.resp != nil {
		conn.resp.Body.Close()
	}
}

func (conn *streamConn) isStale() bool {
	conn.mu.Lock()
	r := conn.stale
	conn.mu.Unlock()
	return r
}

func readStream(client *twittergo.Client, sc streamConn, path string, query url.Values,
	resp *twittergo.APIResponse, handler func([]byte), done chan bool) {

	var reader *bufio.Reader
	reader = bufio.NewReader(resp.Body)

	for {
		//we've been closed
		if sc.isStale() {
			sc.Close()
			fmt.Println("Connection closed, shutting down ")
			break
		}

		line, err := reader.ReadBytes('\n')

		if err != nil {
			if sc.isStale() {
				fmt.Println("conn stale, continue")
				continue
			}

			time.Sleep(time.Second * time.Duration(sc.wait))
			//try reconnecting, but exponentially back off until MaxWait is reached then exit?
			resp, err := Connect(client, path, query)
			if err != nil || resp == nil {
				fmt.Println(" Could not reconnect to source? sleeping and will retry ")
				if sc.wait < sc.maxWait {
					sc.wait = sc.wait * 2
				} else {
					fmt.Println("exiting, max wait reached")
					done <- true
					return
				}
				continue
			}
			if resp.StatusCode != 200 {
				fmt.Printf("resp.StatusCode = %d", resp.StatusCode)
				if sc.wait < sc.maxWait {
					sc.wait = sc.wait * 2
				}
				continue
			}

			reader = bufio.NewReader(resp.Body)
			continue
		} else if sc.wait != 1 {
			sc.wait = 1
		}
		line = bytes.TrimSpace(line)
		if len(line) == 0 {
			continue
		}
		handler(line)
	}
}

func Connect(client *twittergo.Client, path string, query url.Values) (resp *twittergo.APIResponse, err error) {
	var (
		req *http.Request
	)
	url := fmt.Sprintf("https://stream.twitter.com%v?%v", path, query.Encode())
	req, err = http.NewRequest("GET", url, nil)
	if err != nil {
		err = fmt.Errorf("Could not parse request: %v\n", err)
		return
	}
	resp, err = client.SendRequest(req)
	if err != nil {
		err = fmt.Errorf("Could not send request: %v\n", err)
		return
	}
	return
}

func filterStream(client *twittergo.Client, path string, query url.Values) (err error) {
	var (
		resp *twittergo.APIResponse
	)

	sc := NewStreamConn(300)
	resp, err = Connect(client, path, query)

	done := make(chan bool)
	stream := make(chan []byte, 1000)
	go func() {
		for data := range stream {
			tweet := &twittergo.Tweet{}
			err := json.Unmarshal(data, tweet)
			if err == nil {
				fmt.Printf("ID:     %v\n", tweet.Id())
				fmt.Printf("User:   %v\n", tweet.User().ScreenName())
				fmt.Printf("Tweet:  %v\n", tweet.Text())

			}
		}
	}()

	readStream(client, sc, path, query, resp, func(line []byte) {
		stream <- line
	}, done)

	return
}

func main() {
	var (
		err    error
		args   *Args
		client *twittergo.Client
	)
	args = parseArgs()
	if client, err = LoadCredentials(); err != nil {
		fmt.Printf("Could not parse CREDENTIALS file: %v\n", err)
		os.Exit(1)
	}
	fmt.Println(args.Track)
	query := url.Values{}
	query.Set("track", args.Track)

	fmt.Println("Printing everything about data science:")
	fmt.Printf("=========================================================\n")
	if err = filterStream(client, "/1.1/statuses/filter.json", query); err != nil {
		fmt.Println("Error: %v\n", err)
	}
	fmt.Printf("\n\n")
}
<|endoftext|>