<commit_before>//
// Copyright 2015, Sander van Harmelen
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package gitlab

import (
	"fmt"
	"net/url"
)

// LabelsService handles communication with the label related methods
// of the GitLab API.
//
// GitLab API docs: http://doc.gitlab.com/ce/api/labels.html
type LabelsService struct {
	client *Client
}

// Label represents a GitLab label.
//
// GitLab API docs: http://doc.gitlab.com/ce/api/labels.html
type Label struct {
	Name  string `json:"name"`
	Color string `json:"color"`
}

func (l Label) String() string {
	return Stringify(l)
}

// ListLabels gets all labels for given project.
//
// GitLab API docs: http://doc.gitlab.com/ce/api/labels.html#list-labels
func (s *LabelsService) ListLabels(pid interface{}) ([]*Label, *Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("projects/%s/labels", url.QueryEscape(project))

	req, err := s.client.NewRequest("GET", u, nil)
	if err != nil {
		return nil, nil, err
	}

	var l []*Label
	resp, err := s.client.Do(req, &l)
	if err != nil {
		return nil, resp, err
	}

	return l, resp, err
}

// CreateLabelOptions represents the available CreateLabel() options.
//
// GitLab API docs: http://doc.gitlab.com/ce/api/labels.html#create-a-new-label
type CreateLabelOptions struct {
	Name  string `url:"name,omitempty" json:"name,omitempty"`
	Color string `url:"color,omitempty" json:"color,omitempty"`
}

// CreateLabel creates a new label for given repository with given name and
// color.
//
// GitLab API docs: http://doc.gitlab.com/ce/api/labels.html#create-a-new-label
func (s *LabelsService) CreateLabel(
	pid interface{},
	opt *CreateLabelOptions) (*Label, *Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("projects/%s/labels", url.QueryEscape(project))

	req, err := s.client.NewRequest("POST", u, opt)
	if err != nil {
		return nil, nil, err
	}

	l := new(Label)
	resp, err := s.client.Do(req, l)
	if err != nil {
		return nil, resp, err
	}

	return l, resp, err
}

// DeleteLabelOptions represents the available DeleteLabel() options.
//
// GitLab API docs: http://doc.gitlab.com/ce/api/labels.html#delete-a-label
type DeleteLabelOptions struct {
	Name string `url:"name,omitempty" json:"name,omitempty"`
}

// DeleteLabel deletes a label given by its name.
//
// GitLab API docs: http://doc.gitlab.com/ce/api/labels.html#delete-a-label
func (s *LabelsService) DeleteLabel(pid interface{}, opt *DeleteLabelOptions) (*Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, err
	}
	u := fmt.Sprintf("projects/%s/labels", url.QueryEscape(project))

	req, err := s.client.NewRequest("DELETE", u, opt)
	if err != nil {
		return nil, err
	}

	resp, err := s.client.Do(req, nil)
	if err != nil {
		return resp, err
	}

	return resp, err
}

// UpdateLabelOptions represents the available UpdateLabel() options.
//
// GitLab API docs: http://doc.gitlab.com/ce/api/labels.html#delete-a-label
type UpdateLabelOptions struct {
	Name    string `url:"name,omitempty" json:"name,omitempty"`
	NewName string `url:"new_name,omitempty" json:"new_name,omitempty"`
	Color   string `url:"color,omitempty" json:"color,omitempty"`
}

// UpdateLabel updates an existing label with new name or now color. At least
// one parameter is required, to update the label.
//
// GitLab API docs: http://doc.gitlab.com/ce/api/labels.html#edit-an-existing-label
func (s *LabelsService) UpdateLabel(
	pid interface{},
	opt *UpdateLabelOptions) (*Label, *Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("projects/%s/labels", url.QueryEscape(project))

	req, err := s.client.NewRequest("PUT", u, opt)
	if err != nil {
		return nil, nil, err
	}

	l := new(Label)
	resp, err := s.client.Do(req, l)
	if err != nil {
		return nil, resp, err
	}

	return l, resp, err
}
<commit_msg>additional fields in Gitlab<commit_after>//
// Copyright 2015, Sander van Harmelen
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package gitlab

import (
	"fmt"
	"net/url"
)

// LabelsService handles communication with the label related methods
// of the GitLab API.
//
// GitLab API docs: http://doc.gitlab.com/ce/api/labels.html
type LabelsService struct {
	client *Client
}

// Label represents a GitLab label.
//
// GitLab API docs: http://doc.gitlab.com/ce/api/labels.html
type Label struct {
	Name                   string `json:"name"`
	Color                  string `json:"color"`
	Description            string `json:"description"`
	OpenIssuesCount        int    `json:"open_issues_count"`
	ClosedIssuesCount      int    `json:"closed_issues_count"`
	OpenMergeRequestsCount int    `json:"open_merge_requests_count"`
}

func (l Label) String() string {
	return Stringify(l)
}

// ListLabels gets all labels for given project.
//
// GitLab API docs: http://doc.gitlab.com/ce/api/labels.html#list-labels
func (s *LabelsService) ListLabels(pid interface{}) ([]*Label, *Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("projects/%s/labels", url.QueryEscape(project))

	req, err := s.client.NewRequest("GET", u, nil)
	if err != nil {
		return nil, nil, err
	}

	var l []*Label
	resp, err := s.client.Do(req, &l)
	if err != nil {
		return nil, resp, err
	}

	return l, resp, err
}

// CreateLabelOptions represents the available CreateLabel() options.
//
// GitLab API docs: http://doc.gitlab.com/ce/api/labels.html#create-a-new-label
type CreateLabelOptions struct {
	Name        string `url:"name,omitempty" json:"name,omitempty"`
	Color       string `url:"color,omitempty" json:"color,omitempty"`
	Description string `url:"description,omitempty" json:"description,omitempty"`
}

// CreateLabel creates a new label for given repository with given name and
// color.
//
// GitLab API docs: http://doc.gitlab.com/ce/api/labels.html#create-a-new-label
func (s *LabelsService) CreateLabel(
	pid interface{},
	opt *CreateLabelOptions) (*Label, *Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("projects/%s/labels", url.QueryEscape(project))

	req, err := s.client.NewRequest("POST", u, opt)
	if err != nil {
		return nil, nil, err
	}

	l := new(Label)
	resp, err := s.client.Do(req, l)
	if err != nil {
		return nil, resp, err
	}

	return l, resp, err
}

// DeleteLabelOptions represents the available DeleteLabel() options.
//
// GitLab API docs: http://doc.gitlab.com/ce/api/labels.html#delete-a-label
type DeleteLabelOptions struct {
	Name string `url:"name,omitempty" json:"name,omitempty"`
}

// DeleteLabel deletes a label given by its name.
//
// GitLab API docs: http://doc.gitlab.com/ce/api/labels.html#delete-a-label
func (s *LabelsService) DeleteLabel(pid interface{}, opt *DeleteLabelOptions) (*Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, err
	}
	u := fmt.Sprintf("projects/%s/labels", url.QueryEscape(project))

	req, err := s.client.NewRequest("DELETE", u, opt)
	if err != nil {
		return nil, err
	}

	resp, err := s.client.Do(req, nil)
	if err != nil {
		return resp, err
	}

	return resp, err
}

// UpdateLabelOptions represents the available UpdateLabel() options.
//
// GitLab API docs: http://doc.gitlab.com/ce/api/labels.html#delete-a-label
type UpdateLabelOptions struct {
	Name        string `url:"name,omitempty" json:"name,omitempty"`
	NewName     string `url:"new_name,omitempty" json:"new_name,omitempty"`
	Color       string `url:"color,omitempty" json:"color,omitempty"`
	Description string `url:"description,omitempty" json:"description,omitempty"`
}

// UpdateLabel updates an existing label with new name or now color. At least
// one parameter is required, to update the label.
//
// GitLab API docs: http://doc.gitlab.com/ce/api/labels.html#edit-an-existing-label
func (s *LabelsService) UpdateLabel(
	pid interface{},
	opt *UpdateLabelOptions) (*Label, *Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("projects/%s/labels", url.QueryEscape(project))

	req, err := s.client.NewRequest("PUT", u, opt)
	if err != nil {
		return nil, nil, err
	}

	l := new(Label)
	resp, err := s.client.Do(req, l)
	if err != nil {
		return nil, resp, err
	}

	return l, resp, err
}
<|endoftext|>


<commit_before>package bench

import (
	"fmt"
	"net/http"
	"io/ioutil"
	"path/filepath"
	"bytes"
	"time"
	"math/rand"

	"github.com/urfave/cli"

	"github.com/open-lambda/open-lambda/ol/common"
)

type Call struct {
        name string
}

func task(task int, reqQ chan Call, errQ chan error) {
        for {
                call, ok := <- reqQ
                if !ok {
                        errQ <- nil
                        break
                }

                url := fmt.Sprintf("http://localhost:%s/run/%s", common.Conf.Worker_port, call.name)
                resp, err := http.Post(url, "text/json", bytes.NewBuffer([]byte("null")))
                if err != nil {
                        errQ <- fmt.Errorf("failed req to %s: %v", url, err)
                        continue
                }

                body, err := ioutil.ReadAll(resp.Body)
                resp.Body.Close()

                if err != nil {
                        errQ <- fmt.Errorf("failed to %s, could not read body: %v", url, err)
                        continue
                }

                if resp.StatusCode != http.StatusOK {
                        errQ <- fmt.Errorf("failed req to %s: status %d, text '%s'", url, resp.StatusCode, string(body))
                        continue
                }

		errQ <- nil
        }
}

func run_benchmark(ctx *cli.Context, name string, seconds float64, tasks int, functions int, func_template string) error {
	olPath, err := common.GetOlPath(ctx)
	if err != nil {
		return err
	}
	configPath := filepath.Join(olPath, "config.json")
	if err := common.LoadConf(configPath); err != nil {
		return err
	}

	// launch request threads
        reqQ := make(chan Call, tasks)
        errQ := make(chan error, tasks)
        for i := 0; i < tasks; i++ {
                go task(i, reqQ, errQ)
        }

	// warmup: call lambda each once
	// TODO: make warming optional
	fmt.Printf("warming up (calling each lambda once sequentially)\n")
        for i := 0; i<functions; i++ {
		name := fmt.Sprintf(func_template, i)
		fmt.Printf("warmup %s (%d/%d)\n", name, i, functions)
		reqQ <- Call{name: name}
		if err := <- errQ; err != nil {
			return err
		}
        }

	// issue requests for specified number of seconds
	fmt.Printf("start benchmark\n")
	errors := 0
	successes := 0
	waiting := 0

	start := time.Now()
        for time.Since(start).Seconds() < seconds {
                select {
                case reqQ <- Call{name: fmt.Sprintf(func_template, rand.Intn(functions))}:
			waiting += 1
                case err := <- errQ:
			if err != nil {
				errors += 1
				fmt.Printf("%s\n", err.Error())
			} else {
				successes += 1
			}
			waiting -= 1
                }
        }
	seconds = time.Since(start).Seconds()

	// cleanup request threads
	fmt.Printf("cleanup\n")
        close(reqQ)
	waiting += tasks // each needs to send one last nil to indicate it is done
        for waiting > 0 {
		if err := <- errQ; err != nil {
			errors += 1
			fmt.Printf("%s\n", err.Error())
		}
		waiting -= 1
	}

	if errors > (errors + successes) / 100 {
		panic(fmt.Sprintf(">1% of requests failed (%d/%d)", errors, errors + successes))
	}

        fmt.Printf("{\"benchmark\": \"%s\",\"seconds\": %.3f, \"successes\": %d, \"errors\": %d, \"ops/s\": %.3f}\n",
		name, seconds, successes, errors, float64(successes)/seconds)

	return nil
}

func create_lambdas(ctx *cli.Context) error {
	olPath, err := common.GetOlPath(ctx)
	if err != nil {
		return err
	}

	configPath := filepath.Join(olPath, "config.json")

	if err := common.LoadConf(configPath); err != nil {
		return err
	}

	for i := 0; i < 64*1024; i++ {
		// noop
		path := filepath.Join(common.Conf.Registry, fmt.Sprintf("bench-py-%d.py", i))
		code := fmt.Sprintf("def f(event):\n\treturn %d", i)

		fmt.Printf("%s\n", path)
		if err := ioutil.WriteFile(path, []byte(code), 0644); err != nil {
			return err
		}

		// simple pandas operation (correlation between two columns in 1000x10 DataFrame)
		path = filepath.Join(common.Conf.Registry, fmt.Sprintf("bench-pd-%d.py", i))
		code = `# ol-install: numpy,pandas
import numpy as np
import pandas as pd

df = None

def f(event):
    global df
    if df is None:
        df = pd.DataFrame(np.random.random((1000,10)))
    col0 = np.random.randint(len(df.columns))
    col1 = np.random.randint(len(df.columns))
    return df[col0].corr(df[col1])
`

		fmt.Printf("%s\n", path)
		if err := ioutil.WriteFile(path, []byte(code), 0644); err != nil {
			return err
		}
	}

	return nil
}

func make_action(name string, seconds float64, tasks int, functions int, func_template string) (func (ctx *cli.Context) error) {
	return func (ctx *cli.Context) error {
		return run_benchmark(ctx, name, seconds, tasks, functions, func_template)
	}
}

func BenchCommands() []cli.Command {
	cmds := []cli.Command{
		{
                        Name:  "init",
                        Usage: "creates lambdas for benchmarking",
			UsageText: "ol bench init [--path=NAME]",
                        Action: create_lambdas,
			// TODO: add param to decide how many to create
		},
	}

	seconds := 60.0 // TODO: add param

	for _, kind := range []string{"py", "pd"} {
		for _, functions := range []int{64, 1024, 64*1024} {
			for _, tasks := range []int{1, 32} {
				var parseq string
				var par_usage string
				var usage string
				amt := fmt.Sprintf("%d", functions)

				if tasks == 1 {
					parseq = "seq"
					par_usage = "sequentially"
				} else {
					parseq = "par"
					par_usage = fmt.Sprintf("in parallel (%d clients)", tasks)
				}
				if functions >= 1024 {
					amt = fmt.Sprintf("%dk", functions / 1024)
				}
				if kind == "py" {
					usage = fmt.Sprintf(("invoke noop Python lambdas %s for %d seconds, " +
						"randomly+uniformaly selecting 1 of %d lambdas for each request"), par_usage, int(seconds), functions)
				} else if kind == "pd" {
					usage = fmt.Sprintf(("invoke Pandas lambdas that do correlations %s for %d seconds, " +
						"randomly+uniformaly selecting 1 of %d lambdas for each request"), par_usage, int(seconds), functions)
				}

				name := fmt.Sprintf("%s%s-%s", kind, amt, parseq)
				action := make_action(name, seconds, tasks, functions, "bench-"+kind+"-%d")
				cmd := cli.Command{
					Name:  name,
					Usage: usage,
					UsageText: fmt.Sprintf("ol bench init [--path=NAME]"),
					Action: action,
					Flags: []cli.Flag{
						cli.StringFlag{
							Name:  "path, p",
							Usage: "Path location for OL environment",
						},
					},
				}
				cmds = append(cmds, cmd)
			}
		}
	}

	// TODO: add command that runs them all

	return cmds
}
<commit_msg>add seconds option to benchmark<commit_after>package bench

import (
	"fmt"
	"net/http"
	"io/ioutil"
	"path/filepath"
	"bytes"
	"time"
	"math/rand"

	"github.com/urfave/cli"

	"github.com/open-lambda/open-lambda/ol/common"
)

type Call struct {
        name string
}

func task(task int, reqQ chan Call, errQ chan error) {
        for {
                call, ok := <- reqQ
                if !ok {
                        errQ <- nil
                        break
                }

                url := fmt.Sprintf("http://localhost:%s/run/%s", common.Conf.Worker_port, call.name)
                resp, err := http.Post(url, "text/json", bytes.NewBuffer([]byte("null")))
                if err != nil {
                        errQ <- fmt.Errorf("failed req to %s: %v", url, err)
                        continue
                }

                body, err := ioutil.ReadAll(resp.Body)
                resp.Body.Close()

                if err != nil {
                        errQ <- fmt.Errorf("failed to %s, could not read body: %v", url, err)
                        continue
                }

                if resp.StatusCode != http.StatusOK {
                        errQ <- fmt.Errorf("failed req to %s: status %d, text '%s'", url, resp.StatusCode, string(body))
                        continue
                }

		errQ <- nil
        }
}

func run_benchmark(ctx *cli.Context, name string, tasks int, functions int, func_template string) error {
	// config
	olPath, err := common.GetOlPath(ctx)
	if err != nil {
		return err
	}
	configPath := filepath.Join(olPath, "config.json")
	if err := common.LoadConf(configPath); err != nil {
		return err
	}

	seconds := ctx.Float64("seconds")
	if seconds == 0 {
		seconds = 60.0
	}

	// launch request threads
        reqQ := make(chan Call, tasks)
        errQ := make(chan error, tasks)
        for i := 0; i < tasks; i++ {
                go task(i, reqQ, errQ)
        }

	// warmup: call lambda each once
	// TODO: make warming optional
	fmt.Printf("warming up (calling each lambda once sequentially)\n")
        for i := 0; i<functions; i++ {
		name := fmt.Sprintf(func_template, i)
		fmt.Printf("warmup %s (%d/%d)\n", name, i, functions)
		reqQ <- Call{name: name}
		if err := <- errQ; err != nil {
			return err
		}
        }

	// issue requests for specified number of seconds
	fmt.Printf("start benchmark (%.1f seconds)\n", seconds)
	errors := 0
	successes := 0
	waiting := 0

	start := time.Now()
        for time.Since(start).Seconds() < seconds {
                select {
                case reqQ <- Call{name: fmt.Sprintf(func_template, rand.Intn(functions))}:
			waiting += 1
                case err := <- errQ:
			if err != nil {
				errors += 1
				fmt.Printf("%s\n", err.Error())
			} else {
				successes += 1
			}
			waiting -= 1
                }
        }
	seconds = time.Since(start).Seconds()

	// cleanup request threads
	fmt.Printf("cleanup\n")
        close(reqQ)
	waiting += tasks // each needs to send one last nil to indicate it is done
        for waiting > 0 {
		if err := <- errQ; err != nil {
			errors += 1
			fmt.Printf("%s\n", err.Error())
		}
		waiting -= 1
	}

	if errors > (errors + successes) / 100 {
		panic(fmt.Sprintf(">1% of requests failed (%d/%d)", errors, errors + successes))
	}

        fmt.Printf("{\"benchmark\": \"%s\",\"seconds\": %.3f, \"successes\": %d, \"errors\": %d, \"ops/s\": %.3f}\n",
		name, seconds, successes, errors, float64(successes)/seconds)

	return nil
}

func create_lambdas(ctx *cli.Context) error {
	olPath, err := common.GetOlPath(ctx)
	if err != nil {
		return err
	}

	configPath := filepath.Join(olPath, "config.json")

	if err := common.LoadConf(configPath); err != nil {
		return err
	}

	for i := 0; i < 64*1024; i++ {
		// noop
		path := filepath.Join(common.Conf.Registry, fmt.Sprintf("bench-py-%d.py", i))
		code := fmt.Sprintf("def f(event):\n\treturn %d", i)

		fmt.Printf("%s\n", path)
		if err := ioutil.WriteFile(path, []byte(code), 0644); err != nil {
			return err
		}

		// simple pandas operation (correlation between two columns in 1000x10 DataFrame)
		path = filepath.Join(common.Conf.Registry, fmt.Sprintf("bench-pd-%d.py", i))
		code = `# ol-install: numpy,pandas
import numpy as np
import pandas as pd

df = None

def f(event):
    global df
    if df is None:
        df = pd.DataFrame(np.random.random((1000,10)))
    col0 = np.random.randint(len(df.columns))
    col1 = np.random.randint(len(df.columns))
    return df[col0].corr(df[col1])
`

		fmt.Printf("%s\n", path)
		if err := ioutil.WriteFile(path, []byte(code), 0644); err != nil {
			return err
		}
	}

	return nil
}

func make_action(name string, tasks int, functions int, func_template string) (func (ctx *cli.Context) error) {
	return func (ctx *cli.Context) error {
		return run_benchmark(ctx, name, tasks, functions, func_template)
	}
}

func BenchCommands() []cli.Command {
	cmds := []cli.Command{
		{
                        Name:  "init",
                        Usage: "creates lambdas for benchmarking",
			UsageText: "ol bench init [--path=NAME]",
                        Action: create_lambdas,
			// TODO: add param to decide how many to create
		},
	}

	for _, kind := range []string{"py", "pd"} {
		for _, functions := range []int{64, 1024, 64*1024} {
			for _, tasks := range []int{1, 32} {
				var parseq string
				var par_usage string
				var usage string
				amt := fmt.Sprintf("%d", functions)

				if tasks == 1 {
					parseq = "seq"
					par_usage = "sequentially"
				} else {
					parseq = "par"
					par_usage = fmt.Sprintf("in parallel (%d clients)", tasks)
				}
				if functions >= 1024 {
					amt = fmt.Sprintf("%dk", functions / 1024)
				}
				if kind == "py" {
					usage = fmt.Sprintf(("invoke noop Python lambdas %s for S seconds (default 60), " +
						"randomly+uniformaly selecting 1 of %d lambdas for each request"), par_usage, functions)
				} else if kind == "pd" {
					usage = fmt.Sprintf(("invoke Pandas lambdas that do correlations %s for S seconds (default 60), " +
						"randomly+uniformaly selecting 1 of %d lambdas for each request"), par_usage, functions)
				}

				name := fmt.Sprintf("%s%s-%s", kind, amt, parseq)
				action := make_action(name, tasks, functions, "bench-"+kind+"-%d")
				cmd := cli.Command{
					Name:  name,
					Usage: usage,
					UsageText: fmt.Sprintf("ol bench %s [--path=NAME] [--seconds=SECONDS]", name),
					Action: action,
					Flags: []cli.Flag{
						cli.StringFlag{
							Name:  "path, p",
							Usage: "Path location for OL environment",
						},
						cli.Float64Flag{
							Name:  "seconds, s",
							Usage: "Seconds to run (after warmup)",
						},
					},
				}
				cmds = append(cmds, cmd)
			}
		}
	}

	// TODO: add command that runs them all

	return cmds
}
<|endoftext|>


<commit_before>package main

// The Actor struct holds info about an actor I.E the Lad or a Rock
type Actor struct {
	Type       int
	Y          int
	X          int
	Ch         byte
	Dir        Action
	DirRequest Action
}

// The Action constans define what the Actor currently is, or is requested to be, doing
type Action int

const (
	STOPPED Action = iota
	LEFT
	RIGHT
	UP
	DOWN
	FALLING
	JUMP
)

//
//
//
func MoveActor(a Actor, m MapData) Actor {

loopAgain: // If just started falling we need to retest all conditions

	// If stopped or going left or going right and requst to do something else, then try to do it
	if (a.Dir == STOPPED && a.DirRequest != STOPPED) ||
		(a.Dir == LEFT && a.DirRequest == RIGHT) ||
		(a.Dir == RIGHT && a.DirRequest == LEFT) {
		a.Dir = a.DirRequest
		a.DirRequest = STOPPED
	}

	// If at the edige of the playfield stop and exit
	if (a.Dir == LEFT && a.X == 0) ||
		(a.Dir == RIGHT && a.X >= 78) {
		a.Dir = STOPPED
	}

	// If at a ladder and want to go up
	if (a.DirRequest == UP && m.Field[a.Y][a.X] == 'H') ||
		(a.DirRequest == DOWN && m.Field[a.Y][a.X] == 'H') {
		a.Dir = a.DirRequest
		a.DirRequest = STOPPED
	}

	// If falling then continue doing that until not in free space anymore,
	// then continue the previous direction (if any)
	if a.Dir == FALLING {
		if m.Field[a.Y+1][a.X] == ' ' {
			a.Y++
			return a
		}
		a.Dir = a.DirRequest
	}

	// Climb up until ladder is no more
	if a.Dir == UP {
		if m.Field[a.Y-1][a.X] == 'H' {
			a.Y--
		} else {
			a.Dir = STOPPED
		}
	}

	// Climb down until ladder is no more
	if a.Dir == DOWN {
		if m.Field[a.Y+1][a.X] == 'H' {
			a.Y++
		} else {
			a.Dir = STOPPED
		}
	}

	if a.Dir == LEFT && (m.Field[a.Y][a.X] == ' ' || m.Field[a.Y][a.X] == 'H' || m.Field[a.Y][a.X] == '*') {
		// Stepped out into the void?, the start falling, but remember the previous direction
		if m.Field[a.Y+1][a.X] == ' ' {
			a.DirRequest = a.Dir
			a.Dir = FALLING
			goto loopAgain
		}
		a.X--
	}

	if a.Dir == RIGHT && (m.Field[a.Y][a.X] == ' ' || m.Field[a.Y][a.X] == 'H' || m.Field[a.Y][a.X] == '*') {
		// Stepped out into the void?, the start falling, but remember the previous direction
		if m.Field[a.Y+1][a.X] == ' ' {
			a.DirRequest = a.Dir
			a.Dir = FALLING
			goto loopAgain
		}
		a.X++
	}

	return a
}
<commit_msg>Support for jumps<commit_after>package main

//go:generate stringer -type=Action

// The Actor struct holds info about an actor I.E the Lad or a Rock
type Actor struct {
	Type       int
	Y          int
	X          int
	Ch         byte
	Dir        Action
	DirRequest Action
	JumpStep   int
}

type Action int

// The Action constans define what the Actor currently is, or is requested to be, doing
const (
	STOPPED Action = iota
	UP
	UPRIGHT
	RIGHT
	DOWNRIGHT
	DOWN
	DOWNLEFT
	LEFT
	UPLEFT
	FALLING
	JUMP // Generic jump set by keyhandler
	JUMPRIGHT
	JUMPUP
	JUMPLEFT
)

var jumpLeft = []Action{UPLEFT, UPLEFT, LEFT, LEFT, DOWNLEFT, DOWNLEFT}
var jumpRight = []Action{UPRIGHT, UPRIGHT, RIGHT, RIGHT, DOWNRIGHT, DOWNRIGHT}
var jumpUp = []Action{UP, UP, STOPPED, DOWN, DOWN}

var jumpPaths = map[Action][]Action{
	JUMPUP:    jumpUp,
	JUMPLEFT:  jumpLeft,
	JUMPRIGHT: jumpRight,
}

var dirs = map[Action]struct {
	x, y int
}{
	STOPPED:   {0, 0},
	UP:        {0, -1},
	UPRIGHT:   {1, -1},
	RIGHT:     {1, 0},
	DOWNRIGHT: {1, 1},
	DOWN:      {0, 1},
	DOWNLEFT:  {-1, 1},
	LEFT:      {-1, 0},
	UPLEFT:    {-1, -1},
	FALLING:   {0, 1},
	JUMP:      {0, 0},
	JUMPRIGHT: {0, 0},
	JUMPLEFT:  {0, 0},
	JUMPUP:    {0, 0},
}

//
// A moving jummp is UR/UR/R/R/DR/DR
// or                UL/UL/L/L/DL/DL
// A standing jump is U/U/-/D/D
//
// 	 ====================
//   ----234-----23------
//   ---1---5----14------
//   --0-----6---05------
//   ====================
//

//
//
//
func MoveActor(a Actor, m MapData) Actor {

loopAgain: // If just started falling we need to retest all conditions

	// If stopped or going left or going right and requst to do something else, then try to do it
	if (a.Dir == STOPPED && a.DirRequest == LEFT) ||
		(a.Dir == STOPPED && a.DirRequest == RIGHT) ||
		(a.Dir == LEFT && a.DirRequest == RIGHT) ||
		(a.Dir == RIGHT && a.DirRequest == LEFT) {
		a.Dir = a.DirRequest
		a.DirRequest = STOPPED

	}

	// Handle starting of jumps
	if a.DirRequest == JUMP {
		switch a.Dir {
		case STOPPED:
			a.DirRequest = a.Dir
			a.Dir = JUMPUP
			a.JumpStep = 0
		case LEFT:
			a.DirRequest = a.Dir
			a.Dir = JUMPLEFT
			a.JumpStep = 0
		case RIGHT:
			a.DirRequest = a.Dir
			a.Dir = JUMPRIGHT
			a.JumpStep = 0
		}
	}

	// Do the jumping
	if a.Dir == JUMPUP || a.Dir == JUMPLEFT || a.Dir == JUMPRIGHT {
		jd := jumpPaths[a.Dir][a.JumpStep]
		if m.Field[a.Y+dirs[jd].y][a.X+dirs[jd].x] == ' ' {
			a.X += dirs[jd].x
			a.Y += dirs[jd].y
			a.JumpStep++
			if a.JumpStep >= len(jumpPaths[a.Dir]) {
				a.Dir = a.DirRequest
			}
		} else {
			// If bumped into something try falling
			a.Dir = FALLING
		}
	}

	// If at the edige of the playfield stop and exit
	if (a.Dir == LEFT && a.X == 0) ||
		(a.Dir == RIGHT && a.X >= 78) {
		a.Dir = STOPPED
	}

	// If at a ladder and want to go up
	if (a.DirRequest == UP && m.Field[a.Y][a.X] == 'H') ||
		(a.DirRequest == DOWN && m.Field[a.Y][a.X] == 'H') {
		a.Dir = a.DirRequest
		a.DirRequest = STOPPED
	}

	// If falling then continue doing that until not in free space anymore,
	// then continue the previous direction (if any)
	if a.Dir == FALLING {
		if m.Field[a.Y+1][a.X] == ' ' {
			a.Y++
			return a
		}
		a.Dir = a.DirRequest
	}

	// Climb up until ladder is no more
	if a.Dir == UP {
		if m.Field[a.Y-1][a.X] == 'H' {
			a.Y--
		} else {
			a.Dir = STOPPED
		}
	}

	// Climb down until ladder is no more
	if a.Dir == DOWN {
		if m.Field[a.Y+1][a.X] == 'H' {
			a.Y++
		} else {
			a.Dir = STOPPED
		}
	}

	if a.Dir == LEFT && (m.Field[a.Y][a.X] == ' ' || m.Field[a.Y][a.X] == 'H' || m.Field[a.Y][a.X] == '*') {
		// Stepped out into the void?, the start falling, but remember the previous direction
		if m.Field[a.Y+1][a.X] == ' ' {
			a.DirRequest = a.Dir
			a.Dir = FALLING
			goto loopAgain
		}
		a.X--
	}

	if a.Dir == RIGHT && (m.Field[a.Y][a.X] == ' ' || m.Field[a.Y][a.X] == 'H' || m.Field[a.Y][a.X] == '*') {
		// Stepped out into the void?, the start falling, but remember the previous direction
		if m.Field[a.Y+1][a.X] == ' ' {
			a.DirRequest = a.Dir
			a.Dir = FALLING
			goto loopAgain
		}
		a.X++
	}

	return a
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/sha256"
	"flag"
	"log"
	"net/http"
	"os"
	"runtime"
	"time"

	redigo "github.com/garyburd/redigo/redis"
	"github.com/pierrre/githubhook"
	"github.com/pierrre/imageserver"
	imageserver_cache "github.com/pierrre/imageserver/cache"
	imageserver_cache_memory "github.com/pierrre/imageserver/cache/memory"
	imageserver_cache_redis "github.com/pierrre/imageserver/cache/redis"
	imageserver_graphicsmagick "github.com/pierrre/imageserver/graphicsmagick"
	imageserver_http "github.com/pierrre/imageserver/http"
	imageserver_http_graphicsmagick "github.com/pierrre/imageserver/http/graphicsmagick"
	imageserver_testdata "github.com/pierrre/imageserver/testdata"
)

var (
	flagHTTPAddr            = ":8080"
	flagGitHubWebhookSecret string
	flagHTTPExpires         = time.Duration(7 * 24 * time.Hour)
	flagRedis               = "localhost:6379"
	flagRedisExpire         = time.Duration(7 * 24 * time.Hour)
	flagCacheMemory         = int64(64 * (1 << 20))
)

func main() {
	parseFlags()
	log.Println("Start")
	logEnv()
	startHTTPServer()
}

func parseFlags() {
	flag.StringVar(&flagHTTPAddr, "http", flagHTTPAddr, "HTTP addr")
	flag.DurationVar(&flagHTTPExpires, "http-expires", flagHTTPExpires, "HTTP expires")
	flag.StringVar(&flagGitHubWebhookSecret, "github-webhook-secret", "", "GitHub webhook secret")
	flag.StringVar(&flagRedis, "redis", flagRedis, "Redis addr")
	flag.DurationVar(&flagRedisExpire, "redis-expire", flagRedisExpire, "Redis expire")
	flag.Int64Var(&flagCacheMemory, "cache-memory", flagCacheMemory, "Cache memory")
	flag.Parse()
}

func logEnv() {
	log.Printf("Go version: %s", runtime.Version())
	log.Printf("Go max procs: %d", runtime.GOMAXPROCS(0))
}

func startHTTPServer() {
	http.Handle("/", newImageHTTPHandler())
	http.Handle("/favicon.ico", http.NotFoundHandler())
	if h := newGitHubWebhookHTTPHandler(); h != nil {
		http.Handle("/github_webhook", h)
	}
	log.Printf("Start HTTP server on %s", flagHTTPAddr)
	err := http.ListenAndServe(flagHTTPAddr, nil)
	if err != nil {
		log.Panic(err)
	}
}

func newGitHubWebhookHTTPHandler() http.Handler {
	if flagGitHubWebhookSecret == "" {
		return nil
	}
	return &githubhook.Handler{
		Secret: flagGitHubWebhookSecret,
		Delivery: func(event string, deliveryID string, payload interface{}) {
			log.Printf("Received GitHub webhook: %s", event)
			if event == "push" {
				delay := time.Duration(5 * time.Second)
				log.Printf("Killing process in %s", delay)
				time.AfterFunc(delay, func() {
					log.Println("Killing process now")
					os.Exit(0)
				})
			}
		},
		Error: func(err error, req *http.Request) {
			log.Printf("GitHub webhook error: %s", err)
		},
	}
}

func newImageHTTPHandler() http.Handler {
	handler := http.Handler(&imageserver_http.Handler{
		Parser: &imageserver_http.ListParser{
			&imageserver_http.SourceParser{},
			&imageserver_http_graphicsmagick.Parser{},
		},
		Server:   newServer(),
		ETagFunc: imageserver_http.NewParamsHashETagFunc(sha256.New),
		ErrorFunc: func(err error, request *http.Request) {
			log.Printf("Internal error: %s", err)
		},
	})
	if flagHTTPExpires != 0 {
		handler = &imageserver_http.ExpiresHandler{
			Handler: handler,
			Expires: flagHTTPExpires,
		}
	}
	return handler
}

func newServer() imageserver.Server {
	server := imageserver_testdata.Server
	server = newServerGraphicsMagick(server)
	server = newServerLimit(server)
	server = newServerCacheRedis(server)
	server = newServerCacheMemory(server)
	return server
}

func newServerGraphicsMagick(server imageserver.Server) imageserver.Server {
	return &imageserver_graphicsmagick.Server{
		Server:     server,
		Executable: "gm",
		Timeout:    time.Duration(30 * time.Second),
		AllowedFormats: []string{
			"jpeg",
			"png",
			"bmp",
			"gif",
		},
	}
}

func newServerLimit(server imageserver.Server) imageserver.Server {
	return imageserver.NewLimitServer(server, runtime.GOMAXPROCS(0)*2)
}

func newServerCacheRedis(server imageserver.Server) imageserver.Server {
	if flagRedis == "" {
		return server
	}
	var cache imageserver_cache.Cache
	cache = &imageserver_cache_redis.Cache{
		Pool: &redigo.Pool{
			Dial: func() (redigo.Conn, error) {
				return redigo.Dial("tcp", flagRedis)
			},
			MaxIdle: 50,
		},
		Expire: flagRedisExpire,
	}
	cache = &imageserver_cache.IgnoreError{
		Cache: cache,
	}
	cache = &imageserver_cache.Async{
		Cache: cache,
	}
	return &imageserver_cache.Server{
		Server:       server,
		Cache:        cache,
		KeyGenerator: imageserver_cache.NewParamsHashKeyGenerator(sha256.New),
	}
}

func newServerCacheMemory(server imageserver.Server) imageserver.Server {
	if flagCacheMemory <= 0 {
		return server
	}
	return &imageserver_cache.Server{
		Server:       server,
		Cache:        imageserver_cache_memory.New(flagCacheMemory),
		KeyGenerator: imageserver_cache.NewParamsHashKeyGenerator(sha256.New),
	}
}
<commit_msg>change variable declaration in advanced example<commit_after>package main

import (
	"crypto/sha256"
	"flag"
	"log"
	"net/http"
	"os"
	"runtime"
	"time"

	redigo "github.com/garyburd/redigo/redis"
	"github.com/pierrre/githubhook"
	"github.com/pierrre/imageserver"
	imageserver_cache "github.com/pierrre/imageserver/cache"
	imageserver_cache_memory "github.com/pierrre/imageserver/cache/memory"
	imageserver_cache_redis "github.com/pierrre/imageserver/cache/redis"
	imageserver_graphicsmagick "github.com/pierrre/imageserver/graphicsmagick"
	imageserver_http "github.com/pierrre/imageserver/http"
	imageserver_http_graphicsmagick "github.com/pierrre/imageserver/http/graphicsmagick"
	imageserver_testdata "github.com/pierrre/imageserver/testdata"
)

var (
	flagHTTPAddr            = ":8080"
	flagGitHubWebhookSecret string
	flagHTTPExpires         = time.Duration(7 * 24 * time.Hour)
	flagRedis               = "localhost:6379"
	flagRedisExpire         = time.Duration(7 * 24 * time.Hour)
	flagCacheMemory         = int64(64 * (1 << 20))
)

func main() {
	parseFlags()
	log.Println("Start")
	logEnv()
	startHTTPServer()
}

func parseFlags() {
	flag.StringVar(&flagHTTPAddr, "http", flagHTTPAddr, "HTTP addr")
	flag.DurationVar(&flagHTTPExpires, "http-expires", flagHTTPExpires, "HTTP expires")
	flag.StringVar(&flagGitHubWebhookSecret, "github-webhook-secret", "", "GitHub webhook secret")
	flag.StringVar(&flagRedis, "redis", flagRedis, "Redis addr")
	flag.DurationVar(&flagRedisExpire, "redis-expire", flagRedisExpire, "Redis expire")
	flag.Int64Var(&flagCacheMemory, "cache-memory", flagCacheMemory, "Cache memory")
	flag.Parse()
}

func logEnv() {
	log.Printf("Go version: %s", runtime.Version())
	log.Printf("Go max procs: %d", runtime.GOMAXPROCS(0))
}

func startHTTPServer() {
	http.Handle("/", newImageHTTPHandler())
	http.Handle("/favicon.ico", http.NotFoundHandler())
	if h := newGitHubWebhookHTTPHandler(); h != nil {
		http.Handle("/github_webhook", h)
	}
	log.Printf("Start HTTP server on %s", flagHTTPAddr)
	err := http.ListenAndServe(flagHTTPAddr, nil)
	if err != nil {
		log.Panic(err)
	}
}

func newGitHubWebhookHTTPHandler() http.Handler {
	if flagGitHubWebhookSecret == "" {
		return nil
	}
	return &githubhook.Handler{
		Secret: flagGitHubWebhookSecret,
		Delivery: func(event string, deliveryID string, payload interface{}) {
			log.Printf("Received GitHub webhook: %s", event)
			if event == "push" {
				delay := time.Duration(5 * time.Second)
				log.Printf("Killing process in %s", delay)
				time.AfterFunc(delay, func() {
					log.Println("Killing process now")
					os.Exit(0)
				})
			}
		},
		Error: func(err error, req *http.Request) {
			log.Printf("GitHub webhook error: %s", err)
		},
	}
}

func newImageHTTPHandler() http.Handler {
	var handler http.Handler
	handler = &imageserver_http.Handler{
		Parser: &imageserver_http.ListParser{
			&imageserver_http.SourceParser{},
			&imageserver_http_graphicsmagick.Parser{},
		},
		Server:   newServer(),
		ETagFunc: imageserver_http.NewParamsHashETagFunc(sha256.New),
		ErrorFunc: func(err error, request *http.Request) {
			log.Printf("Internal error: %s", err)
		},
	}
	if flagHTTPExpires != 0 {
		handler = &imageserver_http.ExpiresHandler{
			Handler: handler,
			Expires: flagHTTPExpires,
		}
	}
	return handler
}

func newServer() imageserver.Server {
	server := imageserver_testdata.Server
	server = newServerGraphicsMagick(server)
	server = newServerLimit(server)
	server = newServerCacheRedis(server)
	server = newServerCacheMemory(server)
	return server
}

func newServerGraphicsMagick(server imageserver.Server) imageserver.Server {
	return &imageserver_graphicsmagick.Server{
		Server:     server,
		Executable: "gm",
		Timeout:    time.Duration(30 * time.Second),
		AllowedFormats: []string{
			"jpeg",
			"png",
			"bmp",
			"gif",
		},
	}
}

func newServerLimit(server imageserver.Server) imageserver.Server {
	return imageserver.NewLimitServer(server, runtime.GOMAXPROCS(0)*2)
}

func newServerCacheRedis(server imageserver.Server) imageserver.Server {
	if flagRedis == "" {
		return server
	}
	var cache imageserver_cache.Cache
	cache = &imageserver_cache_redis.Cache{
		Pool: &redigo.Pool{
			Dial: func() (redigo.Conn, error) {
				return redigo.Dial("tcp", flagRedis)
			},
			MaxIdle: 50,
		},
		Expire: flagRedisExpire,
	}
	cache = &imageserver_cache.IgnoreError{
		Cache: cache,
	}
	cache = &imageserver_cache.Async{
		Cache: cache,
	}
	return &imageserver_cache.Server{
		Server:       server,
		Cache:        cache,
		KeyGenerator: imageserver_cache.NewParamsHashKeyGenerator(sha256.New),
	}
}

func newServerCacheMemory(server imageserver.Server) imageserver.Server {
	if flagCacheMemory <= 0 {
		return server
	}
	return &imageserver_cache.Server{
		Server:       server,
		Cache:        imageserver_cache_memory.New(flagCacheMemory),
		KeyGenerator: imageserver_cache.NewParamsHashKeyGenerator(sha256.New),
	}
}
<|endoftext|>


<commit_before>package rpm

import (
	"compress/gzip"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/cavaliercoder/go-cpio"
)

// ExampleExtractArchive demonstrates how to use go-rpm with the compress/gzip
// library and go-cpio to extract all files from a package archive.
func ExampleExtractArchive() {
	// create destination directory
	path, err := ioutil.TempDir("", "rpm-example-")
	if err != nil {
		panic(err)
	}

	// open a rpm package for reading
	r, err := os.Open("testdata/centos-release-4-0.1.x86_64.rpm")
	if err != nil {
		panic(err)
	}
	defer r.Close()

	// read package info
	_, err = ReadPackageFile(r)
	if err != nil {
		panic(err)
	}

	// deflate payload with gzip (newer packages need xz)
	gzr, err := gzip.NewReader(r)
	if err != nil {
		panic(err)
	}
	defer gzr.Close()

	// read each file in payload with cpio
	cpr := cpio.NewReader(gzr)
	for {
		fi, err := cpr.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			panic(err)
		}

		dst := filepath.Join(path, fi.Name)

		// create parent directories
		dir := dst
		if !fi.IsDir() {
			dir = filepath.Dir(dst)
		}
		if err := os.MkdirAll(dir, 0755); err != nil {
			panic(err)
		}

		// copy file contents
		if !fi.IsDir() {
			fmt.Printf("extracting %s...\n", fi.Name[1:])
			f, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY, 0644)
			if err != nil {
				panic(err)
			}
			defer f.Close()

			_, err = io.Copy(f, cpr)
			if err != nil {
				panic(err)
			}
			f.Close()
		}

		// clean up
		os.RemoveAll(path)
	}

	// Output:
	// extracting /etc/issue...
	// extracting /etc/issue.net...
	// extracting /etc/redhat-release...
	// extracting /usr/share/doc/centos-release-4/EULA...
	// extracting /usr/share/doc/centos-release-4/GPL...
	// extracting /usr/share/doc/centos-release-4/RELEASE-NOTES-en...
	// extracting /usr/share/doc/centos-release-4/RELEASE-NOTES-en.html...
	// extracting /usr/share/doc/centos-release-4/RPM-GPG-KEY...
	// extracting /usr/share/doc/centos-release-4/RPM-GPG-KEY-centos4...
	// extracting /usr/share/doc/centos-release-4/autorun-template...
	// extracting /usr/share/doc/centos-release-4/centosdocs-man.css...
	// extracting /usr/share/eula/eula.en_US...
	// extracting /usr/share/firstboot/modules/eula.py...
	// extracting /usr/share/firstboot/modules/eula.pyc...
	// extracting /var/lib/supportinfo...
}
<commit_msg>Fixed broken example name<commit_after>package rpm

import (
	"compress/gzip"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/cavaliercoder/go-cpio"
)

// ExampleExtractArchive demonstrates how to use go-rpm with the compress/gzip
// library and go-cpio to extract all files from a package archive.
func Example() {
	// create destination directory
	path, err := ioutil.TempDir("", "rpm-example-")
	if err != nil {
		panic(err)
	}

	// open a rpm package for reading
	r, err := os.Open("testdata/centos-release-4-0.1.x86_64.rpm")
	if err != nil {
		panic(err)
	}
	defer r.Close()

	// read package info
	_, err = ReadPackageFile(r)
	if err != nil {
		panic(err)
	}

	// deflate payload with gzip (newer packages need xz)
	gzr, err := gzip.NewReader(r)
	if err != nil {
		panic(err)
	}
	defer gzr.Close()

	// read each file in payload with cpio
	cpr := cpio.NewReader(gzr)
	for {
		fi, err := cpr.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			panic(err)
		}

		dst := filepath.Join(path, fi.Name)

		// create parent directories
		dir := dst
		if !fi.IsDir() {
			dir = filepath.Dir(dst)
		}
		if err := os.MkdirAll(dir, 0755); err != nil {
			panic(err)
		}

		// copy file contents
		if !fi.IsDir() {
			fmt.Printf("extracting %s...\n", fi.Name[1:])
			f, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY, 0644)
			if err != nil {
				panic(err)
			}
			defer f.Close()

			_, err = io.Copy(f, cpr)
			if err != nil {
				panic(err)
			}
			f.Close()
		}

		// clean up
		os.RemoveAll(path)
	}

	// Output:
	// extracting /etc/issue...
	// extracting /etc/issue.net...
	// extracting /etc/redhat-release...
	// extracting /usr/share/doc/centos-release-4/EULA...
	// extracting /usr/share/doc/centos-release-4/GPL...
	// extracting /usr/share/doc/centos-release-4/RELEASE-NOTES-en...
	// extracting /usr/share/doc/centos-release-4/RELEASE-NOTES-en.html...
	// extracting /usr/share/doc/centos-release-4/RPM-GPG-KEY...
	// extracting /usr/share/doc/centos-release-4/RPM-GPG-KEY-centos4...
	// extracting /usr/share/doc/centos-release-4/autorun-template...
	// extracting /usr/share/doc/centos-release-4/centosdocs-man.css...
	// extracting /usr/share/eula/eula.en_US...
	// extracting /usr/share/firstboot/modules/eula.py...
	// extracting /usr/share/firstboot/modules/eula.pyc...
	// extracting /var/lib/supportinfo...
}
<|endoftext|>


<commit_before>// +build appengine

/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package appengine

import (
	"bytes"
	"errors"
	"fmt"
	"net/http"
	"io"
	"io/ioutil"
	"log"
	"os"
	"regexp"
	"strings"
	"time"

	"appengine"
	"appengine/datastore"
	"appengine/blobstore"

	"camlistore.org/pkg/blobref"
	"camlistore.org/pkg/blobserver"
	"camlistore.org/pkg/jsonconfig"
)

var _ = log.Printf

const (
	blobKind = "Blob"
	memKind  = "NsBlobMember" // blob membership in a namespace
)

var _ blobserver.Storage = (*appengineStorage)(nil)

type appengineStorage struct {
	*blobserver.SimpleBlobHubPartitionMap
	namespace string // never empty; config initializes to at least "-"
	ctx       appengine.Context
}

// blobEnt is stored once per unique blob, keyed by blobref.
type blobEnt struct {
	Size       []byte // an int64 as "%d" to make it unindexed
	BlobKey    []byte // an appengine.BlobKey
	Namespaces []byte // |-separated string of namespaces

	// TODO(bradfitz): IsCamliSchemaBlob bool? ... probably want
	// on enumeration (memEnt) too.
}

// memEnt is stored once per blob in a namespace, keyed by "ns|blobref"
type memEnt struct {
	Size []byte // an int64 as "%d" to make it unindexed
}

func (b *blobEnt) size() (int64, error) {
	return byteDecSize(b.Size)
}

func (m *memEnt) size() (int64, error) {
	return byteDecSize(m.Size)
}

func byteDecSize(b []byte) (int64, error) {
	var size int64
	n, err := fmt.Fscanf(bytes.NewBuffer(b), "%d", &size)
	if n != 1 || err != nil {
		return 0, fmt.Errorf("invalid Size column in datastore: %q", string(b))
	}
	return size, nil
}

func (b *blobEnt) inNamespace(ns string) (out bool) {
	for _, in := range strings.Split(string(b.Namespaces), "|") {
		if ns == in {
			return true
		}
	}
	return false
}

func entKey(c appengine.Context, br *blobref.BlobRef) *datastore.Key {
	return datastore.NewKey(c, blobKind, br.String(), 0, nil)
}

func (s *appengineStorage) memKey(c appengine.Context, br *blobref.BlobRef) *datastore.Key {
	return datastore.NewKey(c, memKind, fmt.Sprintf("%s|%s", s.namespace, br.String()), 0, nil)
}

func fetchEnt(c appengine.Context, br *blobref.BlobRef) (*blobEnt, error) {
	row := new(blobEnt)
	err := datastore.Get(c, entKey(c, br), row)
	if err != nil {
		return nil, err
	}
	return row, nil
}

var errNoContext = errors.New("Internal error: no App Engine context is available")

func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (storage blobserver.Storage, err error) {
	sto := &appengineStorage{
		SimpleBlobHubPartitionMap: &blobserver.SimpleBlobHubPartitionMap{},
	}
	sto.namespace = config.OptionalString("namespace", "")
	if err := config.Validate(); err != nil {
		return nil, err
	}
	sto.namespace, err = sanitizeNamespace(sto.namespace)
	if err != nil {
		return nil, err
	}
	return sto, nil
}

var _ blobserver.ContextWrapper = (*appengineStorage)(nil)

func (sto *appengineStorage) WrapContext(req *http.Request) blobserver.Storage {
	s2 := new(appengineStorage)
	*s2 = *sto
	s2.ctx = appengine.NewContext(req)
	return s2
}

func (sto *appengineStorage) FetchStreaming(br *blobref.BlobRef) (file io.ReadCloser, size int64, err error) {
	if sto.ctx == nil {
		err = errNoContext
		return
	}
	row, err := fetchEnt(sto.ctx, br)
	if err == datastore.ErrNoSuchEntity {
		err = os.ErrNotExist
		return
	}
	if err != nil {
		return
	}
	if !row.inNamespace(sto.namespace) {
		err = os.ErrNotExist
		return
	}
	size, err = row.size()
	if err != nil {
		return
	}
	reader := blobstore.NewReader(sto.ctx, appengine.BlobKey(string(row.BlobKey)))
	return ioutil.NopCloser(reader), size, nil
}

var crossGroupTransaction = &datastore.TransactionOptions{XG: true}

func (sto *appengineStorage) ReceiveBlob(br *blobref.BlobRef, in io.Reader) (sb blobref.SizedBlobRef, err error) {
	if sto.ctx == nil {
		err = errNoContext
		return
	}

	var b bytes.Buffer
	hash := br.Hash()
	written, err := io.Copy(io.MultiWriter(hash, &b), in)
	if err != nil {
		return
	}
	if !br.HashMatches(hash) {
		err = blobserver.ErrCorruptBlob
		return
	}

	// bkey is non-empty once we've uploaded the blob.
	var bkey appengine.BlobKey

	// uploadBlob uploads the blob, unless it's already been done.
	uploadBlob := func(ctx appengine.Context) error {
		if len(bkey) > 0 {
			return nil // already done in previous transaction attempt
		}
		bw, err := blobstore.Create(ctx, "application/octet-stream")
		if err != nil {
			return err
		}
		_, err = io.Copy(bw, &b)
		if err != nil {
			// TODO(bradfitz): try to clean up; close it, see if we can find the key, delete it.
			ctx.Errorf("blobstore Copy error: %v", err)
			return err
		}
		err = bw.Close()
		if err != nil {
			// TODO(bradfitz): try to clean up; see if we can find the key, delete it.
			ctx.Errorf("blobstore Close error: %v", err)
			return err
		}
		k, err := bw.Key()
		if err == nil {
			bkey = k
		}
		return err
	}

	tryFunc := func(tc appengine.Context) error {
		row, err := fetchEnt(sto.ctx, br)
		switch err {
		case datastore.ErrNoSuchEntity:
			if err := uploadBlob(sto.ctx); err != nil {
				tc.Errorf("uploadBlob failed: %v", err)
				return err
			}
			row = &blobEnt{
				Size:       []byte(fmt.Sprintf("%d", written)),
				BlobKey:    []byte(string(bkey)),
				Namespaces: []byte(sto.namespace),
			}
			_, err = datastore.Put(tc, entKey(tc, br), row)
			if err != nil {
				return err
			}
		case nil:
			if row.inNamespace(sto.namespace) {
				// Nothing to do
				return nil
			}
			row.Namespaces = []byte(string(row.Namespaces) + "|" + sto.namespace)
			_, err = datastore.Put(tc, entKey(tc, br), row)
			if err != nil {
				return err
			}
		default:
			return err
		}

		// Add membership row
		_, err = datastore.Put(tc, sto.memKey(tc, br), &memEnt{
			Size: []byte(fmt.Sprintf("%d", written)),
		})
		return err
	}
	err = datastore.RunInTransaction(sto.ctx, tryFunc, crossGroupTransaction)
	if err != nil {
		if len(bkey) > 0 {
			// If we just created this blob but we
			// ultimately failed, try our best to delete
			// it so it's not orphaned.
			blobstore.Delete(sto.ctx, bkey)
		}
		return
	}
	return blobref.SizedBlobRef{br, written}, nil
}

// NOTE(bslatkin): No fucking clue if this works.
func (sto *appengineStorage) RemoveBlobs(blobs []*blobref.BlobRef) error {
	if sto.ctx == nil {
		return errNoContext
	}

	tryFunc := func(tc appengine.Context, br *blobref.BlobRef) error {
		// TODO(bslatkin): Make the DB gets in this a multi-get.
		// Remove the namespace from the blobEnt
		row, err := fetchEnt(sto.ctx, br)
		switch err {
		case datastore.ErrNoSuchEntity:
			// Doesn't exist, that means there should be no memEnt, but let's be
			// paranoid and double check anyways.
		case nil:
			// blobEnt exists, remove our namespace from it if possible.
			newNS := []string{}
			for _, val := range strings.Split(string(row.Namespaces), "|") {
				if val != sto.namespace {
					newNS = append(newNS, val)
				}
			}
			if v := strings.Join(newNS, "|"); v != string(row.Namespaces) {
				row.Namespaces = []byte(v)
				_, err = datastore.Put(tc, entKey(tc, br), row)
				if err != nil {
					return err
				}
			}
		default:
			return err
		}

		// Blindly delete the memEnt.
		err = datastore.Delete(tc, sto.memKey(tc, br));
		return err
	}

	for _, br := range blobs {
		ret := datastore.RunInTransaction(
			sto.ctx,
			func(tc appengine.Context) error {
				return tryFunc(tc, br)
			},
			crossGroupTransaction)
		if ret != nil {
			return ret
		}
	}
	return nil
}

func (sto *appengineStorage) StatBlobs(dest chan<- blobref.SizedBlobRef, blobs []*blobref.BlobRef, wait time.Duration) error {
	if sto.ctx == nil {
		return errNoContext
	}
	var (
		keys = make([]*datastore.Key, 0, len(blobs))
		out  = make([]interface{}, 0, len(blobs))
		errs = make([]error, len(blobs))
	)
	for _, br := range blobs {
		keys = append(keys, sto.memKey(sto.ctx, br))
		out = append(out, new(memEnt))
	}
	err := datastore.GetMulti(sto.ctx, keys, out)
	if merr, ok := err.(appengine.MultiError); ok {
		errs = []error(merr)
		err = nil
	}
	if err != nil {
		return err
	}
	for i, br := range blobs {
		thisErr := errs[i]
		if thisErr == datastore.ErrNoSuchEntity {
			continue
		}
		if thisErr != nil {
			err = errs[i] // just return last one found?
			continue
		}
		ent := out[i].(*memEnt)
		size, err := ent.size()
		if err == nil {
			dest <- blobref.SizedBlobRef{br, size}
		} else {
			sto.ctx.Warningf("skipping corrupt blob %s: %v", br, err)
		}
	}
	return err
}

func (sto *appengineStorage) EnumerateBlobs(dest chan<- blobref.SizedBlobRef, after string, limit int, wait time.Duration) error {
	defer close(dest)
	if sto.ctx == nil {
		return errNoContext
	}
	prefix := sto.namespace + "|"
	keyBegin := datastore.NewKey(sto.ctx, memKind, prefix+after, 0, nil)
	keyEnd := datastore.NewKey(sto.ctx, memKind, sto.namespace+"~", 0, nil)

	q := datastore.NewQuery(memKind).Limit(int(limit)).Filter("__key__>", keyBegin).Filter("__key__<", keyEnd)
	it := q.Run(sto.ctx)
	var row memEnt
	for {
		key, err := it.Next(&row)
		if err == datastore.Done {
			break
		}
		if err != nil {
			return err
		}
		size, err := row.size()
		if err != nil {
			return err
		}
		dest <- blobref.SizedBlobRef{blobref.Parse(key.StringID()[len(prefix):]), size}
	}
	return nil
}

var validQueueName = regexp.MustCompile(`^[a-zA-Z0-9\-\_]+$`)

// TODO(bslatkin): This does not work on App Engine yet because there are no
// background threads to do the sync loop. The plan is to break the
// syncer code up into two parts: 1) accepts notifications of new blobs to
// sync, 2) does one unit of work enumerating recent blobs and syncing them.
// In App Engine land, 1) will result in a task to be enqueued, and 2) will
// be called from within that queue context.

func (sto *appengineStorage) CreateQueue(name string) (blobserver.Storage, error) {
	if !validQueueName.MatchString(name) {
		return nil, fmt.Errorf("invalid queue name %q", name)
	}
	if sto.namespace != "" && strings.HasPrefix(sto.namespace, "queue-") {
		return nil, fmt.Errorf("can't create queue %q on existing queue %q",
			name, sto.namespace)
	}
	q := &appengineStorage{
		SimpleBlobHubPartitionMap: &blobserver.SimpleBlobHubPartitionMap{},
		namespace: "queue-" + name,
	}
	return q, nil
}
<commit_msg>gofmt<commit_after>// +build appengine

/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package appengine

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"regexp"
	"strings"
	"time"

	"appengine"
	"appengine/blobstore"
	"appengine/datastore"

	"camlistore.org/pkg/blobref"
	"camlistore.org/pkg/blobserver"
	"camlistore.org/pkg/jsonconfig"
)

var _ = log.Printf

const (
	blobKind = "Blob"
	memKind  = "NsBlobMember" // blob membership in a namespace
)

var _ blobserver.Storage = (*appengineStorage)(nil)

type appengineStorage struct {
	*blobserver.SimpleBlobHubPartitionMap
	namespace string // never empty; config initializes to at least "-"
	ctx       appengine.Context
}

// blobEnt is stored once per unique blob, keyed by blobref.
type blobEnt struct {
	Size       []byte // an int64 as "%d" to make it unindexed
	BlobKey    []byte // an appengine.BlobKey
	Namespaces []byte // |-separated string of namespaces

	// TODO(bradfitz): IsCamliSchemaBlob bool? ... probably want
	// on enumeration (memEnt) too.
}

// memEnt is stored once per blob in a namespace, keyed by "ns|blobref"
type memEnt struct {
	Size []byte // an int64 as "%d" to make it unindexed
}

func (b *blobEnt) size() (int64, error) {
	return byteDecSize(b.Size)
}

func (m *memEnt) size() (int64, error) {
	return byteDecSize(m.Size)
}

func byteDecSize(b []byte) (int64, error) {
	var size int64
	n, err := fmt.Fscanf(bytes.NewBuffer(b), "%d", &size)
	if n != 1 || err != nil {
		return 0, fmt.Errorf("invalid Size column in datastore: %q", string(b))
	}
	return size, nil
}

func (b *blobEnt) inNamespace(ns string) (out bool) {
	for _, in := range strings.Split(string(b.Namespaces), "|") {
		if ns == in {
			return true
		}
	}
	return false
}

func entKey(c appengine.Context, br *blobref.BlobRef) *datastore.Key {
	return datastore.NewKey(c, blobKind, br.String(), 0, nil)
}

func (s *appengineStorage) memKey(c appengine.Context, br *blobref.BlobRef) *datastore.Key {
	return datastore.NewKey(c, memKind, fmt.Sprintf("%s|%s", s.namespace, br.String()), 0, nil)
}

func fetchEnt(c appengine.Context, br *blobref.BlobRef) (*blobEnt, error) {
	row := new(blobEnt)
	err := datastore.Get(c, entKey(c, br), row)
	if err != nil {
		return nil, err
	}
	return row, nil
}

var errNoContext = errors.New("Internal error: no App Engine context is available")

func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (storage blobserver.Storage, err error) {
	sto := &appengineStorage{
		SimpleBlobHubPartitionMap: &blobserver.SimpleBlobHubPartitionMap{},
	}
	sto.namespace = config.OptionalString("namespace", "")
	if err := config.Validate(); err != nil {
		return nil, err
	}
	sto.namespace, err = sanitizeNamespace(sto.namespace)
	if err != nil {
		return nil, err
	}
	return sto, nil
}

var _ blobserver.ContextWrapper = (*appengineStorage)(nil)

func (sto *appengineStorage) WrapContext(req *http.Request) blobserver.Storage {
	s2 := new(appengineStorage)
	*s2 = *sto
	s2.ctx = appengine.NewContext(req)
	return s2
}

func (sto *appengineStorage) FetchStreaming(br *blobref.BlobRef) (file io.ReadCloser, size int64, err error) {
	if sto.ctx == nil {
		err = errNoContext
		return
	}
	row, err := fetchEnt(sto.ctx, br)
	if err == datastore.ErrNoSuchEntity {
		err = os.ErrNotExist
		return
	}
	if err != nil {
		return
	}
	if !row.inNamespace(sto.namespace) {
		err = os.ErrNotExist
		return
	}
	size, err = row.size()
	if err != nil {
		return
	}
	reader := blobstore.NewReader(sto.ctx, appengine.BlobKey(string(row.BlobKey)))
	return ioutil.NopCloser(reader), size, nil
}

var crossGroupTransaction = &datastore.TransactionOptions{XG: true}

func (sto *appengineStorage) ReceiveBlob(br *blobref.BlobRef, in io.Reader) (sb blobref.SizedBlobRef, err error) {
	if sto.ctx == nil {
		err = errNoContext
		return
	}

	var b bytes.Buffer
	hash := br.Hash()
	written, err := io.Copy(io.MultiWriter(hash, &b), in)
	if err != nil {
		return
	}
	if !br.HashMatches(hash) {
		err = blobserver.ErrCorruptBlob
		return
	}

	// bkey is non-empty once we've uploaded the blob.
	var bkey appengine.BlobKey

	// uploadBlob uploads the blob, unless it's already been done.
	uploadBlob := func(ctx appengine.Context) error {
		if len(bkey) > 0 {
			return nil // already done in previous transaction attempt
		}
		bw, err := blobstore.Create(ctx, "application/octet-stream")
		if err != nil {
			return err
		}
		_, err = io.Copy(bw, &b)
		if err != nil {
			// TODO(bradfitz): try to clean up; close it, see if we can find the key, delete it.
			ctx.Errorf("blobstore Copy error: %v", err)
			return err
		}
		err = bw.Close()
		if err != nil {
			// TODO(bradfitz): try to clean up; see if we can find the key, delete it.
			ctx.Errorf("blobstore Close error: %v", err)
			return err
		}
		k, err := bw.Key()
		if err == nil {
			bkey = k
		}
		return err
	}

	tryFunc := func(tc appengine.Context) error {
		row, err := fetchEnt(sto.ctx, br)
		switch err {
		case datastore.ErrNoSuchEntity:
			if err := uploadBlob(sto.ctx); err != nil {
				tc.Errorf("uploadBlob failed: %v", err)
				return err
			}
			row = &blobEnt{
				Size:       []byte(fmt.Sprintf("%d", written)),
				BlobKey:    []byte(string(bkey)),
				Namespaces: []byte(sto.namespace),
			}
			_, err = datastore.Put(tc, entKey(tc, br), row)
			if err != nil {
				return err
			}
		case nil:
			if row.inNamespace(sto.namespace) {
				// Nothing to do
				return nil
			}
			row.Namespaces = []byte(string(row.Namespaces) + "|" + sto.namespace)
			_, err = datastore.Put(tc, entKey(tc, br), row)
			if err != nil {
				return err
			}
		default:
			return err
		}

		// Add membership row
		_, err = datastore.Put(tc, sto.memKey(tc, br), &memEnt{
			Size: []byte(fmt.Sprintf("%d", written)),
		})
		return err
	}
	err = datastore.RunInTransaction(sto.ctx, tryFunc, crossGroupTransaction)
	if err != nil {
		if len(bkey) > 0 {
			// If we just created this blob but we
			// ultimately failed, try our best to delete
			// it so it's not orphaned.
			blobstore.Delete(sto.ctx, bkey)
		}
		return
	}
	return blobref.SizedBlobRef{br, written}, nil
}

// NOTE(bslatkin): No fucking clue if this works.
func (sto *appengineStorage) RemoveBlobs(blobs []*blobref.BlobRef) error {
	if sto.ctx == nil {
		return errNoContext
	}

	tryFunc := func(tc appengine.Context, br *blobref.BlobRef) error {
		// TODO(bslatkin): Make the DB gets in this a multi-get.
		// Remove the namespace from the blobEnt
		row, err := fetchEnt(sto.ctx, br)
		switch err {
		case datastore.ErrNoSuchEntity:
			// Doesn't exist, that means there should be no memEnt, but let's be
			// paranoid and double check anyways.
		case nil:
			// blobEnt exists, remove our namespace from it if possible.
			newNS := []string{}
			for _, val := range strings.Split(string(row.Namespaces), "|") {
				if val != sto.namespace {
					newNS = append(newNS, val)
				}
			}
			if v := strings.Join(newNS, "|"); v != string(row.Namespaces) {
				row.Namespaces = []byte(v)
				_, err = datastore.Put(tc, entKey(tc, br), row)
				if err != nil {
					return err
				}
			}
		default:
			return err
		}

		// Blindly delete the memEnt.
		err = datastore.Delete(tc, sto.memKey(tc, br))
		return err
	}

	for _, br := range blobs {
		ret := datastore.RunInTransaction(
			sto.ctx,
			func(tc appengine.Context) error {
				return tryFunc(tc, br)
			},
			crossGroupTransaction)
		if ret != nil {
			return ret
		}
	}
	return nil
}

func (sto *appengineStorage) StatBlobs(dest chan<- blobref.SizedBlobRef, blobs []*blobref.BlobRef, wait time.Duration) error {
	if sto.ctx == nil {
		return errNoContext
	}
	var (
		keys = make([]*datastore.Key, 0, len(blobs))
		out  = make([]interface{}, 0, len(blobs))
		errs = make([]error, len(blobs))
	)
	for _, br := range blobs {
		keys = append(keys, sto.memKey(sto.ctx, br))
		out = append(out, new(memEnt))
	}
	err := datastore.GetMulti(sto.ctx, keys, out)
	if merr, ok := err.(appengine.MultiError); ok {
		errs = []error(merr)
		err = nil
	}
	if err != nil {
		return err
	}
	for i, br := range blobs {
		thisErr := errs[i]
		if thisErr == datastore.ErrNoSuchEntity {
			continue
		}
		if thisErr != nil {
			err = errs[i] // just return last one found?
			continue
		}
		ent := out[i].(*memEnt)
		size, err := ent.size()
		if err == nil {
			dest <- blobref.SizedBlobRef{br, size}
		} else {
			sto.ctx.Warningf("skipping corrupt blob %s: %v", br, err)
		}
	}
	return err
}

func (sto *appengineStorage) EnumerateBlobs(dest chan<- blobref.SizedBlobRef, after string, limit int, wait time.Duration) error {
	defer close(dest)
	if sto.ctx == nil {
		return errNoContext
	}
	prefix := sto.namespace + "|"
	keyBegin := datastore.NewKey(sto.ctx, memKind, prefix+after, 0, nil)
	keyEnd := datastore.NewKey(sto.ctx, memKind, sto.namespace+"~", 0, nil)

	q := datastore.NewQuery(memKind).Limit(int(limit)).Filter("__key__>", keyBegin).Filter("__key__<", keyEnd)
	it := q.Run(sto.ctx)
	var row memEnt
	for {
		key, err := it.Next(&row)
		if err == datastore.Done {
			break
		}
		if err != nil {
			return err
		}
		size, err := row.size()
		if err != nil {
			return err
		}
		dest <- blobref.SizedBlobRef{blobref.Parse(key.StringID()[len(prefix):]), size}
	}
	return nil
}

var validQueueName = regexp.MustCompile(`^[a-zA-Z0-9\-\_]+$`)

// TODO(bslatkin): This does not work on App Engine yet because there are no
// background threads to do the sync loop. The plan is to break the
// syncer code up into two parts: 1) accepts notifications of new blobs to
// sync, 2) does one unit of work enumerating recent blobs and syncing them.
// In App Engine land, 1) will result in a task to be enqueued, and 2) will
// be called from within that queue context.

func (sto *appengineStorage) CreateQueue(name string) (blobserver.Storage, error) {
	if !validQueueName.MatchString(name) {
		return nil, fmt.Errorf("invalid queue name %q", name)
	}
	if sto.namespace != "" && strings.HasPrefix(sto.namespace, "queue-") {
		return nil, fmt.Errorf("can't create queue %q on existing queue %q",
			name, sto.namespace)
	}
	q := &appengineStorage{
		SimpleBlobHubPartitionMap: &blobserver.SimpleBlobHubPartitionMap{},
		namespace:                 "queue-" + name,
	}
	return q, nil
}
<|endoftext|>


<commit_before>package dskvs

import (
	"bytes"
	"encoding/json"
	"strconv"
	"testing"
)

type Data struct {
	Descr string
}

/*
	Boilerplate
*/

func setUp(t *testing.T) *Store {
	store, err := NewStore("./db")
	if err != nil {
		t.Fatalf("Error creating store", err)
	}

	err = store.Load()
	if err != nil {
		t.Fatalf("Error loading store", err)
	}
	return store
}

func tearDown(store *Store, t *testing.T) {
	err := store.Close()
	if err != nil {
		t.Fatalf("Error closing store", err)
	}
}

func getData(d Data, t *testing.T) []byte {
	dataBytes, err := json.Marshal(d)
	if err != nil {
		t.Fatal("Error with test data", err)
	}
	return dataBytes
}

/*
	Test cases
*/

func TestCreatingStore(t *testing.T) {
	store := setUp(t)
	tearDown(store, t)
}

func TestSingleOperation(t *testing.T) {

	store := setUp(t)
	defer tearDown(store, t)

	key := "artist/daftpunk"
	expected := getData(Data{"The peak of awesome"}, t)

	err := store.Put(key, expected)
	if err != nil {
		t.Fatalf("Error putting data in", err)
	}

	actual, err := store.Get(key)
	if err != nil {
		t.Fatalf("Error getting data back", err)
	}

	if !bytes.Equal(expected, actual) {
		t.Fatalf("Expected <%s> but was <%s>",
			expected,
			actual)
	}

	err = store.Delete(key)
	if err != nil {
		t.Fatalf("Error deleting data we just put", err)
	}
	_, err = store.Get(key)
	if err == nil {
		t.Fatalf("Expected to receive KeyError but no error")
	}
	switch err := err.(type) {
	case KeyError:
		// Expected
		break
	default:
		t.Fatalf("Expected to receive KeyError but got <%s>", err)
	}

}

func TestMultipleOperations(t *testing.T) {

	store := setUp(t)
	defer tearDown(store, t)

	var key string
	var expected []byte
	var actual []byte
	for i := int(0); i < 10; i++ {
		key = "artist/daftpunk" + strconv.Itoa(i)
		expected = getData(Data{"The peak of awesome" + strconv.Itoa(i)}, t)

		err := store.Put(key, expected)
		if err != nil {
			t.Errorf("Error putting data in", err)
		}
		actual, err = store.Get(key)
		if err != nil {
			t.Errorf("Error getting data back", err)
		}

		if !bytes.Equal(expected, actual) {
			t.Errorf("Expected <%s> but was <%s>",
				expected,
				actual)
		}
	}

	err := store.DeleteAll("artist")
	if err != nil {
		t.Fatal("Error deleting key", err)
	}
	for i := int(0); i < 10; i++ {
		key = "artist/daftpunk" + strconv.Itoa(i)
		_, err = store.Get(key)
		if err == nil {
			t.Fatalf("Expected to receive KeyError, but got no error at all")
		}
		switch err := err.(type) {
		case KeyError:
			// Expected
			break
		default:
			t.Fatalf("Expected to receive KeyError but got <%s>", err)
		}
	}
}
<commit_msg>Implement concurrent test.<commit_after>package dskvs

import (
	"bytes"
	"encoding/json"
	"fmt"
	"runtime"
	"strconv"
	"sync"
	"sync/atomic"
	"testing"
	"time"
)

type Data struct {
	Descr string
}

/*
	Boilerplate
*/

func setUp(t *testing.T) *Store {
	store, err := NewStore("./db")
	if err != nil {
		t.Fatalf("Error creating store", err)
	}

	err = store.Load()
	if err != nil {
		t.Fatalf("Error loading store", err)
	}
	return store
}

func tearDown(store *Store, t *testing.T) {
	err := store.Close()
	if err != nil {
		t.Fatalf("Error closing store", err)
	}
}

func generateData(d Data, t *testing.T) []byte {
	dataBytes, err := json.Marshal(d)
	if err != nil {
		t.Fatal("Error with test data", err)
	}
	return dataBytes
}

/*
	Common checks
*/

func checkGetIsEmpty(store *Store, key string, t *testing.T) {
	_, err := store.Get(key)
	if err == nil {
		t.Fatalf("Expected to receive KeyError but no error")
	}
	switch err := err.(type) {
	case KeyError:
		// Expected
		break
	default:
		t.Fatalf("Expected to receive KeyError but got <%s>", err)
	}
}

/*
	Test cases, single goroutine
*/

func TestCreatingStore(t *testing.T) {
	store := setUp(t)
	tearDown(store, t)
}

func TestSingleOperation(t *testing.T) {

	store := setUp(t)
	defer tearDown(store, t)

	key := "artist/daftpunk"
	expected := generateData(Data{"The peak of awesome"}, t)

	err := store.Put(key, expected)
	if err != nil {
		t.Fatalf("Error putting data in", err)
	}

	actual, err := store.Get(key)
	if err != nil {
		t.Fatalf("Error getting data back", err)
	}

	if !bytes.Equal(expected, actual) {
		t.Fatalf("Expected <%s> but was <%s>",
			expected,
			actual)
	}

	err = store.Delete(key)
	if err != nil {
		t.Fatalf("Error deleting data we just put", err)
	}

	checkGetIsEmpty(store, key, t)

}

func TestMultipleOperations(t *testing.T) {

	store := setUp(t)
	defer tearDown(store, t)

	coll := "artist"
	baseKey := "daftpunk"

	var key string
	var expected []byte
	var expectedList [][]byte
	var actual []byte
	for i := int(0); i < 10; i++ {
		key = coll + CollKeySep + baseKey + strconv.Itoa(i)
		expected = generateData(Data{"The peak of awesome" + strconv.Itoa(i)}, t)

		err := store.Put(key, expected)
		if err != nil {
			t.Errorf("Error putting data in", err)
		}
		actual, err = store.Get(key)
		if err != nil {
			t.Errorf("Error getting data back", err)
		}

		if !bytes.Equal(expected, actual) {
			t.Errorf("Expected <%s> but was <%s>",
				expected,
				actual)
		}

		expectedList = append(expectedList, expected)
	}

	actuaList, err := store.GetAll(coll)
	if err != nil {
		t.Fatalf("Error getting all keys with coll <%s>", coll)
	}
	if len(actuaList) != len(expectedList) {
		t.Fatalf("Expected to read %d items, but read %d",
			len(expectedList),
			len(actuaList))
	}

	err = store.DeleteAll("artist")
	if err != nil {
		t.Fatal("Error deleting key", err)
	}
	for i := int(0); i < 10; i++ {
		key = coll + CollKeySep + baseKey + strconv.Itoa(i)
		checkGetIsEmpty(store, key, t)
	}
}

/*
	Test cases, multiple goroutine
*/

func TestMultipleGoroutine(t *testing.T) {
	var kvCount int
	if testing.Short() {
		kvCount = 100
	} else {
		kvCount = 10000
	}
	coll := "games"
	baseKey := "total annihilation #"

	type KeyValue struct {
		Key   string
		Value []byte
	}
	countGet := int64(0)
	countPut := int64(0)
	countDelete := int64(0)
	start := time.Now()

	store := setUp(t)
	defer tearDown(store, t)
	defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(runtime.NumCPU()))

	var writeGroup sync.WaitGroup
	var readGroup sync.WaitGroup
	var deleteGroup sync.WaitGroup
	writeErr := make(chan error)
	readErr := make(chan error)
	deleteErr := make(chan error)

	expectedList := make([]KeyValue, 0)
	for i := int(0); i < kvCount; i++ {
		key := coll + CollKeySep + baseKey + strconv.Itoa(i)
		data := generateData(Data{"It's fun!"}, t)
		kv := KeyValue{key, data}
		expectedList = append(expectedList, kv)
	}

	for _, kv := range expectedList {
		// Put all the values concurrently, ensure there's no error
		writeGroup.Add(1)
		go func(pair KeyValue, cErr chan error) {
			defer writeGroup.Done()
			err := store.Put(pair.Key, pair.Value)
			atomic.AddInt64(&countPut, 1)
			if err != nil {
				t.Fatal("Received an error", err)
				cErr <- err
			}

		}(kv, writeErr)
	}

	writeGroup.Wait()

	if len(writeErr) != 0 {
		t.Fatalf("Failed to write values concurrently, got %d errors",
			len(writeErr))
	}

	for _, kv := range expectedList {
		// Get each value, ensure they're the same
		readGroup.Add(1)
		go func(kv KeyValue, cErr chan error) {
			defer readGroup.Done()

			expected := kv.Value
			actual, err := store.Get(kv.Key)
			atomic.AddInt64(&countGet, 1)
			if err != nil {
				cErr <- err
			} else if !bytes.Equal(expected, actual) {
				t.Errorf("Expected <%s> but was <%s>",
					expected,
					actual)
			}

		}(kv, readErr)

		// GetAll values, ensure this slice is as big as all the value
		// we've put
		readGroup.Add(1)
		go func(cErr chan error) {
			defer readGroup.Done()

			actual, err := store.GetAll(coll)
			atomic.AddInt64(&countGet, int64(len(actual)))
			if err != nil {
				cErr <- err
			} else if len(actual) != len(expectedList) {
				t.Errorf("Expected len(store.GetAll)=<%s> but was <%s>",
					len(expectedList),
					len(actual))
			}
		}(readErr)
	}

	readGroup.Wait()

	if len(readErr) != 0 {
		t.Fatalf("Failed to write values concurrently, got %d errors",
			len(readErr))
	}

	for _, kv := range expectedList {
		// Put all the values concurrently, ensure there's no error
		deleteGroup.Add(1)
		go func(pair KeyValue, cErr chan error) {
			defer deleteGroup.Done()
			err := store.Delete(pair.Key)
			atomic.AddInt64(&countDelete, 1)
			if err != nil {
				t.Fatal("Received an error", err)
				cErr <- err
			}

		}(kv, deleteErr)
	}

	deleteGroup.Wait()

	if len(deleteErr) != 0 {
		t.Fatalf("Failed to write values concurrently, got %d errors",
			len(deleteErr))
	}

	err := store.DeleteAll(coll)
	atomic.AddInt64(&countDelete, int64(len(expectedList)))
	if err != nil {
		t.Fatalf("Error deleting all", err)
	}
	for _, kv := range expectedList {
		atomic.AddInt64(&countGet, 1)
		checkGetIsEmpty(store, kv.Key, t)
	}
	duration := time.Now().Sub(start)
	fmt.Printf("Concurrent test\n")
	fmt.Printf("- Get operations=%d\n", countGet)
	fmt.Printf("- Put operations=%d\n", countPut)
	fmt.Printf("- Delete operations=%d\n", countPut)
	fmt.Printf("in %fs ", duration.Seconds())
	fmt.Printf("by %d cpus, using %d concurrent goroutines\n",
		runtime.NumCPU(), kvCount)

}
<|endoftext|>