<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
)

func genTruncIndex(normalName string) string {
	return normalName[:(len(normalName)+1)/2]
}

func TestTruncIndex(t *testing.T) {
	t.Logf("Pull an image")
	const appImage = "busybox"
	imgID, err := imageService.PullImage(&runtimeapi.ImageSpec{Image: appImage}, nil)
	require.NoError(t, err)
	imgTruncId := genTruncIndex(imgID)
	defer func() {
		assert.NoError(t, imageService.RemoveImage(&runtimeapi.ImageSpec{Image: imgTruncId}))
	}()

	t.Logf("Get image status by truncindex, truncId: %s", imgTruncId)
	res, err := imageService.ImageStatus(&runtimeapi.ImageSpec{Image: imgTruncId})
	require.NoError(t, err)
	require.NotEqual(t, nil, res)
	assert.Equal(t, imgID, res.Id)

	// TODO(yanxuean): for failure test case where there are two images with the same truncindex.
	// if you add n images at least two will share the same leading digit.
	// "sha256:n" where n is the a number from 0-9 where two images have the same trunc,
	// for example sha256:9
	// https://github.com/kubernetes-incubator/cri-containerd/pull/352
	// I am thinking how I get the two image which have same trunc.

	// TODO(yanxuean): add test case for ListImages

	t.Logf("Create a sandbox")
	sbConfig := PodSandboxConfig("sandbox", "truncindex")
	sb, err := runtimeService.RunPodSandbox(sbConfig)
	require.NoError(t, err)
	sbTruncIndex := genTruncIndex(sb)
	var hasStopedSandbox bool
	defer func() {
		// The 2th StopPodSandbox will fail, the 2th RemovePodSandbox will success.
		if !hasStopedSandbox {
			assert.NoError(t, runtimeService.StopPodSandbox(sbTruncIndex))
		}
		assert.NoError(t, runtimeService.RemovePodSandbox(sbTruncIndex))
	}()

	t.Logf("Get sandbox status by truncindex")
	sbStatus, err := runtimeService.PodSandboxStatus(sbTruncIndex)
	require.NoError(t, err)
	assert.Equal(t, sb, sbStatus.Id)

	t.Logf("Forward port for sandbox by truncindex")
	_, err = runtimeService.PortForward(&runtimeapi.PortForwardRequest{sbTruncIndex, []int32{80}})
	assert.NoError(t, err)

	// TODO(yanxuean): add test case for ListPodSandbox

	t.Logf("Create a container")
	cnConfig := ContainerConfig(
		"containerTruncIndex",
		appImage,
		WithCommand("top"),
	)
	cn, err := runtimeService.CreateContainer(sbTruncIndex, cnConfig, sbConfig)
	require.NoError(t, err)
	cnTruncIndex := genTruncIndex(cn)
	defer func() {
		// the 2th RemovePodSandbox will success.
		assert.NoError(t, runtimeService.RemoveContainer(cnTruncIndex))
	}()

	t.Logf("Get container status by truncindex")
	cStatus, err := runtimeService.ContainerStatus(cnTruncIndex)
	require.NoError(t, err)
	assert.Equal(t, cn, cStatus.Id)

	t.Logf("Start the container")
	require.NoError(t, runtimeService.StartContainer(cnTruncIndex))
	var hasStopedContainer bool
	defer func() {
		// The 2th StopPodSandbox will fail
		if !hasStopedContainer {
			assert.NoError(t, runtimeService.StopContainer(cnTruncIndex, 10))
		}
	}()

	t.Logf("Stats the container")
	cStats, err := runtimeService.ContainerStats(cnTruncIndex)
	require.NoError(t, err)
	assert.Equal(t, cn, cStats.Attributes.Id)

	t.Logf("Update container memory limit after started")
	err = runtimeService.UpdateContainerResources(cnTruncIndex, &runtimeapi.LinuxContainerResources{
		MemoryLimitInBytes: 50 * 1024 * 1024,
	})
	assert.NoError(t, err)

	t.Logf("Execute cmd in container")
	execReq := &runtimeapi.ExecRequest{
		ContainerId: cnTruncIndex,
		Cmd:         []string{"pwd"},
		Stdout:      true,
	}
	_, err = runtimeService.Exec(execReq)
	assert.NoError(t, err)

	t.Logf("Execute cmd in container by sync")
	_, _, err = runtimeService.ExecSync(cnTruncIndex, []string{"pwd"}, 10)
	assert.NoError(t, err)

	// TODO(yanxuean): add test case for ListContainers

	t.Logf("Get a non exist container status by truncindex")
	err = runtimeService.StopContainer(cnTruncIndex, 10)
	assert.NoError(t, err)
	if err == nil {
		hasStopedContainer = true
	}
	_, err = runtimeService.ContainerStats(cnTruncIndex)
	assert.Error(t, err)
	assert.NoError(t, runtimeService.RemoveContainer(cnTruncIndex))
	_, err = runtimeService.ContainerStatus(cnTruncIndex)
	assert.Error(t, err)

	t.Logf("Get a non exist sandbox status by truncindex")
	err = runtimeService.StopPodSandbox(sbTruncIndex)
	assert.NoError(t, err)
	if err == nil {
		hasStopedSandbox = true
	}
	assert.NoError(t, runtimeService.RemovePodSandbox(sbTruncIndex))
	_, err = runtimeService.PodSandboxStatus(sbTruncIndex)
	assert.Error(t, err)
}
<commit_msg>Fix typos.<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
)

func genTruncIndex(normalName string) string {
	return normalName[:(len(normalName)+1)/2]
}

func TestTruncIndex(t *testing.T) {
	t.Logf("Pull an image")
	const appImage = "busybox"
	imgID, err := imageService.PullImage(&runtimeapi.ImageSpec{Image: appImage}, nil)
	require.NoError(t, err)
	imgTruncId := genTruncIndex(imgID)
	defer func() {
		assert.NoError(t, imageService.RemoveImage(&runtimeapi.ImageSpec{Image: imgTruncId}))
	}()

	t.Logf("Get image status by truncindex, truncId: %s", imgTruncId)
	res, err := imageService.ImageStatus(&runtimeapi.ImageSpec{Image: imgTruncId})
	require.NoError(t, err)
	require.NotEqual(t, nil, res)
	assert.Equal(t, imgID, res.Id)

	// TODO(yanxuean): for failure test case where there are two images with the same truncindex.
	// if you add n images at least two will share the same leading digit.
	// "sha256:n" where n is the a number from 0-9 where two images have the same trunc,
	// for example sha256:9
	// https://github.com/kubernetes-incubator/cri-containerd/pull/352
	// I am thinking how I get the two image which have same trunc.

	// TODO(yanxuean): add test case for ListImages

	t.Logf("Create a sandbox")
	sbConfig := PodSandboxConfig("sandbox", "truncindex")
	sb, err := runtimeService.RunPodSandbox(sbConfig)
	require.NoError(t, err)
	sbTruncIndex := genTruncIndex(sb)
	var hasStoppedSandbox bool
	defer func() {
		// The 2th StopPodSandbox will fail, the 2th RemovePodSandbox will success.
		if !hasStoppedSandbox {
			assert.NoError(t, runtimeService.StopPodSandbox(sbTruncIndex))
		}
		assert.NoError(t, runtimeService.RemovePodSandbox(sbTruncIndex))
	}()

	t.Logf("Get sandbox status by truncindex")
	sbStatus, err := runtimeService.PodSandboxStatus(sbTruncIndex)
	require.NoError(t, err)
	assert.Equal(t, sb, sbStatus.Id)

	t.Logf("Forward port for sandbox by truncindex")
	_, err = runtimeService.PortForward(&runtimeapi.PortForwardRequest{sbTruncIndex, []int32{80}})
	assert.NoError(t, err)

	// TODO(yanxuean): add test case for ListPodSandbox

	t.Logf("Create a container")
	cnConfig := ContainerConfig(
		"containerTruncIndex",
		appImage,
		WithCommand("top"),
	)
	cn, err := runtimeService.CreateContainer(sbTruncIndex, cnConfig, sbConfig)
	require.NoError(t, err)
	cnTruncIndex := genTruncIndex(cn)
	defer func() {
		// the 2th RemovePodSandbox will success.
		assert.NoError(t, runtimeService.RemoveContainer(cnTruncIndex))
	}()

	t.Logf("Get container status by truncindex")
	cStatus, err := runtimeService.ContainerStatus(cnTruncIndex)
	require.NoError(t, err)
	assert.Equal(t, cn, cStatus.Id)

	t.Logf("Start the container")
	require.NoError(t, runtimeService.StartContainer(cnTruncIndex))
	var hasStoppedContainer bool
	defer func() {
		// The 2th StopPodSandbox will fail
		if !hasStoppedContainer {
			assert.NoError(t, runtimeService.StopContainer(cnTruncIndex, 10))
		}
	}()

	t.Logf("Stats the container")
	cStats, err := runtimeService.ContainerStats(cnTruncIndex)
	require.NoError(t, err)
	assert.Equal(t, cn, cStats.Attributes.Id)

	t.Logf("Update container memory limit after started")
	err = runtimeService.UpdateContainerResources(cnTruncIndex, &runtimeapi.LinuxContainerResources{
		MemoryLimitInBytes: 50 * 1024 * 1024,
	})
	assert.NoError(t, err)

	t.Logf("Execute cmd in container")
	execReq := &runtimeapi.ExecRequest{
		ContainerId: cnTruncIndex,
		Cmd:         []string{"pwd"},
		Stdout:      true,
	}
	_, err = runtimeService.Exec(execReq)
	assert.NoError(t, err)

	t.Logf("Execute cmd in container by sync")
	_, _, err = runtimeService.ExecSync(cnTruncIndex, []string{"pwd"}, 10)
	assert.NoError(t, err)

	// TODO(yanxuean): add test case for ListContainers

	t.Logf("Get a non exist container status by truncindex")
	err = runtimeService.StopContainer(cnTruncIndex, 10)
	assert.NoError(t, err)
	if err == nil {
		hasStoppedContainer = true
	}
	_, err = runtimeService.ContainerStats(cnTruncIndex)
	assert.Error(t, err)
	assert.NoError(t, runtimeService.RemoveContainer(cnTruncIndex))
	_, err = runtimeService.ContainerStatus(cnTruncIndex)
	assert.Error(t, err)

	t.Logf("Get a non exist sandbox status by truncindex")
	err = runtimeService.StopPodSandbox(sbTruncIndex)
	assert.NoError(t, err)
	if err == nil {
		hasStoppedSandbox = true
	}
	assert.NoError(t, runtimeService.RemovePodSandbox(sbTruncIndex))
	_, err = runtimeService.PodSandboxStatus(sbTruncIndex)
	assert.Error(t, err)
}
<|endoftext|>


<commit_before>package pow

import (
	"fmt"
	"math/big"
	"sync"
	"testing"

	"github.com/ericlagergren/decimal/internal/c"
)

func TestBigTen(t *testing.T) {
	errc := make(chan error)

	var wg sync.WaitGroup
	for i := uint64(0); i < BigTabLen+10; i++ {
		wg.Add(1)
		go func(i uint64) {
			comp := BigTen(i)
			n := new(big.Int).SetUint64(i)
			act := n.Exp(c.TenInt, n, nil)
			if act.Cmp(comp) != 0 {
				cs, as := comp.String(), act.String()
				errc <- fmt.Errorf(`%d:
got   : (%d) %s
wanted: (%d) %s
`, i, len(cs), cs, len(as), as)
			}
			wg.Done()
		}(i)
	}

	donec := make(chan struct{})
	go func() {
		wg.Wait()
		donec <- struct{}{}
	}()

	for {
		select {
		case err := <-errc:
			t.Fatal(err.Error())
		case <-donec:
			return
		}
	}
}
<commit_msg>internal/arith/pow: limit number of workers to 1000 so we don't hit race's limit of 8192 goroutines<commit_after>package pow

import (
	"fmt"
	"math/big"
	"sync"
	"testing"

	"github.com/ericlagergren/decimal/internal/c"
)

func TestBigTen(t *testing.T) {
	errc := make(chan error)
	workc := make(chan struct{}, 1000)
	for i := 0; i < 1000; i++ {
		workc <- struct{}{}
	}

	var wg sync.WaitGroup
	for i := uint64(0); i < BigTabLen+10; i++ {
		<-workc
		wg.Add(1)
		go func(i uint64) {
			comp := BigTen(i)
			n := new(big.Int).SetUint64(i)
			act := n.Exp(c.TenInt, n, nil)
			if act.Cmp(comp) != 0 {
				cs, as := comp.String(), act.String()
				errc <- fmt.Errorf(`%d:
got   : (%d) %s
wanted: (%d) %s
`, i, len(cs), cs, len(as), as)
			}
			wg.Done()
			workc <- struct{}{}
		}(i)
	}

	donec := make(chan struct{})
	go func() {
		wg.Wait()
		donec <- struct{}{}
	}()

	for {
		select {
		case err := <-errc:
			t.Fatal(err.Error())
		case <-donec:
			return
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2018 The Go Cloud Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// contributebot is a service for keeping the Go Cloud project tidy.
package main

import (
	"context"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"sync"
	"time"

	"cloud.google.com/go/pubsub"
	"github.com/google/go-github/github"
)

const userAgent = "google/go-cloud Contribute Bot"

type flagConfig struct {
	project      string
	subscription string
	gitHubAppID  int64
	keyPath      string
}

func main() {
	addr := flag.String("listen", ":8080", "address to listen for health checks")
	var cfg flagConfig
	flag.StringVar(&cfg.project, "project", "", "GCP project for topic")
	flag.StringVar(&cfg.subscription, "subscription", "contributebot-github-events", "subscription name inside project")
	flag.Int64Var(&cfg.gitHubAppID, "github_app", 0, "GitHub application ID")
	flag.StringVar(&cfg.keyPath, "github_key", "", "path to GitHub application private key")
	flag.Parse()
	if cfg.project == "" || cfg.gitHubAppID == 0 || cfg.keyPath == "" {
		fmt.Fprintln(os.Stderr, "contributebot: must specify -project, -github_app, and -github_key")
		os.Exit(2)
	}

	ctx := context.Background()
	w, server, cleanup, err := setup(ctx, cfg)
	if err != nil {
		log.Fatal(err)
	}
	defer cleanup()
	log.Printf("Serving health checks at %s", *addr)
	go server.ListenAndServe(*addr, w)
	log.Fatal(w.receive(ctx))
}

// worker contains the connections used by this server.
type worker struct {
	sub  *pubsub.Subscription
	auth *gitHubAppAuth

	mu          sync.Mutex
	configCache map[repoKey]*repoConfigCacheEntry
}

func newWorker(sub *pubsub.Subscription, auth *gitHubAppAuth) *worker {
	return &worker{
		sub:         sub,
		auth:        auth,
		configCache: make(map[repoKey]*repoConfigCacheEntry),
	}
}

const configCacheTTL = 2 * time.Minute

type repoKey struct {
	owner string
	repo  string
}

type repoConfigCacheEntry struct {
	// On initial placement in the cache, ready will be an open channel.
	// It will be closed once the entry has been fetched and the other
	// fields are thus safe to read.
	ready <-chan struct{}

	config  repoConfig
	err     error
	fetched time.Time
}

// receive listens for events on its subscription and handles them.
func (w *worker) receive(ctx context.Context) error {
	return w.sub.Receive(ctx, func(ctx context.Context, msg *pubsub.Message) {
		id := msg.Attributes["X-GitHub-Delivery"]
		eventType := msg.Attributes["X-GitHub-Event"]
		if eventType == "integration_installation" || eventType == "integration_installation_repositories" {
			// Deprecated event types. Ignore them in favor of supported ones.
			log.Printf("Skipped event %s of deprecated type %s", id, eventType)
			msg.Ack()
			return
		}
		event, err := github.ParseWebHook(eventType, msg.Data)
		if err != nil {
			log.Printf("Parsing %s event %s: %v", eventType, id, err)
			msg.Nack()
			return
		}
		var handleErr error
		switch event := event.(type) {
		case *github.IssuesEvent:
			handleErr = w.receiveIssueEvent(ctx, event)
		case *github.PullRequestEvent:
			handleErr = w.receivePullRequestEvent(ctx, event)
		case *github.PingEvent, *github.InstallationEvent, *github.CheckSuiteEvent:
			// No-op.
		default:
			log.Printf("Unhandled webhook event type %s (%T) for %s", eventType, event, id)
			msg.Nack()
			return
		}
		if handleErr != nil {
			log.Printf("Failed processing %s event %s: %v", eventType, id, handleErr)
			msg.Nack()
			return
		}
		msg.Ack()
		log.Printf("Processed %s event %s", eventType, id)
	})
}

func (w *worker) receiveIssueEvent(ctx context.Context, e *github.IssuesEvent) error {
	client := w.ghClient(e.GetInstallation().GetID())

	// Pull out the interesting data from the event.
	data := &issueData{
		Action: e.GetAction(),
		Owner:  e.GetRepo().GetOwner().GetLogin(),
		Repo:   e.GetRepo().GetName(),
		Issue:  e.GetIssue(),
		Change: e.GetChanges(),
	}

	// Fetch repository configuration.
	cfg, err := w.repoConfig(ctx, client, data.Owner, data.Repo)
	if err != nil {
		return err
	}

	// Refetch the issue in case the event data is stale.
	iss, _, err := client.Issues.Get(ctx, data.Owner, data.Repo, data.Issue.GetNumber())
	if err != nil {
		return err
	}
	data.Issue = iss

	// Process the issue, deciding what actions to take (if any).
	edits := processIssueEvent(cfg, data)
	// Execute the actions (if any).
	return edits.Execute(ctx, client, data)
}

func (w *worker) receivePullRequestEvent(ctx context.Context, e *github.PullRequestEvent) error {
	client := w.ghClient(e.GetInstallation().GetID())

	// Pull out the interesting data from the event.
	data := &pullRequestData{
		Action:      e.GetAction(),
		OwnerLogin:  e.GetRepo().GetOwner().GetLogin(),
		Repo:        e.GetRepo().GetName(),
		PullRequest: e.GetPullRequest(),
		Change:      e.GetChanges(),
	}

	// Fetch repository configuration.
	cfg, err := w.repoConfig(ctx, client, data.OwnerLogin, data.Repo)
	if err != nil {
		return err
	}

	// Refetch the pull request in case the event data is stale.
	pr, _, err := client.PullRequests.Get(ctx, data.OwnerLogin, data.Repo, data.PullRequest.GetNumber())
	if err != nil {
		return err
	}
	data.PullRequest = pr

	// Process the pull request, deciding what actions to take (if any).
	edits := processPullRequestEvent(cfg, data)
	// Execute the actions (if any).
	return edits.Execute(ctx, client, data)
}

// repoConfig fetches the parsed Contribute Bot configuration for the given
// repository. The result may be cached.
func (w *worker) repoConfig(ctx context.Context, client *github.Client, owner, repo string) (_ *repoConfig, err error) {
	cacheKey := repoKey{owner, repo}
	queryTime := time.Now()
	w.mu.Lock()
	ent := w.configCache[cacheKey]
	if ent != nil {
		select {
		case <-ent.ready:
			// Entry has been fully written. Check if we can use its results.
			if ent.err == nil && queryTime.Sub(ent.fetched) < configCacheTTL {
				// Cache hit; no processing necessary.
				w.mu.Unlock()
				return &ent.config, nil
			}
		default:
			// Another goroutine is currently retrieving the configuration. Block on that result.
			w.mu.Unlock()
			select {
			case <-ent.ready:
				if ent.err != nil {
					return nil, ent.err
				}
				return &ent.config, nil
			case <-ctx.Done():
				return nil, fmt.Errorf("read repository %s/%s config: %v", owner, repo, ctx.Err())
			}
		}
	}
	// Cache miss. Reserve the fetch work.
	done := make(chan struct{})
	ent = &repoConfigCacheEntry{
		ready:  done,
		config: *defaultRepoConfig(),
	}
	w.configCache[cacheKey] = ent
	w.mu.Unlock()
	defer func() {
		ent.fetched = time.Now()
		ent.err = err // err is the named return value.
		close(done)
	}()

	// Fetch the configuration from the repository.
	content, _, response, err := client.Repositories.GetContents(ctx, owner, repo, ".contributebot", nil)
	if response != nil && response.StatusCode == http.StatusNotFound {
		// File not found. Use default configuration.
		return &ent.config, nil
	}
	if err != nil {
		return nil, fmt.Errorf("read repository %s/%s config: %v", owner, repo, err)
	}
	data, err := content.GetContent()
	if err != nil {
		return nil, fmt.Errorf("read repository %s/%s config: %v", owner, repo, err)
	}
	if err := json.Unmarshal([]byte(data), &ent.config); err != nil {
		return nil, fmt.Errorf("read repository %s/%s config: %v", owner, repo, err)
	}
	return &ent.config, nil
}

// isClosed tests whether a channel is closed without blocking.
func isClosed(c <-chan struct{}) bool {
	select {
	case _, ok := <-c:
		return !ok
	default:
		return false
	}
}

// ghClient creates a GitHub client authenticated for the given installation.
func (w *worker) ghClient(installID int64) *github.Client {
	c := github.NewClient(&http.Client{Transport: w.auth.forInstall(installID)})
	c.UserAgent = userAgent
	return c
}

// ServeHTTP serves a page explaining that this port is only open for health checks.
func (w *worker) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
	if req.URL.Path != "/" {
		http.NotFound(resp, req)
		return
	}
	const responseData = `<!DOCTYPE html>
<title>Go Cloud Contribute Bot Worker</title>

Go Cloud Contribute Bot Worker

This HTTP port is only open to serve health checks.
`
	resp.Header().Set("Content-Length", fmt.Sprint(len(responseData)))
	resp.Header().Set("Content-Type", "text/html; charset=utf-8")
	io.WriteString(resp, responseData)
}

func (w *worker) CheckHealth() error {
	return w.auth.CheckHealth()
}
<commit_msg>internal/contributebot: ignore check run and push events (#995)<commit_after>// Copyright 2018 The Go Cloud Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// contributebot is a service for keeping the Go Cloud project tidy.
package main

import (
	"context"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"sync"
	"time"

	"cloud.google.com/go/pubsub"
	"github.com/google/go-github/github"
)

const userAgent = "google/go-cloud Contribute Bot"

type flagConfig struct {
	project      string
	subscription string
	gitHubAppID  int64
	keyPath      string
}

func main() {
	addr := flag.String("listen", ":8080", "address to listen for health checks")
	var cfg flagConfig
	flag.StringVar(&cfg.project, "project", "", "GCP project for topic")
	flag.StringVar(&cfg.subscription, "subscription", "contributebot-github-events", "subscription name inside project")
	flag.Int64Var(&cfg.gitHubAppID, "github_app", 0, "GitHub application ID")
	flag.StringVar(&cfg.keyPath, "github_key", "", "path to GitHub application private key")
	flag.Parse()
	if cfg.project == "" || cfg.gitHubAppID == 0 || cfg.keyPath == "" {
		fmt.Fprintln(os.Stderr, "contributebot: must specify -project, -github_app, and -github_key")
		os.Exit(2)
	}

	ctx := context.Background()
	w, server, cleanup, err := setup(ctx, cfg)
	if err != nil {
		log.Fatal(err)
	}
	defer cleanup()
	log.Printf("Serving health checks at %s", *addr)
	go server.ListenAndServe(*addr, w)
	log.Fatal(w.receive(ctx))
}

// worker contains the connections used by this server.
type worker struct {
	sub  *pubsub.Subscription
	auth *gitHubAppAuth

	mu          sync.Mutex
	configCache map[repoKey]*repoConfigCacheEntry
}

func newWorker(sub *pubsub.Subscription, auth *gitHubAppAuth) *worker {
	return &worker{
		sub:         sub,
		auth:        auth,
		configCache: make(map[repoKey]*repoConfigCacheEntry),
	}
}

const configCacheTTL = 2 * time.Minute

type repoKey struct {
	owner string
	repo  string
}

type repoConfigCacheEntry struct {
	// On initial placement in the cache, ready will be an open channel.
	// It will be closed once the entry has been fetched and the other
	// fields are thus safe to read.
	ready <-chan struct{}

	config  repoConfig
	err     error
	fetched time.Time
}

// receive listens for events on its subscription and handles them.
func (w *worker) receive(ctx context.Context) error {
	return w.sub.Receive(ctx, func(ctx context.Context, msg *pubsub.Message) {
		id := msg.Attributes["X-GitHub-Delivery"]
		eventType := msg.Attributes["X-GitHub-Event"]
		if eventType == "integration_installation" || eventType == "integration_installation_repositories" {
			// Deprecated event types. Ignore them in favor of supported ones.
			log.Printf("Skipped event %s of deprecated type %s", id, eventType)
			msg.Ack()
			return
		}
		event, err := github.ParseWebHook(eventType, msg.Data)
		if err != nil {
			log.Printf("Parsing %s event %s: %v", eventType, id, err)
			msg.Nack()
			return
		}
		var handleErr error
		switch event := event.(type) {
		case *github.IssuesEvent:
			handleErr = w.receiveIssueEvent(ctx, event)
		case *github.PullRequestEvent:
			handleErr = w.receivePullRequestEvent(ctx, event)
		case *github.PingEvent, *github.InstallationEvent, *github.CheckRunEvent, *github.CheckSuiteEvent, *github.PushEvent:
			// No-op.
		default:
			log.Printf("Unhandled webhook event type %s (%T) for %s", eventType, event, id)
			msg.Nack()
			return
		}
		if handleErr != nil {
			log.Printf("Failed processing %s event %s: %v", eventType, id, handleErr)
			msg.Nack()
			return
		}
		msg.Ack()
		log.Printf("Processed %s event %s", eventType, id)
	})
}

func (w *worker) receiveIssueEvent(ctx context.Context, e *github.IssuesEvent) error {
	client := w.ghClient(e.GetInstallation().GetID())

	// Pull out the interesting data from the event.
	data := &issueData{
		Action: e.GetAction(),
		Owner:  e.GetRepo().GetOwner().GetLogin(),
		Repo:   e.GetRepo().GetName(),
		Issue:  e.GetIssue(),
		Change: e.GetChanges(),
	}

	// Fetch repository configuration.
	cfg, err := w.repoConfig(ctx, client, data.Owner, data.Repo)
	if err != nil {
		return err
	}

	// Refetch the issue in case the event data is stale.
	iss, _, err := client.Issues.Get(ctx, data.Owner, data.Repo, data.Issue.GetNumber())
	if err != nil {
		return err
	}
	data.Issue = iss

	// Process the issue, deciding what actions to take (if any).
	edits := processIssueEvent(cfg, data)
	// Execute the actions (if any).
	return edits.Execute(ctx, client, data)
}

func (w *worker) receivePullRequestEvent(ctx context.Context, e *github.PullRequestEvent) error {
	client := w.ghClient(e.GetInstallation().GetID())

	// Pull out the interesting data from the event.
	data := &pullRequestData{
		Action:      e.GetAction(),
		OwnerLogin:  e.GetRepo().GetOwner().GetLogin(),
		Repo:        e.GetRepo().GetName(),
		PullRequest: e.GetPullRequest(),
		Change:      e.GetChanges(),
	}

	// Fetch repository configuration.
	cfg, err := w.repoConfig(ctx, client, data.OwnerLogin, data.Repo)
	if err != nil {
		return err
	}

	// Refetch the pull request in case the event data is stale.
	pr, _, err := client.PullRequests.Get(ctx, data.OwnerLogin, data.Repo, data.PullRequest.GetNumber())
	if err != nil {
		return err
	}
	data.PullRequest = pr

	// Process the pull request, deciding what actions to take (if any).
	edits := processPullRequestEvent(cfg, data)
	// Execute the actions (if any).
	return edits.Execute(ctx, client, data)
}

// repoConfig fetches the parsed Contribute Bot configuration for the given
// repository. The result may be cached.
func (w *worker) repoConfig(ctx context.Context, client *github.Client, owner, repo string) (_ *repoConfig, err error) {
	cacheKey := repoKey{owner, repo}
	queryTime := time.Now()
	w.mu.Lock()
	ent := w.configCache[cacheKey]
	if ent != nil {
		select {
		case <-ent.ready:
			// Entry has been fully written. Check if we can use its results.
			if ent.err == nil && queryTime.Sub(ent.fetched) < configCacheTTL {
				// Cache hit; no processing necessary.
				w.mu.Unlock()
				return &ent.config, nil
			}
		default:
			// Another goroutine is currently retrieving the configuration. Block on that result.
			w.mu.Unlock()
			select {
			case <-ent.ready:
				if ent.err != nil {
					return nil, ent.err
				}
				return &ent.config, nil
			case <-ctx.Done():
				return nil, fmt.Errorf("read repository %s/%s config: %v", owner, repo, ctx.Err())
			}
		}
	}
	// Cache miss. Reserve the fetch work.
	done := make(chan struct{})
	ent = &repoConfigCacheEntry{
		ready:  done,
		config: *defaultRepoConfig(),
	}
	w.configCache[cacheKey] = ent
	w.mu.Unlock()
	defer func() {
		ent.fetched = time.Now()
		ent.err = err // err is the named return value.
		close(done)
	}()

	// Fetch the configuration from the repository.
	content, _, response, err := client.Repositories.GetContents(ctx, owner, repo, ".contributebot", nil)
	if response != nil && response.StatusCode == http.StatusNotFound {
		// File not found. Use default configuration.
		return &ent.config, nil
	}
	if err != nil {
		return nil, fmt.Errorf("read repository %s/%s config: %v", owner, repo, err)
	}
	data, err := content.GetContent()
	if err != nil {
		return nil, fmt.Errorf("read repository %s/%s config: %v", owner, repo, err)
	}
	if err := json.Unmarshal([]byte(data), &ent.config); err != nil {
		return nil, fmt.Errorf("read repository %s/%s config: %v", owner, repo, err)
	}
	return &ent.config, nil
}

// isClosed tests whether a channel is closed without blocking.
func isClosed(c <-chan struct{}) bool {
	select {
	case _, ok := <-c:
		return !ok
	default:
		return false
	}
}

// ghClient creates a GitHub client authenticated for the given installation.
func (w *worker) ghClient(installID int64) *github.Client {
	c := github.NewClient(&http.Client{Transport: w.auth.forInstall(installID)})
	c.UserAgent = userAgent
	return c
}

// ServeHTTP serves a page explaining that this port is only open for health checks.
func (w *worker) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
	if req.URL.Path != "/" {
		http.NotFound(resp, req)
		return
	}
	const responseData = `<!DOCTYPE html>
<title>Go Cloud Contribute Bot Worker</title>

Go Cloud Contribute Bot Worker

This HTTP port is only open to serve health checks.
`
	resp.Header().Set("Content-Length", fmt.Sprint(len(responseData)))
	resp.Header().Set("Content-Type", "text/html; charset=utf-8")
	io.WriteString(resp, responseData)
}

func (w *worker) CheckHealth() error {
	return w.auth.CheckHealth()
}
<|endoftext|>


<commit_before>package mount

import (
	"context"
	"net"
	"net/http"
	"os"

	"github.com/pkg/errors"
	"golang.org/x/net/webdav"

	"github.com/kopia/kopia/fs"
	"github.com/kopia/kopia/internal/webdavmount"
)

func webdavServerLogger(r *http.Request, err error) {
	var maybeRange string
	if r := r.Header.Get("Range"); r != "" {
		maybeRange = " " + r
	}

	if err != nil {
		log(r.Context()).Warningf("%v %v%v err: %v\n", r.Method, r.URL.RequestURI(), maybeRange, err)
	} else {
		log(r.Context()).Debugf("%v %v%v OK\n", r.Method, r.URL.RequestURI(), maybeRange)
	}
}

// DirectoryWebDAV exposes the provided filesystem directory via WebDAV on a random port on localhost
// and returns a controller.
func DirectoryWebDAV(ctx context.Context, entry fs.Directory) (Controller, error) {
	log(ctx).Debugf("creating webdav server...")

	mux := http.NewServeMux()

	var logger func(r *http.Request, err error)

	if os.Getenv("WEBDAV_LOG_REQUESTS") != "" {
		logger = webdavServerLogger
	}

	mux.Handle("/", &webdav.Handler{
		FileSystem: webdavmount.WebDAVFS(entry),
		LockSystem: webdav.NewMemLS(),
		Logger:     logger,
	})

	l, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		return nil, errors.Wrap(err, "listen error")
	}

	srv := &http.Server{
		Handler: mux,
	}

	done := make(chan struct{})

	go func() {
		log(ctx).Debugf("web server finished with %v", srv.Serve(l))
	}()

	return webdavController{"http://" + l.Addr().String(), srv, done}, nil
}

type webdavController struct {
	webdavURL string
	s         *http.Server
	done      chan struct{}
}

func (c webdavController) Unmount(ctx context.Context) error {
	return c.s.Shutdown(ctx)
}

func (c webdavController) MountPath() string {
	return c.webdavURL
}

func (c webdavController) Done() <-chan struct{} {
	return c.done
}
<commit_msg>Fix hang at unmount when Ctrl-C is pressed (#754)<commit_after>package mount

import (
	"context"
	"net"
	"net/http"
	"os"

	"github.com/pkg/errors"
	"golang.org/x/net/webdav"

	"github.com/kopia/kopia/fs"
	"github.com/kopia/kopia/internal/webdavmount"
)

func webdavServerLogger(r *http.Request, err error) {
	var maybeRange string
	if r := r.Header.Get("Range"); r != "" {
		maybeRange = " " + r
	}

	if err != nil {
		log(r.Context()).Warningf("%v %v%v err: %v\n", r.Method, r.URL.RequestURI(), maybeRange, err)
	} else {
		log(r.Context()).Debugf("%v %v%v OK\n", r.Method, r.URL.RequestURI(), maybeRange)
	}
}

// DirectoryWebDAV exposes the provided filesystem directory via WebDAV on a random port on localhost
// and returns a controller.
func DirectoryWebDAV(ctx context.Context, entry fs.Directory) (Controller, error) {
	log(ctx).Debugf("creating webdav server...")

	mux := http.NewServeMux()

	var logger func(r *http.Request, err error)

	if os.Getenv("WEBDAV_LOG_REQUESTS") != "" {
		logger = webdavServerLogger
	}

	mux.Handle("/", &webdav.Handler{
		FileSystem: webdavmount.WebDAVFS(entry),
		LockSystem: webdav.NewMemLS(),
		Logger:     logger,
	})

	l, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		return nil, errors.Wrap(err, "listen error")
	}

	srv := &http.Server{
		Handler: mux,
	}

	done := make(chan struct{})

	srv.RegisterOnShutdown(func() {
		close(done)
	})

	go func() {
		log(ctx).Debugf("web server finished with %v", srv.Serve(l))
	}()

	return webdavController{"http://" + l.Addr().String(), srv, done}, nil
}

type webdavController struct {
	webdavURL string
	s         *http.Server
	done      chan struct{}
}

func (c webdavController) Unmount(ctx context.Context) error {
	return c.s.Shutdown(ctx)
}

func (c webdavController) MountPath() string {
	return c.webdavURL
}

func (c webdavController) Done() <-chan struct{} {
	return c.done
}
<|endoftext|>


<commit_before>package paillier

import (
	"crypto/rand"
	"math/big"
	"reflect"
	"testing"
)

var MockGenerateSafePrimes = func() (*big.Int, *big.Int, error) {
	return big.NewInt(887), big.NewInt(443), nil
}

func TestGenerateSafePrimesOfThresholdKeyGenerator(t *testing.T) {
	tkh := new(ThresholdKeyGenerator)
	tkh.publicKeyBitLength = 32
	tkh.Random = rand.Reader
	p, q, err := tkh.generateSafePrimes()
	if err != nil {
		t.Error(err)
	}
	AreSafePrimes(p, q, 16, t)
}

func TestInitPandP1(t *testing.T) {
	tkh := new(ThresholdKeyGenerator)
	tkh.publicKeyBitLength = 32
	tkh.Random = rand.Reader

	tkh.initPandP1()
	AreSafePrimes(tkh.p, tkh.p1, 16, t)
}

func TestInitQandQ1(t *testing.T) {
	tkh := new(ThresholdKeyGenerator)
	tkh.publicKeyBitLength = 32
	tkh.Random = rand.Reader

	tkh.initQandQ1()
	AreSafePrimes(tkh.q, tkh.q1, 16, t)
}

func TestInitPsAndQs(t *testing.T) {
	tkh := new(ThresholdKeyGenerator)
	tkh.publicKeyBitLength = 32
	tkh.Random = rand.Reader

	tkh.initPsAndQs()

	AreSafePrimes(tkh.p, tkh.p1, 16, t)
	AreSafePrimes(tkh.q, tkh.q1, 16, t)
}

func TestArePsAndQsGood(t *testing.T) {
	tkh := new(ThresholdKeyGenerator)
	tkh.p, tkh.p1, tkh.q, tkh.q1 = b(887), b(443), b(839), b(419)
	if !tkh.arePsAndQsGood() {
		t.Fail()
	}

	tkh.p, tkh.p1, tkh.q, tkh.q1 = b(887), b(443), b(887), b(443)
	if tkh.arePsAndQsGood() {
		t.Fail()
	}

	tkh.p, tkh.p1, tkh.q, tkh.q1 = b(887), b(443), b(443), b(221)
	if tkh.arePsAndQsGood() {
		t.Fail()
	}
}

func TestInitShortcuts(t *testing.T) {
	tkh := new(ThresholdKeyGenerator)
	tkh.p, tkh.p1, tkh.q, tkh.q1 = b(839), b(419), b(887), b(443)
	tkh.initShortcuts()

	if !reflect.DeepEqual(tkh.n, b(744193)) {
		t.Error("wrong n", tkh.n)
	}
	if !reflect.DeepEqual(tkh.m, b(185617)) {
		t.Error("wrong m", tkh.m)
	}
	if !reflect.DeepEqual(tkh.nm, new(big.Int).Mul(b(744193), b(185617))) {
		t.Error("wrong nm", tkh.nm)
	}
	if !reflect.DeepEqual(tkh.nSquare, new(big.Int).Mul(b(744193), b(744193))) {
		t.Error("wrong nSquare", tkh.nSquare)
	}
}

func TestInitD(t *testing.T) {
	tkh := new(ThresholdKeyGenerator)
	tkh.p, tkh.p1, tkh.q, tkh.q1 = b(863), b(431), b(839), b(419)
	tkh.initShortcuts()
	tkh.initD()
	if n(tkh.d)%n(tkh.m) != 0 {
		t.Fail()
	}
	if n(tkh.d)%n(tkh.n) != 1 {
		t.Fail()
	}
}

func TestInitNumerialValues(t *testing.T) {
	tkh := new(ThresholdKeyGenerator)
	tkh.publicKeyBitLength = 32
	tkh.Random = rand.Reader

	if err := tkh.initNumerialValues(); err != nil {
		t.Error(err)
	}
}

func TestGenerateHidingPolynomial(t *testing.T) {
	tkh := new(ThresholdKeyGenerator)
	tkh.publicKeyBitLength = 32
	tkh.Threshold = 10
	tkh.Random = rand.Reader
	if err := tkh.initNumerialValues(); err != nil {
		t.Error(err)
	}
	if err := tkh.generateHidingPolynomial(); err != nil {
		t.Error(err)
	}
	p := tkh.polynomialCoefficients
	if len(p) != tkh.Threshold {
		t.Fail()
	}
	if n(p[0]) != n(tkh.d) {
		t.Fail()
	}
	for i := 1; i < len(p); i++ {
		if j := n(p[i]); j < 0 || j >= n(tkh.nm) {
			t.Fail()
		}
	}
}

func TestComputeShare(t *testing.T) {
	tkh := new(ThresholdKeyGenerator)
	tkh.publicKeyBitLength = 32
	tkh.Threshold = 3
	tkh.TotalNumberOfDecryptionServers = 5
	tkh.nm = b(103)
	tkh.polynomialCoefficients = []*big.Int{b(29), b(88), b(51)}
	share := tkh.computeShare(2)
	if n(share) != 31 {
		t.Error("error computing a share.  ", share)
	}
}

func TestCreateShares(t *testing.T) {
	tkh := new(ThresholdKeyGenerator)
	tkh.publicKeyBitLength = 32
	tkh.Threshold = 10
	tkh.TotalNumberOfDecryptionServers = 100
	tkh.Random = rand.Reader
	if err := tkh.initNumerialValues(); err != nil {
		t.Error(err)
	}
	if err := tkh.generateHidingPolynomial(); err != nil {
		t.Error(err)
	}

	if shares := tkh.createShares(); len(shares) != 100 {
		t.Fail()
	}
}

func TestCreateViArray(t *testing.T) {
	tkh := new(ThresholdKeyGenerator)
	tkh.TotalNumberOfDecryptionServers = 10
	tkh.v = b(54)
	tkh.nSquare = b(101 * 101)
	vArr := tkh.createViArray([]*big.Int{b(12), b(90), b(103)})
	exp := []*big.Int{b(6162), b(304), b(2728)}
	if !reflect.DeepEqual(vArr, exp) {
		t.Fail()
	}
}

func TestGetThresholdKeyGenerator(t *testing.T) {
	tkh := GetThresholdKeyGenerator(50, 10, 6, rand.Reader)
	if err := tkh.initNumerialValues(); err != nil {
		t.Error(nil)
	}
}

func TestGenerate(t *testing.T) {
	tkh := GetThresholdKeyGenerator(32, 10, 6, rand.Reader)
	tpks, err := tkh.Generate()
	if err != nil {
		t.Error(err)
		return
	}
	if len(tpks) != 10 {
		t.Fail()
	}
	for i, tpk := range tpks {
		if tpk.Id != i+1 {
			t.Fail()
		}
		if len(tpk.Vi) != 10 {
			t.Fail()
		}
		if tpk.N == nil {
			t.Fail()
		}
		if tpk.Threshold != 6 || tpk.TotalNumberOfDecryptionServers != 10 {
			t.Fail()
		}
	}
}

func TestComputeV(t *testing.T) {
	tkh := GetThresholdKeyGenerator(32, 10, 6, rand.Reader)
	tkh.n = b(1907 * 1823)
	tkh.nSquare = new(big.Int).Mul(tkh.n, tkh.n)
	for i := 0; i < 100; i++ {
		if err := tkh.computeV(); err != nil {
			t.Error(err)
		}
		if tkh.v.Cmp(tkh.nSquare) > 0 {
			t.Error("v is too big")
		}
		if tkh.v.Cmp(tkh.n) > 0 {
			return
		}
	}
	t.Error(`v has never been bigger than n.  It is suspicious in the sense<
	than it was taken in the range 0...n**2 -1
	`)
}
<commit_msg>Use GenThresholdKeyGenerator whenever it's possible<commit_after>package paillier

import (
	"crypto/rand"
	"math/big"
	"reflect"
	"testing"
)

var MockGenerateSafePrimes = func() (*big.Int, *big.Int, error) {
	return big.NewInt(887), big.NewInt(443), nil
}

func TestGenerateSafePrimesOfThresholdKeyGenerator(t *testing.T) {
	tkh := GetThresholdKeyGenerator(32, 4, 3, rand.Reader)

	p, q, err := tkh.generateSafePrimes()
	if err != nil {
		t.Error(err)
	}
	AreSafePrimes(p, q, 16, t)
}

func TestInitPandP1(t *testing.T) {
	tkh := GetThresholdKeyGenerator(32, 4, 3, rand.Reader)

	tkh.initPandP1()
	AreSafePrimes(tkh.p, tkh.p1, 16, t)
}

func TestInitQandQ1(t *testing.T) {
	tkh := GetThresholdKeyGenerator(32, 4, 3, rand.Reader)

	tkh.initQandQ1()
	AreSafePrimes(tkh.q, tkh.q1, 16, t)
}

func TestInitPsAndQs(t *testing.T) {
	tkh := GetThresholdKeyGenerator(32, 4, 3, rand.Reader)

	tkh.initPsAndQs()

	AreSafePrimes(tkh.p, tkh.p1, 16, t)
	AreSafePrimes(tkh.q, tkh.q1, 16, t)
}

func TestArePsAndQsGood(t *testing.T) {
	tkh := new(ThresholdKeyGenerator)
	tkh.p, tkh.p1, tkh.q, tkh.q1 = b(887), b(443), b(839), b(419)
	if !tkh.arePsAndQsGood() {
		t.Fail()
	}

	tkh.p, tkh.p1, tkh.q, tkh.q1 = b(887), b(443), b(887), b(443)
	if tkh.arePsAndQsGood() {
		t.Fail()
	}

	tkh.p, tkh.p1, tkh.q, tkh.q1 = b(887), b(443), b(443), b(221)
	if tkh.arePsAndQsGood() {
		t.Fail()
	}
}

func TestInitShortcuts(t *testing.T) {
	tkh := new(ThresholdKeyGenerator)
	tkh.p, tkh.p1, tkh.q, tkh.q1 = b(839), b(419), b(887), b(443)
	tkh.initShortcuts()

	if !reflect.DeepEqual(tkh.n, b(744193)) {
		t.Error("wrong n", tkh.n)
	}
	if !reflect.DeepEqual(tkh.m, b(185617)) {
		t.Error("wrong m", tkh.m)
	}
	if !reflect.DeepEqual(tkh.nm, new(big.Int).Mul(b(744193), b(185617))) {
		t.Error("wrong nm", tkh.nm)
	}
	if !reflect.DeepEqual(tkh.nSquare, new(big.Int).Mul(b(744193), b(744193))) {
		t.Error("wrong nSquare", tkh.nSquare)
	}
}

func TestInitD(t *testing.T) {
	tkh := new(ThresholdKeyGenerator)
	tkh.p, tkh.p1, tkh.q, tkh.q1 = b(863), b(431), b(839), b(419)
	tkh.initShortcuts()
	tkh.initD()
	if n(tkh.d)%n(tkh.m) != 0 {
		t.Fail()
	}
	if n(tkh.d)%n(tkh.n) != 1 {
		t.Fail()
	}
}

func TestInitNumerialValues(t *testing.T) {
	tkh := GetThresholdKeyGenerator(32, 4, 3, rand.Reader)

	if err := tkh.initNumerialValues(); err != nil {
		t.Error(err)
	}
}

func TestGenerateHidingPolynomial(t *testing.T) {
	tkh := GetThresholdKeyGenerator(32, 15, 10, rand.Reader)
	if err := tkh.initNumerialValues(); err != nil {
		t.Error(err)
	}
	if err := tkh.generateHidingPolynomial(); err != nil {
		t.Error(err)
	}
	p := tkh.polynomialCoefficients
	if len(p) != tkh.Threshold {
		t.Fail()
	}
	if n(p[0]) != n(tkh.d) {
		t.Fail()
	}
	for i := 1; i < len(p); i++ {
		if j := n(p[i]); j < 0 || j >= n(tkh.nm) {
			t.Fail()
		}
	}
}

func TestComputeShare(t *testing.T) {
	tkh := GetThresholdKeyGenerator(32, 5, 3, rand.Reader)
	tkh.nm = b(103)
	tkh.polynomialCoefficients = []*big.Int{b(29), b(88), b(51)}
	share := tkh.computeShare(2)
	if n(share) != 31 {
		t.Error("error computing a share.  ", share)
	}
}

func TestCreateShares(t *testing.T) {
	tkh := GetThresholdKeyGenerator(32, 100, 10, rand.Reader)
	tkh.Random = rand.Reader
	if err := tkh.initNumerialValues(); err != nil {
		t.Error(err)
	}
	if err := tkh.generateHidingPolynomial(); err != nil {
		t.Error(err)
	}

	if shares := tkh.createShares(); len(shares) != 100 {
		t.Fail()
	}
}

func TestCreateViArray(t *testing.T) {
	tkh := new(ThresholdKeyGenerator)
	tkh.TotalNumberOfDecryptionServers = 10
	tkh.v = b(54)
	tkh.nSquare = b(101 * 101)
	vArr := tkh.createViArray([]*big.Int{b(12), b(90), b(103)})
	exp := []*big.Int{b(6162), b(304), b(2728)}
	if !reflect.DeepEqual(vArr, exp) {
		t.Fail()
	}
}

func TestGetThresholdKeyGenerator(t *testing.T) {
	tkh := GetThresholdKeyGenerator(50, 10, 6, rand.Reader)
	if err := tkh.initNumerialValues(); err != nil {
		t.Error(nil)
	}
}

func TestGenerate(t *testing.T) {
	tkh := GetThresholdKeyGenerator(32, 10, 6, rand.Reader)
	tpks, err := tkh.Generate()
	if err != nil {
		t.Error(err)
		return
	}
	if len(tpks) != 10 {
		t.Fail()
	}
	for i, tpk := range tpks {
		if tpk.Id != i+1 {
			t.Fail()
		}
		if len(tpk.Vi) != 10 {
			t.Fail()
		}
		if tpk.N == nil {
			t.Fail()
		}
		if tpk.Threshold != 6 || tpk.TotalNumberOfDecryptionServers != 10 {
			t.Fail()
		}
	}
}

func TestComputeV(t *testing.T) {
	tkh := GetThresholdKeyGenerator(32, 10, 6, rand.Reader)
	tkh.n = b(1907 * 1823)
	tkh.nSquare = new(big.Int).Mul(tkh.n, tkh.n)
	for i := 0; i < 100; i++ {
		if err := tkh.computeV(); err != nil {
			t.Error(err)
		}
		if tkh.v.Cmp(tkh.nSquare) > 0 {
			t.Error("v is too big")
		}
		if tkh.v.Cmp(tkh.n) > 0 {
			return
		}
	}
	t.Error(`v has never been bigger than n.  It is suspicious in the sense<
	than it was taken in the range 0...n**2 -1
	`)
}
<|endoftext|>


<commit_before>package dns

import (
	"github.com/alekar/route53/src/route53"
	"github.com/crowdmob/goamz/aws"
	"strings"
	"time"
)

type Route53Provider struct {
	r53  *route53.Route53
	Zone route53.HostedZone
	TTL  uint
}

func (r *Route53Provider) createRecords(comment string, rrsets ...route53.RRSet) (error, chan error) {
	changes := make([]route53.RRSetChange, len(rrsets))
	for i, rrset := range rrsets {
		changes[i] = route53.RRSetChange{
			Action: "CREATE",
			RRSet:  rrset,
		}
	}
	info, err := r.r53.ChangeRRSet(r.Zone.Id, changes, comment)
	if err != nil {
		return err, nil
	}
	return nil, info.PollForSync(time.Second, 60*time.Second)
}

func (r *Route53Provider) baseRRSet(id, name, failover string) route53.RRSet {
	rrset := route53.RRSet{
		Name:          name,
		Type:          "A",
		TTL:           r.TTL,
		SetIdentifier: id,
		Weight:        0,
	}
	if failover == "PRIMARY" || failover == "SECONDARY" {
		rrset.Failover = failover
	}
	return rrset
}

func (r *Route53Provider) CreateAliases(comment string, aliases []Alias) (error, chan error) {
	rrsets := make([]route53.RRSet, len(aliases))
	count := 0
	for _, alias := range aliases {
		rrsets[count] = r.baseRRSet(alias.Id(), alias.Alias, alias.Failover)
		rrsets[count].HostedZoneId = r.Zone.Id
		rrsets[count].DNSName = alias.Original
		rrsets[count].EvaluateTargetHealth = true
		count++
	}
	return r.createRecords(comment, rrsets...)
}

func (r *Route53Provider) CreateCNames(comment string, cnames []CName) (error, chan error) {
	rrsets := make([]route53.RRSet, len(cnames))
	count := 0
	for _, cname := range cnames {
		rrsets[count] = r.baseRRSet(cname.Id(), cname.CName, cname.Failover)
		rrsets[count].Values = []string{cname.IP}
		rrsets[count].HealthCheckId = cname.HealthCheckId
		count++
	}
	return r.createRecords(comment, rrsets...)
}

func (r *Route53Provider) DeleteRecords(comment string, ids ...string) (error, chan error) {
	if len(ids) == 0 {
		errChan := make(chan error)
		go func(ch chan error) { // fake channel with nil error
			ch <- nil
		}(errChan)
		return nil, errChan
	}
	// fetch all records
	rrsets, err := r.r53.ListRRSets(r.Zone.Id)
	if err != nil {
		return err, nil
	}
	// create record map to make things easier
	rrsetMap := map[string]route53.RRSet{}
	for _, rrset := range rrsets {
		rrsetMap[rrset.SetIdentifier] = rrset
	}
	// filter by id and delete
	toDelete := []route53.RRSet{}
	for _, id := range ids {
		if rrset, exists := rrsetMap[id]; exists {
			toDelete = append(toDelete, rrset)
		}
	}
	changes := make([]route53.RRSetChange, len(toDelete))
	for i, rrset := range toDelete {
		changes[i] = route53.RRSetChange{
			Action: "DELETE",
			RRSet:  rrset,
		}
	}
	info, err := r.r53.ChangeRRSet(r.Zone.Id, changes, comment)
	if err != nil {
		return err, nil
	}
	return nil, info.PollForSync(time.Second, 60*time.Second)
}

func (r *Route53Provider) CreateHealthCheck(ip string, port uint16) (string, error) {
	// health check to make sure TCP 80 is reachable
	config := route53.HealthCheckConfig{
		IPAddress: ip,
		Port:      port,
		Type:      "TCP",
	}
	// add health check for ip, return health check id
	return r.r53.CreateHealthCheck(config, "")
}

func (r *Route53Provider) DeleteHealthCheck(id string) error {
	return r.r53.DeleteHealthCheck(id)
}

func (r *Route53Provider) GetRecordsForIP(ip string) ([]string, error) {
	rrsets, err := r.r53.ListRRSets(r.Zone.Id)
	if err != nil {
		return nil, err
	}
	ids := []string{}
	for _, rrset := range rrsets {
		for _, value := range rrset.Values {
			if value == ip {
				ids = append(ids, rrset.SetIdentifier)
			}
		}
	}
	return ids, nil
}

func (r *Route53Provider) Suffix() string {
	return strings.TrimRight(r.Zone.Name, ".")
}

func NewRoute53Provider(zoneId string, ttl uint) (*Route53Provider, error) {
	route53.DebugOn()
	auth, err := aws.GetAuth("", "", "", time.Time{})
	if err != nil {
		return nil, err
	}
	r53 := route53.New(auth)
	zone, err := r53.GetHostedZone(zoneId)
	if err != nil {
		return nil, err
	}
	return &Route53Provider{r53: r53, Zone: zone, TTL: ttl}, nil
}
<commit_msg>Zone.Id returns weird shit<commit_after>package dns

import (
	"github.com/alekar/route53/src/route53"
	"github.com/crowdmob/goamz/aws"
	"strings"
	"time"
)

type Route53Provider struct {
	r53  *route53.Route53
	Zone route53.HostedZone
	ZoneId string
	TTL  uint
}

func (r *Route53Provider) createRecords(comment string, rrsets ...route53.RRSet) (error, chan error) {
	changes := make([]route53.RRSetChange, len(rrsets))
	for i, rrset := range rrsets {
		changes[i] = route53.RRSetChange{
			Action: "CREATE",
			RRSet:  rrset,
		}
	}
	info, err := r.r53.ChangeRRSet(r.ZoneId, changes, comment)
	if err != nil {
		return err, nil
	}
	return nil, info.PollForSync(time.Second, 60*time.Second)
}

func (r *Route53Provider) baseRRSet(id, name, failover string) route53.RRSet {
	rrset := route53.RRSet{
		Name:          name,
		Type:          "A",
		TTL:           r.TTL,
		SetIdentifier: id,
		Weight:        0,
	}
	if failover == "PRIMARY" || failover == "SECONDARY" {
		rrset.Failover = failover
	}
	return rrset
}

func (r *Route53Provider) CreateAliases(comment string, aliases []Alias) (error, chan error) {
	rrsets := make([]route53.RRSet, len(aliases))
	count := 0
	for _, alias := range aliases {
		rrsets[count] = r.baseRRSet(alias.Id(), alias.Alias, alias.Failover)
		rrsets[count].HostedZoneId = r.ZoneId
		rrsets[count].DNSName = alias.Original
		rrsets[count].EvaluateTargetHealth = true
		count++
	}
	return r.createRecords(comment, rrsets...)
}

func (r *Route53Provider) CreateCNames(comment string, cnames []CName) (error, chan error) {
	rrsets := make([]route53.RRSet, len(cnames))
	count := 0
	for _, cname := range cnames {
		rrsets[count] = r.baseRRSet(cname.Id(), cname.CName, cname.Failover)
		rrsets[count].Values = []string{cname.IP}
		rrsets[count].HealthCheckId = cname.HealthCheckId
		count++
	}
	return r.createRecords(comment, rrsets...)
}

func (r *Route53Provider) DeleteRecords(comment string, ids ...string) (error, chan error) {
	if len(ids) == 0 {
		errChan := make(chan error)
		go func(ch chan error) { // fake channel with nil error
			ch <- nil
		}(errChan)
		return nil, errChan
	}
	// fetch all records
	rrsets, err := r.r53.ListRRSets(r.ZoneId)
	if err != nil {
		return err, nil
	}
	// create record map to make things easier
	rrsetMap := map[string]route53.RRSet{}
	for _, rrset := range rrsets {
		rrsetMap[rrset.SetIdentifier] = rrset
	}
	// filter by id and delete
	toDelete := []route53.RRSet{}
	for _, id := range ids {
		if rrset, exists := rrsetMap[id]; exists {
			toDelete = append(toDelete, rrset)
		}
	}
	changes := make([]route53.RRSetChange, len(toDelete))
	for i, rrset := range toDelete {
		changes[i] = route53.RRSetChange{
			Action: "DELETE",
			RRSet:  rrset,
		}
	}
	info, err := r.r53.ChangeRRSet(r.ZoneId, changes, comment)
	if err != nil {
		return err, nil
	}
	return nil, info.PollForSync(time.Second, 60*time.Second)
}

func (r *Route53Provider) CreateHealthCheck(ip string, port uint16) (string, error) {
	// health check to make sure TCP 80 is reachable
	config := route53.HealthCheckConfig{
		IPAddress: ip,
		Port:      port,
		Type:      "TCP",
	}
	// add health check for ip, return health check id
	return r.r53.CreateHealthCheck(config, "")
}

func (r *Route53Provider) DeleteHealthCheck(id string) error {
	return r.r53.DeleteHealthCheck(id)
}

func (r *Route53Provider) GetRecordsForIP(ip string) ([]string, error) {
	rrsets, err := r.r53.ListRRSets(r.ZoneId)
	if err != nil {
		return nil, err
	}
	ids := []string{}
	for _, rrset := range rrsets {
		for _, value := range rrset.Values {
			if value == ip {
				ids = append(ids, rrset.SetIdentifier)
			}
		}
	}
	return ids, nil
}

func (r *Route53Provider) Suffix() string {
	return strings.TrimRight(r.Zone.Name, ".")
}

func NewRoute53Provider(zoneId string, ttl uint) (*Route53Provider, error) {
	route53.DebugOn()
	auth, err := aws.GetAuth("", "", "", time.Time{})
	if err != nil {
		return nil, err
	}
	r53 := route53.New(auth)
	zone, err := r53.GetHostedZone(zoneId)
	if err != nil {
		return nil, err
	}
	return &Route53Provider{r53: r53, Zone: zone, ZoneId: zoneId, TTL: ttl}, nil
}
<|endoftext|>


<commit_before>// Copyright 2016, RadiantBlue Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package workflow

import (
	"encoding/json"
	"fmt"

	"github.com/venicegeo/pz-gocommon/elasticsearch"
	"github.com/venicegeo/pz-gocommon/gocommon"
)

type EventDB struct {
	*ResourceDB
}

func NewEventDB(service *WorkflowService, esi elasticsearch.IIndex) (*EventDB, error) {
	rdb, err := NewResourceDB(service, esi, EventIndexSettings)
	if err != nil {
		return nil, err
	}
	erdb := EventDB{ResourceDB: rdb}
	return &erdb, nil
}

func (db *EventDB) PostData(mapping string, obj interface{}, id piazza.Ident) (piazza.Ident, error) {
	var event Event
	ok1 := false
	event, ok1 = obj.(Event)
	if !ok1 {
		temp, ok2 := obj.(*Event)
		if !ok2 {
			return piazza.NoIdent, LoggedError("EventDB.PostData failed: was not given an Event")
		}
		event = *temp
	}

	err := verifyEventReadyToPost(&event, db)
	if err != nil {
		return piazza.NoIdent, err
	}

	indexResult, err := db.Esi.PostData(mapping, id.String(), obj)
	if err != nil {
		return piazza.NoIdent, LoggedError("EventDB.PostData failed: %s", err)
	}
	if !indexResult.Created {
		return piazza.NoIdent, LoggedError("EventDB.PostData failed: not created")
	}

	return id, nil
}

func verifyEventReadyToPost(event *Event, db *EventDB) error {
	eventTypeJson := db.service.GetEventType(event.EventTypeId)
	eventTypeObj := eventTypeJson.Data
	eventType, ok := eventTypeObj.(*EventType)
	if !ok {
		return LoggedError("EventDB.PostData failed: unable to obtain specified eventtype")
	}
	eventTypeMappingVars, err := piazza.GetVarsFromStruct(eventType.Mapping)
	if err != nil {
		return LoggedError("EventDB.PostData failed: %s", err)
	}
	eventDataVars, err := piazza.GetVarsFromStruct(event.Data)
	if err != nil {
		return LoggedError("EventDB.PostData failed: %s", err)
	}
	notFound := []string{}
	for k, v := range eventTypeMappingVars {
		found := false
		for k2, v2 := range eventDataVars {
			if k2 == k {
				found = true
				if !elasticsearch.IsValidArrayTypeMapping(v) {
					if piazza.ValueIsValidArray(v2) {
						return LoggedError("EventDB.PostData failed: an array was passed into the non-array field %s", k)
					}
				} else {
					if !piazza.ValueIsValidArray(v2) {
						return LoggedError("EventDB.PostData failed: a non-array was pasted into the array field %s", k)
					}
				}
				break
			}
		}
		if !found {
			notFound = append(notFound, k)
		}
	}
	if len(notFound) > 0 {
		return LoggedError("EventDB.PostData failed: the variables %s were specified in the EventType but were not found in the Event", notFound)
	}
	return nil
}

func (db *EventDB) GetAll(mapping string, format *piazza.JsonPagination) ([]Event, int64, error) {
	events := []Event{}
	var err error

	exists := true
	if mapping != "" {
		exists, err = db.Esi.TypeExists(mapping)
		if err != nil {
			return events, 0, err
		}
	}
	if !exists {
		return nil, 0, fmt.Errorf("Type %s does not exist", mapping)
	}

	searchResult, err := db.Esi.FilterByMatchAll(mapping, format)
	if err != nil {
		return nil, 0, LoggedError("EventDB.GetAll failed: %s", err)
	}
	if searchResult == nil {
		return nil, 0, LoggedError("EventDB.GetAll failed: no searchResult")
	}

	if searchResult != nil && searchResult.GetHits() != nil {
		for _, hit := range *searchResult.GetHits() {
			var event Event
			err := json.Unmarshal(*hit.Source, &event)
			if err != nil {
				return nil, 0, err
			}
			events = append(events, event)
		}
	}

	return events, searchResult.TotalHits(), nil
}

func (db *EventDB) lookupEventTypeNameByEventID(id piazza.Ident) (string, error) {
	var mapping string = ""

	types, err := db.Esi.GetTypes()
	if err != nil {
		return "", err
	}
	for _, typ := range types {
		ok, err := db.Esi.ItemExists(typ, id.String())
		if err != nil {
			return "", err
		}
		if ok {
			mapping = typ
			break
		}
	}
	if mapping == "" {
		return "", LoggedError("EventDB.lookupEventTypeNameByEventID failed: [Item %s in index events does not exist]", id.String())
	}

	return mapping, nil
}

// NameExists checks if an EventType name exists.
// This is easier to check in EventDB, as the mappings use the EventType.Name.
func (db *EventDB) NameExists(name string) (bool, error) {
	return db.Esi.TypeExists(name)
}

func (db *EventDB) GetOne(mapping string, id piazza.Ident) (*Event, bool, error) {
	getResult, err := db.Esi.GetByID(mapping, id.String())
	if err != nil {
		return nil, false, LoggedError("EventDB.GetOne failed: %s", err)
	}
	if getResult == nil {
		return nil, true, LoggedError("EventDB.GetOne failed: no getResult")
	}

	src := getResult.Source
	var event Event
	err = json.Unmarshal(*src, &event)
	if err != nil {
		return nil, getResult.Found, err
	}

	return &event, getResult.Found, nil
}

func (db *EventDB) DeleteByID(mapping string, id piazza.Ident) (bool, error) {
	deleteResult, err := db.Esi.DeleteByID(mapping, string(id))
	if err != nil {
		return deleteResult.Found, LoggedError("EventDB.DeleteById failed: %s", err)
	}
	if deleteResult == nil {
		return false, LoggedError("EventDB.DeleteById failed: no deleteResult")
	}

	return deleteResult.Found, nil
}

func (db *EventDB) AddMapping(name string, mapping map[string]interface{}) error {
	jsn, err := ConstructEventMappingSchema(name, mapping)
	if err != nil {
		return LoggedError("EventDB.AddMapping failed: %s", err)
	}
	err = db.Esi.SetMapping(name, jsn)
	if err != nil {
		return LoggedError("EventDB.AddMapping SetMapping failed: %s", err)
	}

	return nil
}

func ConstructEventMappingSchema(name string, mapping map[string]interface{}) (piazza.JsonString, error) {
	const template string = `{
		"%s":{
			"properties":{
				"data":{
					"dynamic": "strict",
					"properties": %s
				}
			}
		}
	}`
	esdsl, err := buildMapping(mapping)
	if err != nil {
		return piazza.JsonString(""), err
	}
	strDsl, err := piazza.StructInterfaceToString(esdsl)
	if err != nil {
		return piazza.JsonString(""), err
	}
	jsn := fmt.Sprintf(template, name, strDsl)
	return piazza.JsonString(jsn), nil
}

func buildMapping(input map[string]interface{}) (map[string]interface{}, error) {
	return visitNode(input)
}
func visitNode(inputObj map[string]interface{}) (map[string]interface{}, error) {
	outputObj := map[string]interface{}{}
	for k, v := range inputObj {
		switch t := v.(type) {
		case string:
			tree, err := visitLeaf(k, v)
			if err != nil {
				return nil, err
			}
			outputObj[k] = tree
		case map[string]interface{}:
			tree, err := visitTree(k, v.(map[string]interface{}))
			if err != nil {
				return nil, err
			}
			outputObj[k] = tree
		default:
			return nil, LoggedError("EventDB.ConstructEventMappingSchema failed: unexpected type %T", t)
		}
	}
	return outputObj, nil
}
func visitTree(k string, v map[string]interface{}) (map[string]interface{}, error) {
	subtree, err := visitNode(v)
	if err != nil {
		return nil, err
	}
	wrapperTree := map[string]interface{}{}
	wrapperTree["dynamic"] = "strict"
	wrapperTree["properties"] = subtree
	return wrapperTree, nil
}
func visitLeaf(k string, v interface{}) (map[string]interface{}, error) {
	if !elasticsearch.IsValidMappingType(v) {
		return nil, LoggedError("EventDB.ConstructEventMappingSchema failed: %s was not recognized as a valid mapping type")
	}
	if elasticsearch.IsValidArrayTypeMapping(v) {
		v = v.(string)[1 : len(v.(string))-1]
	}
	tree := map[string]interface{}{}
	tree["type"] = v
	return tree, nil
}

func (db *EventDB) PercolateEventData(eventType string, data map[string]interface{}, id piazza.Ident) (*[]piazza.Ident, error) {
	fixed := map[string]interface{}{}
	fixed["data"] = data
	percolateResponse, err := db.Esi.AddPercolationDocument(eventType, fixed)

	if err != nil {
		return nil, LoggedError("EventDB.PercolateEventData failed: %s", err)
	}
	if percolateResponse == nil {
		return nil, LoggedError("EventDB.PercolateEventData failed: no percolateResult")
	}

	// add the triggers to the alert queue
	ids := make([]piazza.Ident, len(percolateResponse.Matches))
	for i, v := range percolateResponse.Matches {
		ids[i] = piazza.Ident(v.Id)
	}

	return &ids, nil
}
<commit_msg>format typo<commit_after>// Copyright 2016, RadiantBlue Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package workflow

import (
	"encoding/json"
	"fmt"

	"github.com/venicegeo/pz-gocommon/elasticsearch"
	"github.com/venicegeo/pz-gocommon/gocommon"
)

type EventDB struct {
	*ResourceDB
}

func NewEventDB(service *WorkflowService, esi elasticsearch.IIndex) (*EventDB, error) {
	rdb, err := NewResourceDB(service, esi, EventIndexSettings)
	if err != nil {
		return nil, err
	}
	erdb := EventDB{ResourceDB: rdb}
	return &erdb, nil
}

func (db *EventDB) PostData(mapping string, obj interface{}, id piazza.Ident) (piazza.Ident, error) {
	var event Event
	ok1 := false
	event, ok1 = obj.(Event)
	if !ok1 {
		temp, ok2 := obj.(*Event)
		if !ok2 {
			return piazza.NoIdent, LoggedError("EventDB.PostData failed: was not given an Event")
		}
		event = *temp
	}

	err := verifyEventReadyToPost(&event, db)
	if err != nil {
		return piazza.NoIdent, err
	}

	indexResult, err := db.Esi.PostData(mapping, id.String(), obj)
	if err != nil {
		return piazza.NoIdent, LoggedError("EventDB.PostData failed: %s", err)
	}
	if !indexResult.Created {
		return piazza.NoIdent, LoggedError("EventDB.PostData failed: not created")
	}

	return id, nil
}

func verifyEventReadyToPost(event *Event, db *EventDB) error {
	eventTypeJson := db.service.GetEventType(event.EventTypeId)
	eventTypeObj := eventTypeJson.Data
	eventType, ok := eventTypeObj.(*EventType)
	if !ok {
		return LoggedError("EventDB.PostData failed: unable to obtain specified eventtype")
	}
	eventTypeMappingVars, err := piazza.GetVarsFromStruct(eventType.Mapping)
	if err != nil {
		return LoggedError("EventDB.PostData failed: %s", err)
	}
	eventDataVars, err := piazza.GetVarsFromStruct(event.Data)
	if err != nil {
		return LoggedError("EventDB.PostData failed: %s", err)
	}
	notFound := []string{}
	for k, v := range eventTypeMappingVars {
		found := false
		for k2, v2 := range eventDataVars {
			if k2 == k {
				found = true
				if !elasticsearch.IsValidArrayTypeMapping(v) {
					if piazza.ValueIsValidArray(v2) {
						return LoggedError("EventDB.PostData failed: an array was passed into the non-array field %s", k)
					}
				} else {
					if !piazza.ValueIsValidArray(v2) {
						return LoggedError("EventDB.PostData failed: a non-array was pasted into the array field %s", k)
					}
				}
				break
			}
		}
		if !found {
			notFound = append(notFound, k)
		}
	}
	if len(notFound) > 0 {
		return LoggedError("EventDB.PostData failed: the variables %s were specified in the EventType but were not found in the Event", notFound)
	}
	return nil
}

func (db *EventDB) GetAll(mapping string, format *piazza.JsonPagination) ([]Event, int64, error) {
	events := []Event{}
	var err error

	exists := true
	if mapping != "" {
		exists, err = db.Esi.TypeExists(mapping)
		if err != nil {
			return events, 0, err
		}
	}
	if !exists {
		return nil, 0, fmt.Errorf("Type %s does not exist", mapping)
	}

	searchResult, err := db.Esi.FilterByMatchAll(mapping, format)
	if err != nil {
		return nil, 0, LoggedError("EventDB.GetAll failed: %s", err)
	}
	if searchResult == nil {
		return nil, 0, LoggedError("EventDB.GetAll failed: no searchResult")
	}

	if searchResult != nil && searchResult.GetHits() != nil {
		for _, hit := range *searchResult.GetHits() {
			var event Event
			err := json.Unmarshal(*hit.Source, &event)
			if err != nil {
				return nil, 0, err
			}
			events = append(events, event)
		}
	}

	return events, searchResult.TotalHits(), nil
}

func (db *EventDB) lookupEventTypeNameByEventID(id piazza.Ident) (string, error) {
	var mapping string = ""

	types, err := db.Esi.GetTypes()
	if err != nil {
		return "", err
	}
	for _, typ := range types {
		ok, err := db.Esi.ItemExists(typ, id.String())
		if err != nil {
			return "", err
		}
		if ok {
			mapping = typ
			break
		}
	}
	if mapping == "" {
		return "", LoggedError("EventDB.lookupEventTypeNameByEventID failed: [Item %s in index events does not exist]", id.String())
	}

	return mapping, nil
}

// NameExists checks if an EventType name exists.
// This is easier to check in EventDB, as the mappings use the EventType.Name.
func (db *EventDB) NameExists(name string) (bool, error) {
	return db.Esi.TypeExists(name)
}

func (db *EventDB) GetOne(mapping string, id piazza.Ident) (*Event, bool, error) {
	getResult, err := db.Esi.GetByID(mapping, id.String())
	if err != nil {
		return nil, false, LoggedError("EventDB.GetOne failed: %s", err)
	}
	if getResult == nil {
		return nil, true, LoggedError("EventDB.GetOne failed: no getResult")
	}

	src := getResult.Source
	var event Event
	err = json.Unmarshal(*src, &event)
	if err != nil {
		return nil, getResult.Found, err
	}

	return &event, getResult.Found, nil
}

func (db *EventDB) DeleteByID(mapping string, id piazza.Ident) (bool, error) {
	deleteResult, err := db.Esi.DeleteByID(mapping, string(id))
	if err != nil {
		return deleteResult.Found, LoggedError("EventDB.DeleteById failed: %s", err)
	}
	if deleteResult == nil {
		return false, LoggedError("EventDB.DeleteById failed: no deleteResult")
	}

	return deleteResult.Found, nil
}

func (db *EventDB) AddMapping(name string, mapping map[string]interface{}) error {
	jsn, err := ConstructEventMappingSchema(name, mapping)
	if err != nil {
		return LoggedError("EventDB.AddMapping failed: %s", err)
	}
	err = db.Esi.SetMapping(name, jsn)
	if err != nil {
		return LoggedError("EventDB.AddMapping SetMapping failed: %s", err)
	}

	return nil
}

func ConstructEventMappingSchema(name string, mapping map[string]interface{}) (piazza.JsonString, error) {
	const template string = `{
		"%s":{
			"properties":{
				"data":{
					"dynamic": "strict",
					"properties": %s
				}
			}
		}
	}`
	esdsl, err := buildMapping(mapping)
	if err != nil {
		return piazza.JsonString(""), err
	}
	strDsl, err := piazza.StructInterfaceToString(esdsl)
	if err != nil {
		return piazza.JsonString(""), err
	}
	jsn := fmt.Sprintf(template, name, strDsl)
	return piazza.JsonString(jsn), nil
}

func buildMapping(input map[string]interface{}) (map[string]interface{}, error) {
	return visitNode(input)
}
func visitNode(inputObj map[string]interface{}) (map[string]interface{}, error) {
	outputObj := map[string]interface{}{}
	for k, v := range inputObj {
		switch t := v.(type) {
		case string:
			tree, err := visitLeaf(k, v)
			if err != nil {
				return nil, err
			}
			outputObj[k] = tree
		case map[string]interface{}:
			tree, err := visitTree(k, v.(map[string]interface{}))
			if err != nil {
				return nil, err
			}
			outputObj[k] = tree
		default:
			return nil, LoggedError("EventDB.ConstructEventMappingSchema failed: unexpected type %T", t)
		}
	}
	return outputObj, nil
}
func visitTree(k string, v map[string]interface{}) (map[string]interface{}, error) {
	subtree, err := visitNode(v)
	if err != nil {
		return nil, err
	}
	wrapperTree := map[string]interface{}{}
	wrapperTree["dynamic"] = "strict"
	wrapperTree["properties"] = subtree
	return wrapperTree, nil
}
func visitLeaf(k string, v interface{}) (map[string]interface{}, error) {
	if !elasticsearch.IsValidMappingType(v) {
		return nil, LoggedError("EventDB.ConstructEventMappingSchema failed: \"%#v\" was not recognized as a valid mapping type", v)
	}
	if elasticsearch.IsValidArrayTypeMapping(v) {
		v = v.(string)[1 : len(v.(string))-1]
	}
	tree := map[string]interface{}{}
	tree["type"] = v
	return tree, nil
}

func (db *EventDB) PercolateEventData(eventType string, data map[string]interface{}, id piazza.Ident) (*[]piazza.Ident, error) {
	fixed := map[string]interface{}{}
	fixed["data"] = data
	percolateResponse, err := db.Esi.AddPercolationDocument(eventType, fixed)

	if err != nil {
		return nil, LoggedError("EventDB.PercolateEventData failed: %s", err)
	}
	if percolateResponse == nil {
		return nil, LoggedError("EventDB.PercolateEventData failed: no percolateResult")
	}

	// add the triggers to the alert queue
	ids := make([]piazza.Ident, len(percolateResponse.Matches))
	for i, v := range percolateResponse.Matches {
		ids[i] = piazza.Ident(v.Id)
	}

	return &ids, nil
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/tls"
	"flag"
	"fmt"
	"os"
	"time"

	"github.com/coreos/etcd/clientv3"
	"github.com/coreos/etcd/pkg/transport"
	"golang.org/x/net/context"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/runtime/serializer/json"
	"k8s.io/kubernetes/pkg/runtime/serializer/protobuf"

	// install all APIs
	_ "github.com/openshift/origin/pkg/api/install"
	_ "github.com/openshift/origin/pkg/quota/api/install"
	_ "k8s.io/kubernetes/pkg/api/install"
)

func main() {
	var endpoint, keyFile, certFile, caFile string
	flag.StringVar(&endpoint, "endpoint", "https://127.0.0.1:4001", "Etcd endpoint.")
	flag.StringVar(&keyFile, "key-file", "", "TLS client key.")
	flag.StringVar(&certFile, "cert-file", "", "TLS client certificate.")
	flag.StringVar(&caFile, "ca-file", "", "Server TLS CA certificate.")
	flag.Parse()

	if flag.NArg() == 0 {
		fmt.Fprintf(os.Stderr, "ERROR: you need to specify action ls [<key>] or get <key>\n")
		os.Exit(1)
	}
	if flag.Arg(0) == "get" && flag.NArg() == 1 {
		fmt.Fprintf(os.Stderr, "ERROR: you need to specify <key> for get operation\n")
		os.Exit(1)
	}
	action := flag.Arg(0)
	key := ""
	if flag.NArg() > 1 {
		key = flag.Arg(1)
	}

	var tlsConfig *tls.Config
	if len(certFile) != 0 || len(keyFile) != 0 || len(caFile) != 0 {
		tlsInfo := transport.TLSInfo{
			CertFile: certFile,
			KeyFile:  keyFile,
			CAFile:   caFile,
		}
		var err error
		tlsConfig, err = tlsInfo.ClientConfig()
		if err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: unable to create client config: %v\n", err)
			os.Exit(1)
		}
	}

	config := clientv3.Config{
		Endpoints:   []string{endpoint},
		TLS:         tlsConfig,
		DialTimeout: 5 * time.Second,
	}
	client, err := clientv3.New(config)
	if err != nil {
		fmt.Fprintf(os.Stderr, "ERROR: unable to connect to etcd: %v\n", err)
		os.Exit(1)
	}
	defer client.Close()

	switch action {
	case "ls":
		err = listKeys(client, key)
	case "get":
		err = getKey(client, key)
	}
	if err != nil {
		fmt.Fprintf(os.Stderr, "ERROR: %s-ing %s: %v\n", action, key, err)
		os.Exit(1)
	}
}

func listKeys(client *clientv3.Client, key string) error {
	var resp *clientv3.GetResponse
	var err error
	if len(key) == 0 {
		resp, err = client.Get(context.TODO(), "/", clientv3.WithFromKey(), clientv3.WithKeysOnly())
	} else {
		resp, err = client.Get(context.TODO(), key, clientv3.WithPrefix(), clientv3.WithKeysOnly())
	}
	if err != nil {
		return err
	}
	for _, ev := range resp.Kvs {
		fmt.Printf("%s\n", ev.Key)
	}
	return nil
}

func getKey(client *clientv3.Client, key string) error {
	resp, err := client.Get(context.TODO(), key)
	if err != nil {
		return err
	}
	ps := protobuf.NewSerializer(api.Scheme, api.Scheme, "application/vnd.kubernetes.protobuf")
	js := json.NewSerializer(json.DefaultMetaFactory, api.Scheme, api.Scheme, true)
	for _, ev := range resp.Kvs {
		obj, gvk, err := ps.Decode(ev.Value, nil, nil)
		if err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: unable to decode %s: %v\n", ev.Key, err)
			continue
		}
		fmt.Println(gvk)
		err = js.Encode(obj, os.Stdout)
		if err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: unable to decode %s: %v\n", ev.Key, err)
			continue
		}
	}
	return nil
}
<commit_msg>Fix etcdhelper imports<commit_after>package main

import (
	"crypto/tls"
	"flag"
	"fmt"
	"os"
	"time"

	"github.com/coreos/etcd/clientv3"
	"github.com/coreos/etcd/pkg/transport"
	"golang.org/x/net/context"
	"k8s.io/apimachinery/pkg/runtime/serializer/json"
	"k8s.io/apimachinery/pkg/runtime/serializer/protobuf"
	"k8s.io/kubernetes/pkg/api"

	// install all APIs
	_ "github.com/openshift/origin/pkg/api/install"
	_ "github.com/openshift/origin/pkg/quota/api/install"
	_ "k8s.io/kubernetes/pkg/api/install"
)

func main() {
	var endpoint, keyFile, certFile, caFile string
	flag.StringVar(&endpoint, "endpoint", "https://127.0.0.1:4001", "Etcd endpoint.")
	flag.StringVar(&keyFile, "key-file", "", "TLS client key.")
	flag.StringVar(&certFile, "cert-file", "", "TLS client certificate.")
	flag.StringVar(&caFile, "ca-file", "", "Server TLS CA certificate.")
	flag.Parse()

	if flag.NArg() == 0 {
		fmt.Fprintf(os.Stderr, "ERROR: you need to specify action ls [<key>] or get <key>\n")
		os.Exit(1)
	}
	if flag.Arg(0) == "get" && flag.NArg() == 1 {
		fmt.Fprintf(os.Stderr, "ERROR: you need to specify <key> for get operation\n")
		os.Exit(1)
	}
	action := flag.Arg(0)
	key := ""
	if flag.NArg() > 1 {
		key = flag.Arg(1)
	}

	var tlsConfig *tls.Config
	if len(certFile) != 0 || len(keyFile) != 0 || len(caFile) != 0 {
		tlsInfo := transport.TLSInfo{
			CertFile: certFile,
			KeyFile:  keyFile,
			CAFile:   caFile,
		}
		var err error
		tlsConfig, err = tlsInfo.ClientConfig()
		if err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: unable to create client config: %v\n", err)
			os.Exit(1)
		}
	}

	config := clientv3.Config{
		Endpoints:   []string{endpoint},
		TLS:         tlsConfig,
		DialTimeout: 5 * time.Second,
	}
	client, err := clientv3.New(config)
	if err != nil {
		fmt.Fprintf(os.Stderr, "ERROR: unable to connect to etcd: %v\n", err)
		os.Exit(1)
	}
	defer client.Close()

	switch action {
	case "ls":
		err = listKeys(client, key)
	case "get":
		err = getKey(client, key)
	}
	if err != nil {
		fmt.Fprintf(os.Stderr, "ERROR: %s-ing %s: %v\n", action, key, err)
		os.Exit(1)
	}
}

func listKeys(client *clientv3.Client, key string) error {
	var resp *clientv3.GetResponse
	var err error
	if len(key) == 0 {
		resp, err = client.Get(context.TODO(), "/", clientv3.WithFromKey(), clientv3.WithKeysOnly())
	} else {
		resp, err = client.Get(context.TODO(), key, clientv3.WithPrefix(), clientv3.WithKeysOnly())
	}
	if err != nil {
		return err
	}
	for _, ev := range resp.Kvs {
		fmt.Printf("%s\n", ev.Key)
	}
	return nil
}

func getKey(client *clientv3.Client, key string) error {
	resp, err := client.Get(context.TODO(), key)
	if err != nil {
		return err
	}
	ps := protobuf.NewSerializer(api.Scheme, api.Scheme, "application/vnd.kubernetes.protobuf")
	js := json.NewSerializer(json.DefaultMetaFactory, api.Scheme, api.Scheme, true)
	for _, ev := range resp.Kvs {
		obj, gvk, err := ps.Decode(ev.Value, nil, nil)
		if err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: unable to decode %s: %v\n", ev.Key, err)
			continue
		}
		fmt.Println(gvk)
		err = js.Encode(obj, os.Stdout)
		if err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: unable to decode %s: %v\n", ev.Key, err)
			continue
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package ploop

import "strings"
import "os/exec"
import "sync"

// #include <ploop/libploop.h>
import "C"

// Possible SetVerboseLevel arguments
const (
	NoConsole  = C.PLOOP_LOG_NOCONSOLE
	NoStdout   = C.PLOOP_LOG_NOSTDOUT
	Timestamps = C.PLOOP_LOG_TIMESTAMPS
)

// SetVerboseLevel sets a level of verbosity when logging to stdout/stderr
func SetVerboseLevel(v int) {
	C.ploop_set_verbose_level(C.int(v))
}

// SetLogFile enables logging to a file and sets log file name
func SetLogFile(file string) error {
	cfile := C.CString(file)
	defer cfree(cfile)

	ret := C.ploop_set_log_file(cfile)

	return mkerr(ret)
}

// SetLogLevel sets a level of verbosity when logging to a file
func SetLogLevel(v int) {
	C.ploop_set_log_level(C.int(v))
}

// Ploop is a type containing DiskDescriptor.xml opened by the library
type Ploop struct {
	d *C.struct_ploop_disk_images_data
}

var once sync.Once

// load ploop modules
func loadKmod() {
	// try to load ploop modules
	modules := []string{"ploop", "pfmt_ploop1", "pfmt_raw", "pio_direct", "pio_nfs", "pio_kaio"}
	for _, m := range modules {
		exec.Command("modprobe", m).Run()
	}
}

// Open opens a ploop DiskDescriptor.xml, most ploop operations require it
func Open(file string) (Ploop, error) {
	var d Ploop

	once.Do(loadKmod)

	cfile := C.CString(file)
	defer cfree(cfile)

	ret := C.ploop_open_dd(&d.d, cfile)

	return d, mkerr(ret)
}

// Close closes a ploop disk descriptor when it is no longer needed
func (d Ploop) Close() {
	C.ploop_close_dd(d.d)
}

type ImageMode int

// Possible values for ImageMode
const (
	Expanded     ImageMode = C.PLOOP_EXPANDED_MODE
	Preallocated ImageMode = C.PLOOP_EXPANDED_PREALLOCATED_MODE
	Raw          ImageMode = C.PLOOP_RAW_MODE
)

// ParseImageMode converts a string to ImageMode value
func ParseImageMode(s string) (ImageMode, error) {
	switch strings.ToLower(s) {
	case "expanded":
		return Expanded, nil
	case "preallocated":
		return Preallocated, nil
	case "raw":
		return Raw, nil
	default:
		return Expanded, mkerr(E_PARAM)
	}
}

// String converts an ImageMode value to string
func (m ImageMode) String() string {
	switch m {
	case Expanded:
		return "Expanded"
	case Preallocated:
		return "Preallocated"
	case Raw:
		return "Raw"
	}
	return "<unknown>"
}

// CreateFlags is a type for CreateParam.Flags
type CreateFlags uint

// Possible values for CreateFlags
const (
	NoLazy CreateFlags = C.PLOOP_CREATE_NOLAZY
)

// CreateParam is a set of parameters for a newly created ploop
type CreateParam struct {
	Size  uint64      // image size, in kilobytes (FS size is about 10% smaller)
	Mode  ImageMode   // image mode
	File  string      // path to and a file name for base delta image
	CLog  uint        // cluster block size log (6 to 15, default 11)
	Flags CreateFlags // flags
}

// Create creates a ploop image and its DiskDescriptor.xml
func Create(p *CreateParam) error {
	var a C.struct_ploop_create_param

	once.Do(loadKmod)

	// default image file name
	if p.File == "" {
		p.File = "root.hdd"
	}

	a.size = convertSize(p.Size)
	a.mode = C.int(p.Mode)
	if p.CLog != 0 {
		// ploop cluster block size, in 512-byte sectors
		// default is 1M cluster block size (CLog=11)
		// 2^11 = 2048 sectors, 2048*512 = 1M
		a.blocksize = 1 << p.CLog
	}
	a.flags = C.uint(p.Flags)
	a.image = C.CString(p.File)
	defer cfree(a.image)
	a.fstype = C.CString("ext4")
	defer cfree(a.fstype)

	ret := C.ploop_create_image(&a)
	return mkerr(ret)
}

// MountParam is a set of parameters to pass to Mount()
type MountParam struct {
	UUID     string // snapshot uuid (empty for top delta)
	Target   string // mount point (empty if no mount is needed)
	Flags    int    // bit mount flags such as MS_NOATIME
	Data     string // auxiliary mount options
	Readonly bool   // mount read-only
	Fsck     bool   // do fsck before mounting inner FS
	Quota    bool   // enable quota for inner FS
}

// Mount creates a ploop device and (optionally) mounts it
func (d Ploop) Mount(p *MountParam) (string, error) {
	var a C.struct_ploop_mount_param
	var device string

	if p.UUID != "" {
		a.guid = C.CString(p.UUID)
		defer cfree(a.guid)
	}
	if p.Target != "" {
		a.target = C.CString(p.Target)
		defer cfree(a.target)
	}

	// mount_data should not be NULL
	a.mount_data = C.CString(p.Data)
	defer cfree(a.mount_data)

	a.flags = C.int(p.Flags)
	a.ro = boolToC(p.Readonly)
	a.fsck = boolToC(p.Fsck)
	a.quota = boolToC(p.Quota)

	ret := C.ploop_mount_image(d.d, &a)
	if ret == 0 {
		device = C.GoString(&a.device[0])
		// TODO? fsck_code = C.GoString(a.fsck_rc)
	}
	return device, mkerr(ret)
}

// Umount unmounts the ploop filesystem and dismantles the device
func (d Ploop) Umount() error {
	ret := C.ploop_umount_image(d.d)

	return mkerr(ret)
}

// UmountByDevice unmounts the ploop filesystem and dismantles the device.
// Unlike Umount(), this is a lower-level function meaning it can be less
// safe and should generally not be used.
func UmountByDevice(dev string) error {
	cdev := C.CString(dev)
	defer cfree(cdev)

	ret := C.ploop_umount(cdev, nil)

	return mkerr(ret)
}

// Resize changes the ploop size. Online resize is recommended.
func (d Ploop) Resize(size uint64, offline bool) error {
	var p C.struct_ploop_resize_param

	p.size = convertSize(size)
	p.offline_resize = boolToC(offline)

	ret := C.ploop_resize_image(d.d, &p)
	return mkerr(ret)
}

// Snapshot creates a ploop snapshot, returning its uuid
func (d Ploop) Snapshot() (string, error) {
	var p C.struct_ploop_snapshot_param
	var uuid, err = UUID()
	if err != nil {
		return "", err
	}
	p.guid = C.CString(uuid)
	defer cfree(p.guid)

	ret := C.ploop_create_snapshot(d.d, &p)
	if ret == 0 {
		uuid = C.GoString(p.guid)
	}

	return uuid, mkerr(ret)
}

// SwitchSnapshot switches to a specified snapshot,
// creates a new empty delta on top of it, and makes it a top one
// (i.e. the one new data will be written to).
// Old top delta (i.e. data modified since the last snapshot) is lost.
func (d Ploop) SwitchSnapshot(uuid string) error {
	var p C.struct_ploop_snapshot_switch_param

	p.guid = C.CString(uuid)
	defer cfree(p.guid)

	ret := C.ploop_switch_snapshot_ex(d.d, &p)

	return mkerr(ret)
}

// Possible values for SwitchSnapshotExtended flags argument
type SwitchFlag uint

const (
	// SkipDestroy, if set, modifies the behavior of
	// SwitchSnapshotExtended to not delete the old top delta, but
	// make it a snapshot and return its uuid. Without this flag,
	// old top delta (i.e. data modified since the last snapshot)
	// is lost.
	SkipDestroy SwitchFlag = C.PLOOP_SNAP_SKIP_TOPDELTA_DESTROY
	// SkipCreate flag, if set, modifies the behavior of
	// SwitchSnapshotExtended to not create a new top delta,
	// but rather transform the specified snapshot itself to be
	// the new top delta), so all new changes will be written
	// right to it. Snapshot UUID is lost in this case.
	SkipCreate SwitchFlag = C.PLOOP_SNAP_SKIP_TOPDELTA_CREATE
)

// SwitchSnapshotExtended is same as SwitchSnapshot but with additional
// flags modifying its behavior. Please see individual flags description.
// Returns uuid of what was the old top delta if SkipDestroy flag is set.
func (d Ploop) SwitchSnapshotExtended(uuid string, flags SwitchFlag) (string, error) {
	var p C.struct_ploop_snapshot_switch_param
	oldUUID := ""

	p.guid = C.CString(uuid)
	defer cfree(p.guid)

	p.flags = C.int(flags)

	if flags&SkipDestroy != 0 {
		oldUUID, err := UUID()
		if err != nil {
			return "", err
		}
		p.guid_old = C.CString(oldUUID)
		defer cfree(p.guid_old)
	}

	ret := C.ploop_switch_snapshot_ex(d.d, &p)

	return oldUUID, mkerr(ret)
}

// DeleteSnapshot deletes a snapshot (merging it down if necessary)
func (d Ploop) DeleteSnapshot(uuid string) error {
	cuuid := C.CString(uuid)
	defer cfree(cuuid)

	ret := C.ploop_delete_snapshot(d.d, cuuid)

	return mkerr(ret)
}

type ReplaceFlag int

// Possible values for ReplaceParam.flags
const (
	// KeepName renames the new file to old file name after replace;
	// note that if this option is used the old file is removed.
	KeepName ReplaceFlag = C.PLOOP_REPLACE_KEEP_NAME
)

// ReplaceParam is a set of parameters to Replace()
type ReplaceParam struct {
	File string // new image file name
	// Image to be replaced is specified by either
	// uuid, current file name, or level,
	// in the above order of preference.
	UUID    string
	CurFile string
	Level   int
	Flags   ReplaceFlag
}

// Replace replaces a ploop image to a different (but identical) one
func (d Ploop) Replace(p *ReplaceParam) error {
	var a C.struct_ploop_replace_param

	a.file = C.CString(p.File)
	defer cfree(a.file)

	if p.UUID != "" {
		a.guid = C.CString(p.UUID)
		defer cfree(a.guid)
	} else if p.CurFile != "" {
		a.cur_file = C.CString(p.CurFile)
		defer cfree(a.cur_file)
	} else {
		a.level = C.int(p.Level)
	}

	a.flags = C.int(p.Flags)

	ret := C.ploop_replace_image(d.d, &a)

	return mkerr(ret)
}

// IsMounted returns true if ploop is mounted
func (d Ploop) IsMounted() (bool, error) {
	ret := C.ploop_is_mounted(d.d)
	if ret == 0 {
		return false, nil
	} else if ret == 1 {
		return true, nil
	} else {
		// error, but no code, make our own
		return false, mkerr(E_SYS)
	}
}

// FSInfoData holds information about ploop inner file system
type FSInfoData struct {
	BlockSize  uint64
	Blocks     uint64
	BlocksFree uint64
	Inodes     uint64
	InodesFree uint64
}

// FSInfo gets info of ploop's inner file system
func FSInfo(file string) (FSInfoData, error) {
	var cinfo C.struct_ploop_info
	var info FSInfoData
	cfile := C.CString(file)
	defer cfree(cfile)

	once.Do(loadKmod)

	ret := C.ploop_get_info_by_descr(cfile, &cinfo)
	if ret == 0 {
		info.BlockSize = uint64(cinfo.fs_bsize)
		info.Blocks = uint64(cinfo.fs_blocks)
		info.BlocksFree = uint64(cinfo.fs_bfree)
		info.Inodes = uint64(cinfo.fs_inodes)
		info.InodesFree = uint64(cinfo.fs_ifree)
	}

	return info, mkerr(ret)
}

// ImageInfoData holds information about ploop image
type ImageInfoData struct {
	Blocks    uint64
	BlockSize uint32
	Version   int
}

// ImageInfo gets information about a ploop image
func (d Ploop) ImageInfo() (ImageInfoData, error) {
	var cinfo C.struct_ploop_spec
	var info ImageInfoData

	ret := C.ploop_get_spec(d.d, &cinfo)
	if ret == 0 {
		info.Blocks = uint64(cinfo.size)
		info.BlockSize = uint32(cinfo.blocksize)
		info.Version = int(cinfo.fmt_version)
	}

	return info, mkerr(ret)
}

// TopDeltaFile returns file name of top delta
func (d Ploop) TopDeltaFile() (string, error) {
	const len = 4096 // PATH_MAX
	var out [len]C.char

	ret := C.ploop_get_top_delta_fname(d.d, &out[0], len)
	if ret != 0 {
		// error, but no code, make our own
		return "", mkerr(E_SYS)
	}

	file := C.GoString(&out[0])
	return file, nil
}

// UUID generates a ploop UUID
func UUID() (string, error) {
	var cuuid [39]C.char

	ret := C.ploop_uuid_generate(&cuuid[0], 39)
	if ret != 0 {
		return "", mkerr(ret)
	}

	uuid := C.GoString(&cuuid[0])
	return uuid, nil
}
<commit_msg>ploop.go: add/improve some comments<commit_after>package ploop

import "strings"
import "os/exec"
import "sync"

// #include <ploop/libploop.h>
import "C"

// Possible SetVerboseLevel arguments
const (
	NoConsole  = C.PLOOP_LOG_NOCONSOLE
	NoStdout   = C.PLOOP_LOG_NOSTDOUT
	Timestamps = C.PLOOP_LOG_TIMESTAMPS
)

// SetVerboseLevel sets a level of verbosity when logging to stdout/stderr
func SetVerboseLevel(v int) {
	C.ploop_set_verbose_level(C.int(v))
}

// SetLogFile enables logging to a file and sets log file name
func SetLogFile(file string) error {
	cfile := C.CString(file)
	defer cfree(cfile)

	ret := C.ploop_set_log_file(cfile)

	return mkerr(ret)
}

// SetLogLevel sets a level of verbosity when logging to a file
func SetLogLevel(v int) {
	C.ploop_set_log_level(C.int(v))
}

// Ploop is a type containing DiskDescriptor.xml opened by the library
type Ploop struct {
	d *C.struct_ploop_disk_images_data
}

var once sync.Once

// load ploop modules
func loadKmod() {
	// try to load ploop modules
	modules := []string{"ploop", "pfmt_ploop1", "pfmt_raw", "pio_direct", "pio_nfs", "pio_kaio"}
	for _, m := range modules {
		exec.Command("modprobe", m).Run()
	}
}

// Open opens a ploop DiskDescriptor.xml, most ploop operations require it
func Open(file string) (Ploop, error) {
	var d Ploop

	once.Do(loadKmod)

	cfile := C.CString(file)
	defer cfree(cfile)

	ret := C.ploop_open_dd(&d.d, cfile)

	return d, mkerr(ret)
}

// Close closes a ploop disk descriptor when it is no longer needed
func (d Ploop) Close() {
	C.ploop_close_dd(d.d)
}

// ImageMode is a type for CreateParam.Mode field
type ImageMode int

// Possible values for ImageMode
const (
	Expanded     ImageMode = C.PLOOP_EXPANDED_MODE
	Preallocated ImageMode = C.PLOOP_EXPANDED_PREALLOCATED_MODE
	Raw          ImageMode = C.PLOOP_RAW_MODE
)

// ParseImageMode converts a string to ImageMode value
func ParseImageMode(s string) (ImageMode, error) {
	switch strings.ToLower(s) {
	case "expanded":
		return Expanded, nil
	case "preallocated":
		return Preallocated, nil
	case "raw":
		return Raw, nil
	default:
		return Expanded, mkerr(E_PARAM)
	}
}

// String converts an ImageMode value to string
func (m ImageMode) String() string {
	switch m {
	case Expanded:
		return "Expanded"
	case Preallocated:
		return "Preallocated"
	case Raw:
		return "Raw"
	}
	return "<unknown>"
}

// CreateFlags is a type for CreateParam.Flags
type CreateFlags uint

// Possible values for CreateFlags
const (
	NoLazy CreateFlags = C.PLOOP_CREATE_NOLAZY
)

// CreateParam is a set of parameters for a newly created ploop
type CreateParam struct {
	Size  uint64      // image size, in kilobytes (FS size is about 10% smaller)
	Mode  ImageMode   // image mode
	File  string      // path to and a file name for base delta image
	CLog  uint        // cluster block size log (6 to 15, default 11)
	Flags CreateFlags // flags
}

// Create creates a ploop image and its DiskDescriptor.xml
func Create(p *CreateParam) error {
	var a C.struct_ploop_create_param

	once.Do(loadKmod)

	// default image file name
	if p.File == "" {
		p.File = "root.hdd"
	}

	a.size = convertSize(p.Size)
	a.mode = C.int(p.Mode)
	if p.CLog != 0 {
		// ploop cluster block size, in 512-byte sectors
		// default is 1M cluster block size (CLog=11)
		// 2^11 = 2048 sectors, 2048*512 = 1M
		a.blocksize = 1 << p.CLog
	}
	a.flags = C.uint(p.Flags)
	a.image = C.CString(p.File)
	defer cfree(a.image)
	a.fstype = C.CString("ext4")
	defer cfree(a.fstype)

	ret := C.ploop_create_image(&a)
	return mkerr(ret)
}

// MountParam is a set of parameters to pass to Mount()
type MountParam struct {
	UUID     string // snapshot uuid (empty for top delta)
	Target   string // mount point (empty if no mount is needed)
	Flags    int    // bit mount flags such as MS_NOATIME
	Data     string // auxiliary mount options
	Readonly bool   // mount read-only
	Fsck     bool   // do fsck before mounting inner FS
	Quota    bool   // enable quota for inner FS
}

// Mount creates a ploop device and (optionally) mounts it
func (d Ploop) Mount(p *MountParam) (string, error) {
	var a C.struct_ploop_mount_param
	var device string

	if p.UUID != "" {
		a.guid = C.CString(p.UUID)
		defer cfree(a.guid)
	}
	if p.Target != "" {
		a.target = C.CString(p.Target)
		defer cfree(a.target)
	}

	// mount_data should not be NULL
	a.mount_data = C.CString(p.Data)
	defer cfree(a.mount_data)

	a.flags = C.int(p.Flags)
	a.ro = boolToC(p.Readonly)
	a.fsck = boolToC(p.Fsck)
	a.quota = boolToC(p.Quota)

	ret := C.ploop_mount_image(d.d, &a)
	if ret == 0 {
		device = C.GoString(&a.device[0])
		// TODO? fsck_code = C.GoString(a.fsck_rc)
	}
	return device, mkerr(ret)
}

// Umount unmounts the ploop filesystem and dismantles the device
func (d Ploop) Umount() error {
	ret := C.ploop_umount_image(d.d)

	return mkerr(ret)
}

// UmountByDevice unmounts the ploop filesystem and dismantles the device.
// Unlike Umount(), this is a lower-level function meaning it can be less
// safe and should generally not be used.
func UmountByDevice(dev string) error {
	cdev := C.CString(dev)
	defer cfree(cdev)

	ret := C.ploop_umount(cdev, nil)

	return mkerr(ret)
}

// Resize changes the ploop size. Online resize is recommended.
func (d Ploop) Resize(size uint64, offline bool) error {
	var p C.struct_ploop_resize_param

	p.size = convertSize(size)
	p.offline_resize = boolToC(offline)

	ret := C.ploop_resize_image(d.d, &p)
	return mkerr(ret)
}

// Snapshot creates a ploop snapshot, returning its uuid
func (d Ploop) Snapshot() (string, error) {
	var p C.struct_ploop_snapshot_param
	var uuid, err = UUID()
	if err != nil {
		return "", err
	}
	p.guid = C.CString(uuid)
	defer cfree(p.guid)

	ret := C.ploop_create_snapshot(d.d, &p)
	if ret == 0 {
		uuid = C.GoString(p.guid)
	}

	return uuid, mkerr(ret)
}

// SwitchSnapshot switches to a specified snapshot,
// creates a new empty delta on top of it, and makes it a top one
// (i.e. the one new data will be written to).
// Old top delta (i.e. data modified since the last snapshot) is lost.
func (d Ploop) SwitchSnapshot(uuid string) error {
	var p C.struct_ploop_snapshot_switch_param

	p.guid = C.CString(uuid)
	defer cfree(p.guid)

	ret := C.ploop_switch_snapshot_ex(d.d, &p)

	return mkerr(ret)
}

// SwitchFlag is a type for SwitchSnapshotExtended.Flags
type SwitchFlag uint

const (
	// SkipDestroy flag, if set, modifies the behavior of
	// SwitchSnapshotExtended to not delete the old top delta, but
	// make it a snapshot and return its uuid. Without this flag,
	// old top delta (i.e. data modified since the last snapshot)
	// is lost.
	SkipDestroy SwitchFlag = C.PLOOP_SNAP_SKIP_TOPDELTA_DESTROY
	// SkipCreate flag, if set, modifies the behavior of
	// SwitchSnapshotExtended to not create a new top delta,
	// but rather transform the specified snapshot itself to be
	// the new top delta), so all new changes will be written
	// right to it. Snapshot UUID is lost in this case.
	SkipCreate SwitchFlag = C.PLOOP_SNAP_SKIP_TOPDELTA_CREATE
)

// SwitchSnapshotExtended is same as SwitchSnapshot but with additional
// flags modifying its behavior. Please see individual flags description.
// Returns uuid of what was the old top delta if SkipDestroy flag is set.
func (d Ploop) SwitchSnapshotExtended(uuid string, flags SwitchFlag) (string, error) {
	var p C.struct_ploop_snapshot_switch_param
	oldUUID := ""

	p.guid = C.CString(uuid)
	defer cfree(p.guid)

	p.flags = C.int(flags)

	if flags&SkipDestroy != 0 {
		oldUUID, err := UUID()
		if err != nil {
			return "", err
		}
		p.guid_old = C.CString(oldUUID)
		defer cfree(p.guid_old)
	}

	ret := C.ploop_switch_snapshot_ex(d.d, &p)

	return oldUUID, mkerr(ret)
}

// DeleteSnapshot deletes a snapshot (merging it down if necessary)
func (d Ploop) DeleteSnapshot(uuid string) error {
	cuuid := C.CString(uuid)
	defer cfree(cuuid)

	ret := C.ploop_delete_snapshot(d.d, cuuid)

	return mkerr(ret)
}

// ReplaceFlag is a type for ReplaceParam.Flags field
type ReplaceFlag int

// Possible values for ReplaceParam.Flags field
const (
	// KeepName renames the new file to old file name after replace;
	// note that if this option is used the old file is removed.
	KeepName ReplaceFlag = C.PLOOP_REPLACE_KEEP_NAME
)

// ReplaceParam is a set of parameters to Replace()
type ReplaceParam struct {
	File string // new image file name
	// Image to be replaced is specified by either
	// uuid, current file name, or level,
	// in the above order of preference.
	UUID    string
	CurFile string
	Level   int
	Flags   ReplaceFlag
}

// Replace replaces a ploop image to a different (but identical) one
func (d Ploop) Replace(p *ReplaceParam) error {
	var a C.struct_ploop_replace_param

	a.file = C.CString(p.File)
	defer cfree(a.file)

	if p.UUID != "" {
		a.guid = C.CString(p.UUID)
		defer cfree(a.guid)
	} else if p.CurFile != "" {
		a.cur_file = C.CString(p.CurFile)
		defer cfree(a.cur_file)
	} else {
		a.level = C.int(p.Level)
	}

	a.flags = C.int(p.Flags)

	ret := C.ploop_replace_image(d.d, &a)

	return mkerr(ret)
}

// IsMounted returns true if ploop is mounted
func (d Ploop) IsMounted() (bool, error) {
	ret := C.ploop_is_mounted(d.d)
	if ret == 0 {
		return false, nil
	} else if ret == 1 {
		return true, nil
	} else {
		// error, but no code, make our own
		return false, mkerr(E_SYS)
	}
}

// FSInfoData holds information about ploop inner file system
type FSInfoData struct {
	BlockSize  uint64
	Blocks     uint64
	BlocksFree uint64
	Inodes     uint64
	InodesFree uint64
}

// FSInfo gets info of ploop's inner file system
func FSInfo(file string) (FSInfoData, error) {
	var cinfo C.struct_ploop_info
	var info FSInfoData
	cfile := C.CString(file)
	defer cfree(cfile)

	once.Do(loadKmod)

	ret := C.ploop_get_info_by_descr(cfile, &cinfo)
	if ret == 0 {
		info.BlockSize = uint64(cinfo.fs_bsize)
		info.Blocks = uint64(cinfo.fs_blocks)
		info.BlocksFree = uint64(cinfo.fs_bfree)
		info.Inodes = uint64(cinfo.fs_inodes)
		info.InodesFree = uint64(cinfo.fs_ifree)
	}

	return info, mkerr(ret)
}

// ImageInfoData holds information about ploop image
type ImageInfoData struct {
	Blocks    uint64
	BlockSize uint32
	Version   int
}

// ImageInfo gets information about a ploop image
func (d Ploop) ImageInfo() (ImageInfoData, error) {
	var cinfo C.struct_ploop_spec
	var info ImageInfoData

	ret := C.ploop_get_spec(d.d, &cinfo)
	if ret == 0 {
		info.Blocks = uint64(cinfo.size)
		info.BlockSize = uint32(cinfo.blocksize)
		info.Version = int(cinfo.fmt_version)
	}

	return info, mkerr(ret)
}

// TopDeltaFile returns file name of top delta
func (d Ploop) TopDeltaFile() (string, error) {
	const len = 4096 // PATH_MAX
	var out [len]C.char

	ret := C.ploop_get_top_delta_fname(d.d, &out[0], len)
	if ret != 0 {
		// error, but no code, make our own
		return "", mkerr(E_SYS)
	}

	file := C.GoString(&out[0])
	return file, nil
}

// UUID generates a ploop UUID
func UUID() (string, error) {
	var cuuid [39]C.char

	ret := C.ploop_uuid_generate(&cuuid[0], 39)
	if ret != 0 {
		return "", mkerr(ret)
	}

	uuid := C.GoString(&cuuid[0])
	return uuid, nil
}
<|endoftext|>