<commit_before>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package lib

import (
	"context"
	"io/ioutil"
	"path/filepath"
	"testing"

	"github.com/bazelbuild/remote-apis-sdks/go/pkg/client"
	"github.com/bazelbuild/remote-apis-sdks/go/pkg/fakes"
	. "github.com/smartystreets/goconvey/convey"

	"go.chromium.org/luci/auth"
	"go.chromium.org/luci/common/testing/testfs"
)

func TestArchiveDownload(t *testing.T) {
	ctx := context.Background()

	Convey(`Upload and download`, t, func() {
		fakeEnv, cleanup := fakes.NewTestEnv(t)
		t.Cleanup(cleanup)

		newCasClient = func(ctx context.Context, instance string, opts auth.Options, readOnly bool) (*client.Client, error) {
			return fakeEnv.Server.NewTestClient(ctx)
		}

		uploaded := t.TempDir()
		layout := map[string]string{
			"a":      "a",
			"b/c":    "bc",
			"empty/": "",
		}
		So(testfs.Build(uploaded, layout), ShouldBeNil)

		var ar archiveRun
		ar.dumpDigest = filepath.Join(t.TempDir(), "digest")
		So(ar.paths.Set(uploaded+":."), ShouldBeNil)
		So(ar.doArchive(ctx), ShouldBeNil)

		digest, err := ioutil.ReadFile(ar.dumpDigest)
		So(err, ShouldBeNil)

		var dr downloadRun
		dr.digest = string(digest)
		dr.dir = t.TempDir()
		err = dr.doDownload(ctx)
		So(err, ShouldBeNil)

		downloaded, err := testfs.Collect(dr.dir)
		So(err, ShouldBeNil)
		So(downloaded, ShouldResemble, layout)
	})
}
<commit_msg>[cas] run test in parallel<commit_after>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package lib

import (
	"context"
	"io/ioutil"
	"path/filepath"
	"testing"

	"github.com/bazelbuild/remote-apis-sdks/go/pkg/client"
	"github.com/bazelbuild/remote-apis-sdks/go/pkg/fakes"
	. "github.com/smartystreets/goconvey/convey"

	"go.chromium.org/luci/auth"
	"go.chromium.org/luci/common/testing/testfs"
)

func TestArchiveDownload(t *testing.T) {
	t.Parallel()
	ctx := context.Background()

	Convey(`Upload and download`, t, func() {
		fakeEnv, cleanup := fakes.NewTestEnv(t)
		t.Cleanup(cleanup)

		newCasClient = func(ctx context.Context, instance string, opts auth.Options, readOnly bool) (*client.Client, error) {
			return fakeEnv.Server.NewTestClient(ctx)
		}

		uploaded := t.TempDir()
		layout := map[string]string{
			"a":      "a",
			"b/c":    "bc",
			"empty/": "",
		}
		So(testfs.Build(uploaded, layout), ShouldBeNil)

		var ar archiveRun
		ar.dumpDigest = filepath.Join(t.TempDir(), "digest")
		So(ar.paths.Set(uploaded+":."), ShouldBeNil)
		So(ar.doArchive(ctx), ShouldBeNil)

		digest, err := ioutil.ReadFile(ar.dumpDigest)
		So(err, ShouldBeNil)

		var dr downloadRun
		dr.digest = string(digest)
		dr.dir = t.TempDir()
		err = dr.doDownload(ctx)
		So(err, ShouldBeNil)

		downloaded, err := testfs.Collect(dr.dir)
		So(err, ShouldBeNil)
		So(downloaded, ShouldResemble, layout)
	})
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Jetstack cert-manager contributors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package chart

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path"

	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"github.com/jetstack/cert-manager/test/e2e/framework/addon/tiller"
	"github.com/jetstack/cert-manager/test/e2e/framework/config"
	"github.com/jetstack/cert-manager/test/e2e/framework/log"
)

// Chart is a generic Helm chart addon for the test environment
type Chart struct {
	config        *config.Config
	tillerDetails *tiller.Details
	// temporary directory used as the --home flag to Helm
	home string

	// Tiller is the tiller instance to submit the release to
	Tiller *tiller.Tiller

	// ReleaseName for this Helm release
	// `helm install --name {{ReleaseName}}`
	ReleaseName string

	// Namespace for the Helm release
	// `helm install --namespace {{Namespace}}`
	Namespace string

	// ChartName is the name of the chart to deploy
	// `helm install {{ChartName}}``
	ChartName string

	// ChartVersion is the version of the chart to deploy
	// `helm install --version {{ChartVersion}}`
	ChartVersion string

	// Vars are additional --set arguments for helm install
	// `helm install --set {{Vars[0].Key}}={{Vars[0].Value}} --set {{Vars[1].Key}}={{Vars[1].Value}} ...`
	Vars []StringTuple

	// Values is a list of paths to additional values.yaml files to include
	// `helm install --values {{Values[0]}} --values {{Values[1]}} ...`
	Values []string

	// If UpdateDeps is true, 'helm dep update' will be run against the chart
	// before installing.
	// This should only be set to true when the ChartName is a local path on disk.
	UpdateDeps bool
}

// StringTuple is a tuple of strings, used to create ordered maps
type StringTuple struct {
	Key   string
	Value string
}

// Details return the details about the Tiller instance deployed
type Details struct {
	// Helm chart release name
	ReleaseName string

	// Namespace that Tiller has been deployed into
	Namespace string
}

func (c *Chart) Setup(cfg *config.Config) error {
	var err error

	c.config = cfg
	if c.config.Addons.Helm.Path == "" {
		return fmt.Errorf("--helm-binary-path must be set")
	}
	if c.Tiller == nil {
		return fmt.Errorf("tiller base addon must be provided")
	}
	c.tillerDetails, err = c.Tiller.Details()
	if err != nil {
		return err
	}

	c.home, err = ioutil.TempDir("", "helm-chart-install")
	if err != nil {
		return err
	}

	return nil
}

// Provision an instance of tiller-deploy
func (c *Chart) Provision() error {
	err := c.runHelmClientInit()
	if err != nil {
		return fmt.Errorf("error running 'helm init': %v", err)
	}

	if c.UpdateDeps {
		err := c.runDepUpdate()
		if err != nil {
			return fmt.Errorf("error updating helm chart dependencies: %v", err)
		}
	}

	err = c.runInstall()
	if err != nil {
		return fmt.Errorf("error install helm chart: %v", err)
	}

	err = c.Tiller.Base.Details().Helper().WaitForAllPodsRunningInNamespace(c.Namespace)
	if err != nil {
		return err
	}

	return nil
}

func (c *Chart) runHelmClientInit() error {
	err := c.buildHelmCmd("init", "--client-only").Run()
	if err != nil {
		return err
	}
	return nil
}

func (c *Chart) runDepUpdate() error {
	err := c.buildHelmCmd("dep", "update", c.ChartName).Run()
	if err != nil {
		return err
	}
	return nil
}

func (c *Chart) runInstall() error {
	args := []string{"install", c.ChartName,
		"--wait",
		"--namespace", c.Namespace,
		"--name", c.ReleaseName,
		"--version", c.ChartVersion}

	for _, v := range c.Values {
		args = append(args, "--values", v)
	}

	for _, s := range c.Vars {
		args = append(args, "--set", fmt.Sprintf("%s=%s", s.Key, s.Value))
	}

	cmd := c.buildHelmCmd(args...)
	err := cmd.Run()
	if err != nil {
		return err
	}

	return nil
}

func (c *Chart) buildHelmCmd(args ...string) *exec.Cmd {
	args = append([]string{
		"--home", c.home,
		"--kubeconfig", c.tillerDetails.KubeConfig,
		"--kube-context", c.tillerDetails.KubeContext,
		"--tiller-namespace", c.tillerDetails.Namespace,
	}, args...)
	cmd := exec.Command(c.config.Addons.Helm.Path, args...)
	cmd.Stdout = log.Writer
	cmd.Stderr = log.Writer
	return cmd
}

func (c *Chart) getHelmVersion() (string, error) {
	cmd := c.buildHelmCmd("version", "--template", "{{.Client.Version}}")
	cmd.Stdout = nil
	out, err := cmd.StdoutPipe()
	if err != nil {
		return "", err
	}
	defer out.Close()

	err = cmd.Run()
	if err != nil {
		return "", err
	}

	outBytes, err := ioutil.ReadAll(out)
	if err != nil {
		return "", err
	}

	return string(outBytes), nil
}

// Deprovision the deployed instance of tiller-deploy
func (c *Chart) Deprovision() error {
	err := c.buildHelmCmd("delete", "--purge", c.ReleaseName).Run()
	if err != nil {
		// Ignore deprovisioning errors
		// TODO: only ignore failed to delete because it doesn't exist errors
		return nil
	}

	// attempt to cleanup
	os.RemoveAll(c.home)

	// TODO: delete namespace manually too
	return nil
}

// Details must be possible to compute without Provision being called if we want
// to be able to provision global/shared instances of Tiller.
func (c *Chart) Details() (*Details, error) {
	d := &Details{
		ReleaseName: c.ReleaseName,
		Namespace:   c.Namespace,
	}

	return d, nil
}

func (c *Chart) SupportsGlobal() bool {
	// We can't run in global mode if the release name is not set, as there's
	// no way for us to communicate the generated release name to other test
	// runners when running in parallel mode.
	if c.ReleaseName == "" {
		return false
	}

	return true
}

func (c *Chart) Logs() (map[string]string, error) {
	kc := c.Tiller.Base.Details().KubeClient
	pods, err := kc.CoreV1().Pods(c.Namespace).List(metav1.ListOptions{LabelSelector: "release=" + c.ReleaseName})
	if err != nil {
		return nil, err
	}

	out := make(map[string]string)
	for _, pod := range pods.Items {
		// Only grab logs from the first container in the pod
		// TODO: grab logs from all containers
		containerName := pod.Spec.Containers[0].Name
		resp := kc.CoreV1().Pods(pod.Namespace).GetLogs(pod.Name, &corev1.PodLogOptions{
			Container: containerName,
		}).Do()

		err := resp.Error()
		if err != nil {
			continue
		}

		logs, err := resp.Raw()
		if err != nil {
			continue
		}

		outPath := path.Join(c.Namespace, pod.Name)
		out[outPath] = string(logs)
	}

	return out, nil
}
<commit_msg>Fix fetching logs in e2e test framework<commit_after>/*
Copyright 2019 The Jetstack cert-manager contributors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package chart

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path"

	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"github.com/jetstack/cert-manager/test/e2e/framework/addon/tiller"
	"github.com/jetstack/cert-manager/test/e2e/framework/config"
	"github.com/jetstack/cert-manager/test/e2e/framework/log"
)

// Chart is a generic Helm chart addon for the test environment
type Chart struct {
	config        *config.Config
	tillerDetails *tiller.Details
	// temporary directory used as the --home flag to Helm
	home string

	// Tiller is the tiller instance to submit the release to
	Tiller *tiller.Tiller

	// ReleaseName for this Helm release
	// `helm install --name {{ReleaseName}}`
	ReleaseName string

	// Namespace for the Helm release
	// `helm install --namespace {{Namespace}}`
	Namespace string

	// ChartName is the name of the chart to deploy
	// `helm install {{ChartName}}``
	ChartName string

	// ChartVersion is the version of the chart to deploy
	// `helm install --version {{ChartVersion}}`
	ChartVersion string

	// Vars are additional --set arguments for helm install
	// `helm install --set {{Vars[0].Key}}={{Vars[0].Value}} --set {{Vars[1].Key}}={{Vars[1].Value}} ...`
	Vars []StringTuple

	// Values is a list of paths to additional values.yaml files to include
	// `helm install --values {{Values[0]}} --values {{Values[1]}} ...`
	Values []string

	// If UpdateDeps is true, 'helm dep update' will be run against the chart
	// before installing.
	// This should only be set to true when the ChartName is a local path on disk.
	UpdateDeps bool
}

// StringTuple is a tuple of strings, used to create ordered maps
type StringTuple struct {
	Key   string
	Value string
}

// Details return the details about the Tiller instance deployed
type Details struct {
	// Helm chart release name
	ReleaseName string

	// Namespace that Tiller has been deployed into
	Namespace string
}

func (c *Chart) Setup(cfg *config.Config) error {
	var err error

	c.config = cfg
	if c.config.Addons.Helm.Path == "" {
		return fmt.Errorf("--helm-binary-path must be set")
	}
	if c.Tiller == nil {
		return fmt.Errorf("tiller base addon must be provided")
	}
	c.tillerDetails, err = c.Tiller.Details()
	if err != nil {
		return err
	}

	c.home, err = ioutil.TempDir("", "helm-chart-install")
	if err != nil {
		return err
	}

	return nil
}

// Provision an instance of tiller-deploy
func (c *Chart) Provision() error {
	err := c.runHelmClientInit()
	if err != nil {
		return fmt.Errorf("error running 'helm init': %v", err)
	}

	if c.UpdateDeps {
		err := c.runDepUpdate()
		if err != nil {
			return fmt.Errorf("error updating helm chart dependencies: %v", err)
		}
	}

	err = c.runInstall()
	if err != nil {
		return fmt.Errorf("error install helm chart: %v", err)
	}

	err = c.Tiller.Base.Details().Helper().WaitForAllPodsRunningInNamespace(c.Namespace)
	if err != nil {
		return err
	}

	return nil
}

func (c *Chart) runHelmClientInit() error {
	err := c.buildHelmCmd("init", "--client-only").Run()
	if err != nil {
		return err
	}
	return nil
}

func (c *Chart) runDepUpdate() error {
	err := c.buildHelmCmd("dep", "update", c.ChartName).Run()
	if err != nil {
		return err
	}
	return nil
}

func (c *Chart) runInstall() error {
	args := []string{"install", c.ChartName,
		"--wait",
		"--namespace", c.Namespace,
		"--name", c.ReleaseName,
		"--version", c.ChartVersion}

	for _, v := range c.Values {
		args = append(args, "--values", v)
	}

	for _, s := range c.Vars {
		args = append(args, "--set", fmt.Sprintf("%s=%s", s.Key, s.Value))
	}

	cmd := c.buildHelmCmd(args...)
	err := cmd.Run()
	if err != nil {
		return err
	}

	return nil
}

func (c *Chart) buildHelmCmd(args ...string) *exec.Cmd {
	args = append([]string{
		"--home", c.home,
		"--kubeconfig", c.tillerDetails.KubeConfig,
		"--kube-context", c.tillerDetails.KubeContext,
		"--tiller-namespace", c.tillerDetails.Namespace,
	}, args...)
	cmd := exec.Command(c.config.Addons.Helm.Path, args...)
	cmd.Stdout = log.Writer
	cmd.Stderr = log.Writer
	return cmd
}

func (c *Chart) getHelmVersion() (string, error) {
	cmd := c.buildHelmCmd("version", "--template", "{{.Client.Version}}")
	cmd.Stdout = nil
	out, err := cmd.StdoutPipe()
	if err != nil {
		return "", err
	}
	defer out.Close()

	err = cmd.Run()
	if err != nil {
		return "", err
	}

	outBytes, err := ioutil.ReadAll(out)
	if err != nil {
		return "", err
	}

	return string(outBytes), nil
}

// Deprovision the deployed instance of tiller-deploy
func (c *Chart) Deprovision() error {
	err := c.buildHelmCmd("delete", "--purge", c.ReleaseName).Run()
	if err != nil {
		// Ignore deprovisioning errors
		// TODO: only ignore failed to delete because it doesn't exist errors
		return nil
	}

	// attempt to cleanup
	os.RemoveAll(c.home)

	// TODO: delete namespace manually too
	return nil
}

// Details must be possible to compute without Provision being called if we want
// to be able to provision global/shared instances of Tiller.
func (c *Chart) Details() (*Details, error) {
	d := &Details{
		ReleaseName: c.ReleaseName,
		Namespace:   c.Namespace,
	}

	return d, nil
}

func (c *Chart) SupportsGlobal() bool {
	// We can't run in global mode if the release name is not set, as there's
	// no way for us to communicate the generated release name to other test
	// runners when running in parallel mode.
	if c.ReleaseName == "" {
		return false
	}

	return true
}

func (c *Chart) Logs() (map[string]string, error) {
	kc := c.Tiller.Base.Details().KubeClient
	oldLabelPods, err := kc.CoreV1().Pods(c.Namespace).List(metav1.ListOptions{LabelSelector: "release=" + c.ReleaseName})
	if err != nil {
		return nil, err
	}

	// also check pods with the new style labels used in the cert-manager chart
	newLabelPods, err := kc.CoreV1().Pods(c.Namespace).List(metav1.ListOptions{LabelSelector: "app.kubernetes.io/instance=" + c.ReleaseName})
	if err != nil {
		return nil, err
	}
	podList := append(oldLabelPods.Items, newLabelPods.Items...)

	out := make(map[string]string)
	for _, pod := range podList {
		// Only grab logs from the first container in the pod
		// TODO: grab logs from all containers
		containerName := pod.Spec.Containers[0].Name
		resp := kc.CoreV1().Pods(pod.Namespace).GetLogs(pod.Name, &corev1.PodLogOptions{
			Container: containerName,
		}).Do()

		err := resp.Error()
		if err != nil {
			continue
		}

		logs, err := resp.Raw()
		if err != nil {
			continue
		}

		outPath := path.Join(c.Namespace, pod.Name)
		out[outPath] = string(logs)
	}

	return out, nil
}
<|endoftext|>


<commit_before>package kinesis

import (
	"encoding/json"
	"fmt"
	"github.com/AdRoll/goamz/aws"
	kin "github.com/AdRoll/goamz/kinesis"
	"github.com/mozilla-services/heka/pipeline"
	"time"
)

type KinesisOutput struct {
	auth   aws.Auth
	config *KinesisOutputConfig
	Client *kin.Kinesis
}

type KinesisOutputConfig struct {
	Region          string `toml:"region"`
	Stream          string `toml:"stream"`
	AccessKeyID     string `toml:"access_key_id"`
	SecretAccessKey string `toml:"secret_access_key"`
	Token           string `toml:"token"`
	PayloadOnly     bool   `toml:"payload_only"`
}

func (k *KinesisOutput) ConfigStruct() interface{} {
	return &KinesisOutputConfig{
		Region:          "us-east-1",
		Stream:          "",
		AccessKeyID:     "",
		SecretAccessKey: "",
		Token:           "",
	}
}

func (k *KinesisOutput) Init(config interface{}) error {
	k.config = config.(*KinesisOutputConfig)
	a, err := aws.GetAuth(k.config.AccessKeyID, k.config.SecretAccessKey, k.config.Token, time.Now())
	if err != nil {
		return fmt.Errorf("Error authenticating: %s", err)
	}
	k.auth = a

	region, ok := aws.Regions[k.config.Region]
	if !ok {
		return fmt.Errorf("Region does not exist: %s", k.config.Region)
	}

	k.Client = kin.New(k.auth, region)

	return nil
}

func (k *KinesisOutput) Run(or pipeline.OutputRunner, helper pipeline.PluginHelper) error {
	var (
		pack     *pipeline.PipelinePack
		contents []byte
		msg      []byte
		err      error
	)

	for pack = range or.InChan() {
		msg, err = or.Encode(pack)
		if err != nil {
			or.LogError(fmt.Errorf("Error encoding message: %s", err))
			pack.Recycle()
			continue
		}
		if contents, err = json.Marshal(msg); err != nil {
			or.LogError(fmt.Errorf("Error marshalling: %s", err))
			pack.Recycle()
			continue
		} else {
			pk := fmt.Sprintf("%d-%s", pack.Message.Timestamp, pack.Message.Hostname)
			_, err = k.Client.PutRecord(k.config.Stream, pk, contents, "", "")
			if err != nil {
				or.LogError(fmt.Errorf("Error pushing message to Kinesis: %s", err))
				pack.Recycle()
				continue
			}
		}
		pack.Recycle()
	}

	return nil
}

func init() {
	pipeline.RegisterPlugin("KinesisOutput", func() interface{} { return new(KinesisOutput) })
}
<commit_msg>base64 decode the message<commit_after>package kinesis

import (
	"encoding/base64"
	"encoding/json"
	"fmt"
	"github.com/AdRoll/goamz/aws"
	kin "github.com/AdRoll/goamz/kinesis"
	"github.com/mozilla-services/heka/pipeline"
	"time"
)

type KinesisOutput struct {
	auth   aws.Auth
	config *KinesisOutputConfig
	Client *kin.Kinesis
}

type KinesisOutputConfig struct {
	Region          string `toml:"region"`
	Stream          string `toml:"stream"`
	AccessKeyID     string `toml:"access_key_id"`
	SecretAccessKey string `toml:"secret_access_key"`
	Token           string `toml:"token"`
	PayloadOnly     bool   `toml:"payload_only"`
}

func (k *KinesisOutput) ConfigStruct() interface{} {
	return &KinesisOutputConfig{
		Region:          "us-east-1",
		Stream:          "",
		AccessKeyID:     "",
		SecretAccessKey: "",
		Token:           "",
	}
}

func (k *KinesisOutput) Init(config interface{}) error {
	k.config = config.(*KinesisOutputConfig)
	a, err := aws.GetAuth(k.config.AccessKeyID, k.config.SecretAccessKey, k.config.Token, time.Now())
	if err != nil {
		return fmt.Errorf("Error authenticating: %s", err)
	}
	k.auth = a

	region, ok := aws.Regions[k.config.Region]
	if !ok {
		return fmt.Errorf("Region does not exist: %s", k.config.Region)
	}

	k.Client = kin.New(k.auth, region)

	return nil
}

func (k *KinesisOutput) Run(or pipeline.OutputRunner, helper pipeline.PluginHelper) error {
	var (
		pack     *pipeline.PipelinePack
		contents []byte
		msg      []byte
		err      error
	)

	for pack = range or.InChan() {
		msg, err = or.Encode(pack)
		if err != nil {
			or.LogError(fmt.Errorf("Error encoding message: %s", err))
			pack.Recycle()
			continue
		}
		msg, err = base64.StdEncoding.DecodeString(string(msg))
		if err != nil {
			or.LogError(fmt.Errorf("Error decoding: %s", err))
			pack.Recycle()
			continue
		}
		if contents, err = json.Marshal(msg); err != nil {
			or.LogError(fmt.Errorf("Error marshalling: %s", err))
			pack.Recycle()
			continue
		} else {
			pk := fmt.Sprintf("%d-%s", pack.Message.Timestamp, pack.Message.Hostname)
			_, err = k.Client.PutRecord(k.config.Stream, pk, contents, "", "")
			if err != nil {
				or.LogError(fmt.Errorf("Error pushing message to Kinesis: %s", err))
				pack.Recycle()
				continue
			}
		}
		pack.Recycle()
	}

	return nil
}

func init() {
	pipeline.RegisterPlugin("KinesisOutput", func() interface{} { return new(KinesisOutput) })
}
<|endoftext|>


<commit_before>package cockroach

import (
	"database/sql"
	"fmt"

	"github.com/pressly/goose"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/rs/zerolog/log"

	// use postgres client library to connect to cockroachdb
	_ "github.com/lib/pq"
)

// DBClient is the interface for communicating with CockroachDB
type DBClient interface {
	Connect() error
	ConnectWithDriverAndSource(string, string) error
	MigrateSchema() error
	InsertBuildJobLogs(BuildJobLogs) error
	GetBuildLogs(BuildJobLogs) error
}

type cockroachDBClientImpl struct {
	databaseDriver                  string
	migrationsDir                   string
	cockroachDatabase               string
	cockroachHost                   string
	cockroachInsecure               bool
	cockroachCertificateDir         string
	cockroachPort                   int
	cockroachUser                   string
	cockroachPassword               string
	PrometheusOutboundAPICallTotals *prometheus.CounterVec
	databaseConnection              *sql.DB
}

// NewCockroachDBClient returns a new cockroach.DBClient
func NewCockroachDBClient(cockroachDatabase, cockroachHost string, cockroachInsecure bool, cockroachCertificateDir string, cockroachPort int, cockroachUser, cockroachPassword string, prometheusOutboundAPICallTotals *prometheus.CounterVec) (cockroachDBClient DBClient) {

	cockroachDBClient = &cockroachDBClientImpl{
		databaseDriver:                  "postgres",
		migrationsDir:                   "/migrations",
		cockroachDatabase:               cockroachDatabase,
		cockroachHost:                   cockroachHost,
		cockroachInsecure:               cockroachInsecure,
		cockroachCertificateDir:         cockroachCertificateDir,
		cockroachPort:                   cockroachPort,
		cockroachUser:                   cockroachUser,
		cockroachPassword:               cockroachPassword,
		PrometheusOutboundAPICallTotals: prometheusOutboundAPICallTotals,
	}

	return
}

// Connect sets up a connection with CockroachDB
func (dbc *cockroachDBClientImpl) Connect() (err error) {

	log.Debug().Msgf("Connecting to database %v on host %v...", dbc.cockroachDatabase, dbc.cockroachHost)

	sslMode := ""
	if dbc.cockroachInsecure {
		sslMode = "?sslmode=disable"
	}

	dataSourceName := fmt.Sprintf("postgresql://%v:%v@%v:%v/%v%v", dbc.cockroachUser, dbc.cockroachPassword, dbc.cockroachHost, dbc.cockroachPort, dbc.cockroachDatabase, sslMode)

	return dbc.ConnectWithDriverAndSource(dbc.databaseDriver, dataSourceName)
}

// ConnectWithDriverAndSource set up a connection with any database
func (dbc *cockroachDBClientImpl) ConnectWithDriverAndSource(driverName string, dataSourceName string) (err error) {

	dbc.databaseConnection, err = sql.Open(driverName, dataSourceName)
	if err != nil {
		return
	}

	return
}

// MigrateSchema migrates the schema in CockroachDB
func (dbc *cockroachDBClientImpl) MigrateSchema() (err error) {

	err = goose.SetDialect(dbc.databaseDriver)
	if err != nil {
		return err
	}

	err = goose.Status(dbc.databaseConnection, dbc.migrationsDir)
	if err != nil {
		return err
	}

	err = goose.Up(dbc.databaseConnection, dbc.migrationsDir)
	if err != nil {
		return err
	}

	return
}

// InsertBuildJobLogs inserts build logs into the database
func (dbc *cockroachDBClientImpl) InsertBuildJobLogs(buildJobLogs BuildJobLogs) (err error) {

	dbc.PrometheusOutboundAPICallTotals.With(prometheus.Labels{"target": "cockroachdb"}).Inc()

	// insert logs
	r, err := dbc.databaseConnection.Exec(
		"INSERT INTO build_logs (repo_full_name,repo_branch,repo_revision,repo_source,log_text) VALUES ($1,$2,$3,$4,$5)",
		buildJobLogs.RepoFullName,
		buildJobLogs.RepoBranch,
		buildJobLogs.RepoRevision,
		buildJobLogs.RepoSource,
		buildJobLogs.LogText,
	)

	if err != nil {
		return
	}

	lastInsertID, err := r.LastInsertId()
	if err != nil {
		return
	}

	rowsAffected, err := r.RowsAffected()
	if err != nil {
		return
	}

	log.Debug().
		Int64("LastInsertId", lastInsertID).
		Int64("RowsAffected", rowsAffected).
		Msg("Inserted log record")

	return
}

// GetBuildJobLogs reads a specific log
func (dbc *cockroachDBClientImpl) GetBuildLogs(buildJobLogs BuildJobLogs) (err error) {

	rows, err := dbc.databaseConnection.Query("SELECT * FROM build_logs WHERE repo_full_name=$1 AND repo_branch=$2 AND repo_revision=$3 AND repo_source=$4",
		buildJobLogs.RepoFullName,
		buildJobLogs.RepoBranch,
		buildJobLogs.RepoRevision,
		buildJobLogs.RepoSource,
	)
	if err != nil {
		return
	}

	defer rows.Close()
	for rows.Next() {
		var id int
		var repo_full_name, repo_branch, repo_revision, repo_source, log_text string

		if err := rows.Scan(&id, &repo_full_name, &repo_branch, &repo_revision, &repo_source, &log_text); err != nil {
			return err
		}

		log.Debug().
			Int("id", id).
			Str("repo_full_name", repo_full_name).
			Str("repo_branch", repo_branch).
			Str("repo_revision", repo_revision).
			Str("repo_source", repo_source).
			Str("log_text", log_text).
			Msg("Read logs")
	}

	return
}
<commit_msg>warn if getting LastInsertId or RowsAffected fails, but don't error<commit_after>package cockroach

import (
	"database/sql"
	"fmt"

	"github.com/pressly/goose"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/rs/zerolog/log"

	// use postgres client library to connect to cockroachdb
	_ "github.com/lib/pq"
)

// DBClient is the interface for communicating with CockroachDB
type DBClient interface {
	Connect() error
	ConnectWithDriverAndSource(string, string) error
	MigrateSchema() error
	InsertBuildJobLogs(BuildJobLogs) error
	GetBuildLogs(BuildJobLogs) error
}

type cockroachDBClientImpl struct {
	databaseDriver                  string
	migrationsDir                   string
	cockroachDatabase               string
	cockroachHost                   string
	cockroachInsecure               bool
	cockroachCertificateDir         string
	cockroachPort                   int
	cockroachUser                   string
	cockroachPassword               string
	PrometheusOutboundAPICallTotals *prometheus.CounterVec
	databaseConnection              *sql.DB
}

// NewCockroachDBClient returns a new cockroach.DBClient
func NewCockroachDBClient(cockroachDatabase, cockroachHost string, cockroachInsecure bool, cockroachCertificateDir string, cockroachPort int, cockroachUser, cockroachPassword string, prometheusOutboundAPICallTotals *prometheus.CounterVec) (cockroachDBClient DBClient) {

	cockroachDBClient = &cockroachDBClientImpl{
		databaseDriver:                  "postgres",
		migrationsDir:                   "/migrations",
		cockroachDatabase:               cockroachDatabase,
		cockroachHost:                   cockroachHost,
		cockroachInsecure:               cockroachInsecure,
		cockroachCertificateDir:         cockroachCertificateDir,
		cockroachPort:                   cockroachPort,
		cockroachUser:                   cockroachUser,
		cockroachPassword:               cockroachPassword,
		PrometheusOutboundAPICallTotals: prometheusOutboundAPICallTotals,
	}

	return
}

// Connect sets up a connection with CockroachDB
func (dbc *cockroachDBClientImpl) Connect() (err error) {

	log.Debug().Msgf("Connecting to database %v on host %v...", dbc.cockroachDatabase, dbc.cockroachHost)

	sslMode := ""
	if dbc.cockroachInsecure {
		sslMode = "?sslmode=disable"
	}

	dataSourceName := fmt.Sprintf("postgresql://%v:%v@%v:%v/%v%v", dbc.cockroachUser, dbc.cockroachPassword, dbc.cockroachHost, dbc.cockroachPort, dbc.cockroachDatabase, sslMode)

	return dbc.ConnectWithDriverAndSource(dbc.databaseDriver, dataSourceName)
}

// ConnectWithDriverAndSource set up a connection with any database
func (dbc *cockroachDBClientImpl) ConnectWithDriverAndSource(driverName string, dataSourceName string) (err error) {

	dbc.databaseConnection, err = sql.Open(driverName, dataSourceName)
	if err != nil {
		return
	}

	return
}

// MigrateSchema migrates the schema in CockroachDB
func (dbc *cockroachDBClientImpl) MigrateSchema() (err error) {

	err = goose.SetDialect(dbc.databaseDriver)
	if err != nil {
		return err
	}

	err = goose.Status(dbc.databaseConnection, dbc.migrationsDir)
	if err != nil {
		return err
	}

	err = goose.Up(dbc.databaseConnection, dbc.migrationsDir)
	if err != nil {
		return err
	}

	return
}

// InsertBuildJobLogs inserts build logs into the database
func (dbc *cockroachDBClientImpl) InsertBuildJobLogs(buildJobLogs BuildJobLogs) (err error) {

	dbc.PrometheusOutboundAPICallTotals.With(prometheus.Labels{"target": "cockroachdb"}).Inc()

	// insert logs
	r, err := dbc.databaseConnection.Exec(
		"INSERT INTO build_logs (repo_full_name,repo_branch,repo_revision,repo_source,log_text) VALUES ($1,$2,$3,$4,$5)",
		buildJobLogs.RepoFullName,
		buildJobLogs.RepoBranch,
		buildJobLogs.RepoRevision,
		buildJobLogs.RepoSource,
		buildJobLogs.LogText,
	)

	if err != nil {
		return
	}

	lastInsertID, err := r.LastInsertId()
	if err != nil {
		log.Warn().Err(err).
			Interface("buildJobLogs", buildJobLogs).
			Msgf("Getting LastInsertId for %v failed", buildJobLogs.RepoFullName)
	}

	rowsAffected, err := r.RowsAffected()
	if err != nil {
		log.Warn().Err(err).
			Interface("buildJobLogs", buildJobLogs).
			Msgf("Getting RowsAffected for %v failed", buildJobLogs.RepoFullName)
	}

	log.Debug().
		Int64("LastInsertId", lastInsertID).
		Int64("RowsAffected", rowsAffected).
		Msg("Inserted log record")

	return
}

// GetBuildJobLogs reads a specific log
func (dbc *cockroachDBClientImpl) GetBuildLogs(buildJobLogs BuildJobLogs) (err error) {

	rows, err := dbc.databaseConnection.Query("SELECT * FROM build_logs WHERE repo_full_name=$1 AND repo_branch=$2 AND repo_revision=$3 AND repo_source=$4",
		buildJobLogs.RepoFullName,
		buildJobLogs.RepoBranch,
		buildJobLogs.RepoRevision,
		buildJobLogs.RepoSource,
	)
	if err != nil {
		return
	}

	defer rows.Close()
	for rows.Next() {
		var id int
		var repo_full_name, repo_branch, repo_revision, repo_source, log_text string

		if err := rows.Scan(&id, &repo_full_name, &repo_branch, &repo_revision, &repo_source, &log_text); err != nil {
			return err
		}

		log.Debug().
			Int("id", id).
			Str("repo_full_name", repo_full_name).
			Str("repo_branch", repo_branch).
			Str("repo_revision", repo_revision).
			Str("repo_source", repo_source).
			Str("log_text", log_text).
			Msg("Read logs")
	}

	return
}
<|endoftext|>


<commit_before>package handlers

import (
	"crypto/tls"
	"fmt"
	"log"
	"net/http"
	"time"

	"golang.org/x/crypto/acme/autocert"

	"github.com/googollee/go-socket.io"
	gh "github.com/gorilla/handlers"
	"github.com/gorilla/mux"
	"github.com/play-with-docker/play-with-docker/config"
	"github.com/play-with-docker/play-with-docker/event"
	"github.com/play-with-docker/play-with-docker/pwd"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/urfave/negroni"
)

var core pwd.PWDApi
var e event.EventApi
var ws *socketio.Server

type HandlerExtender func(h *mux.Router)

func Bootstrap(c pwd.PWDApi, ev event.EventApi) {
	core = c
	e = ev
}

func Register(extend HandlerExtender) {
	server, err := socketio.NewServer(nil)
	if err != nil {
		log.Fatal(err)
	}
	server.On("connection", WS)
	server.On("error", WSError)

	RegisterEvents(server)

	r := mux.NewRouter()
	corsRouter := mux.NewRouter()

	corsHandler := gh.CORS(gh.AllowCredentials(), gh.AllowedHeaders([]string{"x-requested-with", "content-type"}), gh.AllowedOrigins([]string{"*"}))

	// Specific routes
	r.HandleFunc("/ping", Ping).Methods("GET")
	corsRouter.HandleFunc("/instances/images", GetInstanceImages).Methods("GET")
	corsRouter.HandleFunc("/sessions/{sessionId}", GetSession).Methods("GET")
	corsRouter.HandleFunc("/sessions/{sessionId}", CloseSession).Methods("DELETE")
	corsRouter.HandleFunc("/sessions/{sessionId}/setup", SessionSetup).Methods("POST")
	corsRouter.HandleFunc("/sessions/{sessionId}/instances", NewInstance).Methods("POST")
	corsRouter.HandleFunc("/sessions/{sessionId}/instances/{instanceName}/uploads", FileUpload).Methods("POST")
	corsRouter.HandleFunc("/sessions/{sessionId}/instances/{instanceName}", DeleteInstance).Methods("DELETE")
	corsRouter.HandleFunc("/sessions/{sessionId}/instances/{instanceName}/exec", Exec).Methods("POST")

	r.HandleFunc("/ooc", func(rw http.ResponseWriter, r *http.Request) {
		http.ServeFile(rw, r, "./www/ooc.html")
	}).Methods("GET")
	r.HandleFunc("/503", func(rw http.ResponseWriter, r *http.Request) {
		http.ServeFile(rw, r, "./www/503.html")
	}).Methods("GET")
	r.HandleFunc("/p/{sessionId}", Home).Methods("GET")
	r.PathPrefix("/assets").Handler(http.FileServer(http.Dir("./www")))
	r.HandleFunc("/robots.txt", func(rw http.ResponseWriter, r *http.Request) {
		http.ServeFile(rw, r, "www/robots.txt")
	})
	r.HandleFunc("/sdk.js", func(rw http.ResponseWriter, r *http.Request) {
		http.ServeFile(rw, r, "www/sdk.js")
	})

	corsRouter.Handle("/sessions/{sessionId}/ws/", server)
	r.Handle("/metrics", promhttp.Handler())

	// Generic routes
	r.HandleFunc("/", func(rw http.ResponseWriter, r *http.Request) {
		http.ServeFile(rw, r, "./www/landing.html")
	}).Methods("GET")

	r.HandleFunc("/users/me", LoggedInUser).Methods("GET")
	r.HandleFunc("/oauth/providers", ListProviders).Methods("GET")
	r.HandleFunc("/oauth/providers/{provider}/login", Login).Methods("GET")
	r.HandleFunc("/oauth/providers/{provider}/callback", LoginCallback).Methods("GET")

	corsRouter.HandleFunc("/", NewSession).Methods("POST")

	if extend != nil {
		extend(corsRouter)
	}

	n := negroni.Classic()
	r.PathPrefix("/").Handler(negroni.New(negroni.Wrap(corsHandler(corsRouter))))
	n.UseHandler(r)

	httpServer := http.Server{
		Addr:              "0.0.0.0:" + config.PortNumber,
		Handler:           n,
		IdleTimeout:       30 * time.Second,
		ReadHeaderTimeout: 5 * time.Second,
	}
	if config.UseLetsEncrypt {
		certManager := autocert.Manager{
			Prompt:     autocert.AcceptTOS,
			HostPolicy: autocert.HostWhitelist(config.LetsEncryptDomains...),
			Cache:      autocert.DirCache(config.LetsEncryptCertsDir),
		}

		httpServer.TLSConfig = &tls.Config{
			GetCertificate: certManager.GetCertificate,
		}

		go func() {
			http.HandleFunc("/", func(rw http.ResponseWriter, r *http.Request) {
				http.Redirect(rw, r, fmt.Sprintf("https://%s", r.Host), http.StatusMovedPermanently)
			})
			log.Println("Starting redirect server")
			log.Fatal(http.ListenAndServe(":3001", nil))
			log.Fatal(httpServer.ListenAndServe())
		}()

		log.Println("Listening on port " + config.PortNumber)
		log.Fatal(httpServer.ListenAndServeTLS("", ""))
	} else {
		log.Println("Listening on port " + config.PortNumber)
		log.Fatal(httpServer.ListenAndServe())
	}

}

func RegisterEvents(s *socketio.Server) {
	ws = s
	e.OnAny(broadcastEvent)
}

func broadcastEvent(eventType event.EventType, sessionId string, args ...interface{}) {
	ws.BroadcastTo(sessionId, eventType.String(), args...)
}
<commit_msg>Add methods in CORS requests<commit_after>package handlers

import (
	"crypto/tls"
	"fmt"
	"log"
	"net/http"
	"time"

	"golang.org/x/crypto/acme/autocert"

	"github.com/googollee/go-socket.io"
	gh "github.com/gorilla/handlers"
	"github.com/gorilla/mux"
	"github.com/play-with-docker/play-with-docker/config"
	"github.com/play-with-docker/play-with-docker/event"
	"github.com/play-with-docker/play-with-docker/pwd"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/urfave/negroni"
)

var core pwd.PWDApi
var e event.EventApi
var ws *socketio.Server

type HandlerExtender func(h *mux.Router)

func Bootstrap(c pwd.PWDApi, ev event.EventApi) {
	core = c
	e = ev
}

func Register(extend HandlerExtender) {
	server, err := socketio.NewServer(nil)
	if err != nil {
		log.Fatal(err)
	}
	server.On("connection", WS)
	server.On("error", WSError)

	RegisterEvents(server)

	r := mux.NewRouter()
	corsRouter := mux.NewRouter()

	corsHandler := gh.CORS(gh.AllowCredentials(), gh.AllowedHeaders([]string{"x-requested-with", "content-type"}), gh.AllowedMethods([]string{"GET", "POST", "HEAD", "DELETE"}), gh.AllowedOrigins([]string{"*"}))

	// Specific routes
	r.HandleFunc("/ping", Ping).Methods("GET")
	corsRouter.HandleFunc("/instances/images", GetInstanceImages).Methods("GET")
	corsRouter.HandleFunc("/sessions/{sessionId}", GetSession).Methods("GET")
	corsRouter.HandleFunc("/sessions/{sessionId}", CloseSession).Methods("DELETE")
	corsRouter.HandleFunc("/sessions/{sessionId}/setup", SessionSetup).Methods("POST")
	corsRouter.HandleFunc("/sessions/{sessionId}/instances", NewInstance).Methods("POST")
	corsRouter.HandleFunc("/sessions/{sessionId}/instances/{instanceName}/uploads", FileUpload).Methods("POST")
	corsRouter.HandleFunc("/sessions/{sessionId}/instances/{instanceName}", DeleteInstance).Methods("DELETE")
	corsRouter.HandleFunc("/sessions/{sessionId}/instances/{instanceName}/exec", Exec).Methods("POST")

	r.HandleFunc("/ooc", func(rw http.ResponseWriter, r *http.Request) {
		http.ServeFile(rw, r, "./www/ooc.html")
	}).Methods("GET")
	r.HandleFunc("/503", func(rw http.ResponseWriter, r *http.Request) {
		http.ServeFile(rw, r, "./www/503.html")
	}).Methods("GET")
	r.HandleFunc("/p/{sessionId}", Home).Methods("GET")
	r.PathPrefix("/assets").Handler(http.FileServer(http.Dir("./www")))
	r.HandleFunc("/robots.txt", func(rw http.ResponseWriter, r *http.Request) {
		http.ServeFile(rw, r, "www/robots.txt")
	})
	r.HandleFunc("/sdk.js", func(rw http.ResponseWriter, r *http.Request) {
		http.ServeFile(rw, r, "www/sdk.js")
	})

	corsRouter.Handle("/sessions/{sessionId}/ws/", server)
	r.Handle("/metrics", promhttp.Handler())

	// Generic routes
	r.HandleFunc("/", func(rw http.ResponseWriter, r *http.Request) {
		http.ServeFile(rw, r, "./www/landing.html")
	}).Methods("GET")

	r.HandleFunc("/users/me", LoggedInUser).Methods("GET")
	r.HandleFunc("/oauth/providers", ListProviders).Methods("GET")
	r.HandleFunc("/oauth/providers/{provider}/login", Login).Methods("GET")
	r.HandleFunc("/oauth/providers/{provider}/callback", LoginCallback).Methods("GET")

	corsRouter.HandleFunc("/", NewSession).Methods("POST")

	if extend != nil {
		extend(corsRouter)
	}

	n := negroni.Classic()
	r.PathPrefix("/").Handler(negroni.New(negroni.Wrap(corsHandler(corsRouter))))
	n.UseHandler(r)

	httpServer := http.Server{
		Addr:              "0.0.0.0:" + config.PortNumber,
		Handler:           n,
		IdleTimeout:       30 * time.Second,
		ReadHeaderTimeout: 5 * time.Second,
	}
	if config.UseLetsEncrypt {
		certManager := autocert.Manager{
			Prompt:     autocert.AcceptTOS,
			HostPolicy: autocert.HostWhitelist(config.LetsEncryptDomains...),
			Cache:      autocert.DirCache(config.LetsEncryptCertsDir),
		}

		httpServer.TLSConfig = &tls.Config{
			GetCertificate: certManager.GetCertificate,
		}

		go func() {
			http.HandleFunc("/", func(rw http.ResponseWriter, r *http.Request) {
				http.Redirect(rw, r, fmt.Sprintf("https://%s", r.Host), http.StatusMovedPermanently)
			})
			log.Println("Starting redirect server")
			log.Fatal(http.ListenAndServe(":3001", nil))
			log.Fatal(httpServer.ListenAndServe())
		}()

		log.Println("Listening on port " + config.PortNumber)
		log.Fatal(httpServer.ListenAndServeTLS("", ""))
	} else {
		log.Println("Listening on port " + config.PortNumber)
		log.Fatal(httpServer.ListenAndServe())
	}

}

func RegisterEvents(s *socketio.Server) {
	ws = s
	e.OnAny(broadcastEvent)
}

func broadcastEvent(eventType event.EventType, sessionId string, args ...interface{}) {
	ws.BroadcastTo(sessionId, eventType.String(), args...)
}
<|endoftext|>


<commit_before>package builder

import (
	"github.com/appc/spec/schema/types"
	"github.com/blablacar/cnt/dist"
	"github.com/blablacar/cnt/log"
	"github.com/blablacar/cnt/spec"
	"github.com/blablacar/cnt/utils"
	"io/ioutil"
	"os"
	"strings"
)

const BATS_ACI = "aci.blablacar.com/aci-bats:2"
const PATH_RESULT = "/result"
const STATUS_SUFFIX = "_status"
const TEST_INIT_SCRIPT = `#!/bin/bash
set -x

%%COMMAND%% &

waitsuccess="0"
if [ -f "/tests/wait.sh" ]; then
	chmod +x /tests/wait.sh
	i="0"
	while [ $i -lt 60 ]; do
	  /tests/wait.sh
	  if [ $? == 0 ]; then
	  	waitsuccess="1"
	  	break;
	  fi
	  i=$[$i+1]
	  sleep 1
	done
fi

if [ wait_success == "0" ]; then
	echo "1\n" > /result/wait.sh
	echo "WAIT FAILED"
	exit 1
fi

/test.sh
`

func (cnt *Img) Test() {
	cnt.Install()
	log.Get().Info("Testing " + cnt.manifest.NameAndVersion)

	if _, err := os.Stat(cnt.path + PATH_TESTS); err != nil {
		if cnt.args.NoTestFail {
			log.Get().Panic("Test directory does not exists but tests are mandatory")
		}
		log.Get().Warn("Tests directory does not exists")
		return
	}

	cnt.importAciBats()

	testAci, err := cnt.prepareTestAci()
	if err != nil {
		log.Get().Panic(err)
	}
	testAci.Build()

	os.MkdirAll(cnt.target+PATH_TESTS+PATH_TARGET+PATH_RESULT, 0777)

	if err := utils.ExecCmd("rkt",
		"--insecure-skip-verify=true",
		"run",
		"--mds-register=false",
		"--volume=result,kind=host,source="+cnt.target+PATH_TESTS+PATH_TARGET+PATH_RESULT,
		cnt.target+PATH_TESTS+PATH_TARGET+"/image.aci"); err != nil {
		// rkt+systemd cannot exit with fail status yet
		log.Get().Panic(err)
	}

	cnt.checkResult()
}

func (cnt *Img) checkResult() {
	files, err := ioutil.ReadDir(cnt.target + PATH_TESTS + PATH_TARGET + PATH_RESULT)
	if err != nil {
		log.Get().Panic("Cannot read test result directory", err)
	}
	testFound := false
	for _, f := range files {
		fullPath := cnt.target + PATH_TESTS + PATH_TARGET + PATH_RESULT + "/" + f.Name()
		content, err := ioutil.ReadFile(fullPath)
		if err != nil {
			log.Get().Panic("Cannot read result file", f.Name(), err)
		}
		if !strings.HasSuffix(f.Name(), STATUS_SUFFIX) {
			if testFound == false && string(content) != "1..0\n" {
				testFound = true
			}
			continue
		}
		if string(content) != "0\n" {
			log.Get().Error("Failed test file : ", f.Name())
			os.Exit(2)
		}
	}

	if cnt.args.NoTestFail && !testFound {
		log.Get().Panic("No tests found")
	}
}

func (cnt *Img) importAciBats() {
	if err := utils.ExecCmd("bash", "-c", "rkt image list --fields name --no-legend | grep -q "+BATS_ACI); err != nil {
		content, _ := dist.Asset("dist/bindata/aci-bats.aci")
		if err := ioutil.WriteFile("/tmp/aci-bats.aci", content, 0644); err != nil {
			log.Get().Panic(err)
		}
		utils.ExecCmd("rkt", "--insecure-skip-verify=true", "fetch", "/tmp/aci-bats.aci")
		os.Remove("/tmp/aci-bats.aci")
	}
}

func (cnt *Img) prepareTestAci() (*Img, error) {
	files, err := ioutil.ReadDir(cnt.path + PATH_TESTS)
	if err != nil {
		return nil, err
	}

	utils.CopyDir(cnt.path+PATH_TESTS+PATH_FILES, cnt.target+PATH_TESTS+PATH_FILES)
	utils.CopyDir(cnt.path+PATH_TESTS+PATH_ATTRIBUTES, cnt.target+PATH_TESTS+PATH_ATTRIBUTES)
	utils.CopyDir(cnt.path+PATH_TESTS+PATH_CONFD, cnt.target+PATH_TESTS+PATH_CONFD)
	utils.CopyDir(cnt.path+PATH_TESTS+PATH_RUNLEVELS, cnt.target+PATH_TESTS+PATH_RUNLEVELS)

	os.MkdirAll(cnt.target+PATH_TESTS+PATH_FILES+PATH_TESTS, 0777)
	for _, f := range files {
		if !f.IsDir() {
			if err := utils.CopyFile(cnt.path+PATH_TESTS+"/"+f.Name(), cnt.target+PATH_TESTS+PATH_FILES+PATH_TESTS+"/"+f.Name()); err != nil {
				log.Get().Panic(err)
			}
		}
	}

	ExecScript := strings.Replace(TEST_INIT_SCRIPT, "%%COMMAND%%", "'"+strings.Join(cnt.manifest.Aci.App.Exec, "' '")+"'", 1)

	ioutil.WriteFile(cnt.target+PATH_TESTS+PATH_FILES+"/init.sh", []byte(ExecScript), 0777)

	fullname, err := spec.NewACFullName(cnt.manifest.NameAndVersion.Name() + "_test:" + cnt.manifest.NameAndVersion.Version())
	if err != nil {
		log.Get().Panic(err)
	}

	resultMountName, _ := types.NewACName("result")
	testAci, err := NewAciWithManifest(cnt.target+PATH_TESTS, cnt.args, spec.AciManifest{
		Aci: spec.AciDefinition{
			App: &spec.CntApp{
				Exec:        []string{"/init.sh"},
				MountPoints: []types.MountPoint{{Path: PATH_RESULT, Name: *resultMountName}},
			},
			Dependencies: []spec.ACFullname{BATS_ACI, cnt.manifest.NameAndVersion},
		},
		NameAndVersion: *fullname,
	})
	if err != nil {
		log.Get().Panic("Cannot build test aci", err)
	}
	return testAci, nil
}
<commit_msg>do not use overlay for tests<commit_after>package builder

import (
	"github.com/appc/spec/schema/types"
	"github.com/blablacar/cnt/dist"
	"github.com/blablacar/cnt/log"
	"github.com/blablacar/cnt/spec"
	"github.com/blablacar/cnt/utils"
	"io/ioutil"
	"os"
	"strings"
)

const BATS_ACI = "aci.blablacar.com/aci-bats:2"
const PATH_RESULT = "/result"
const STATUS_SUFFIX = "_status"
const TEST_INIT_SCRIPT = `#!/bin/bash
set -x

%%COMMAND%% &

waitsuccess="0"
if [ -f "/tests/wait.sh" ]; then
	chmod +x /tests/wait.sh
	i="0"
	while [ $i -lt 60 ]; do
	  /tests/wait.sh
	  if [ $? == 0 ]; then
	  	waitsuccess="1"
	  	break;
	  fi
	  i=$[$i+1]
	  sleep 1
	done
fi

if [ wait_success == "0" ]; then
	echo "1\n" > /result/wait.sh
	echo "WAIT FAILED"
	exit 1
fi

/test.sh
`

func (cnt *Img) Test() {
	cnt.Install()
	log.Get().Info("Testing " + cnt.manifest.NameAndVersion)

	if _, err := os.Stat(cnt.path + PATH_TESTS); err != nil {
		if cnt.args.NoTestFail {
			log.Get().Panic("Test directory does not exists but tests are mandatory")
		}
		log.Get().Warn("Tests directory does not exists")
		return
	}

	cnt.importAciBats()

	testAci, err := cnt.prepareTestAci()
	if err != nil {
		log.Get().Panic(err)
	}
	testAci.Build()

	os.MkdirAll(cnt.target+PATH_TESTS+PATH_TARGET+PATH_RESULT, 0777)

	if err := utils.ExecCmd("rkt",
		"--insecure-skip-verify=true",
		"run",
		"--mds-register=false",
		"--no-overlay=true",
		"--volume=result,kind=host,source="+cnt.target+PATH_TESTS+PATH_TARGET+PATH_RESULT,
		cnt.target+PATH_TESTS+PATH_TARGET+"/image.aci"); err != nil {
		// rkt+systemd cannot exit with fail status yet
		log.Get().Panic(err)
	}

	cnt.checkResult()
}

func (cnt *Img) checkResult() {
	files, err := ioutil.ReadDir(cnt.target + PATH_TESTS + PATH_TARGET + PATH_RESULT)
	if err != nil {
		log.Get().Panic("Cannot read test result directory", err)
	}
	testFound := false
	for _, f := range files {
		fullPath := cnt.target + PATH_TESTS + PATH_TARGET + PATH_RESULT + "/" + f.Name()
		content, err := ioutil.ReadFile(fullPath)
		if err != nil {
			log.Get().Panic("Cannot read result file", f.Name(), err)
		}
		if !strings.HasSuffix(f.Name(), STATUS_SUFFIX) {
			if testFound == false && string(content) != "1..0\n" {
				testFound = true
			}
			continue
		}
		if string(content) != "0\n" {
			log.Get().Error("Failed test file : ", f.Name())
			os.Exit(2)
		}
	}

	if cnt.args.NoTestFail && !testFound {
		log.Get().Panic("No tests found")
	}
}

func (cnt *Img) importAciBats() {
	if err := utils.ExecCmd("bash", "-c", "rkt image list --fields name --no-legend | grep -q "+BATS_ACI); err != nil {
		content, _ := dist.Asset("dist/bindata/aci-bats.aci")
		if err := ioutil.WriteFile("/tmp/aci-bats.aci", content, 0644); err != nil {
			log.Get().Panic(err)
		}
		utils.ExecCmd("rkt", "--insecure-skip-verify=true", "fetch", "/tmp/aci-bats.aci")
		os.Remove("/tmp/aci-bats.aci")
	}
}

func (cnt *Img) prepareTestAci() (*Img, error) {
	files, err := ioutil.ReadDir(cnt.path + PATH_TESTS)
	if err != nil {
		return nil, err
	}

	utils.CopyDir(cnt.path+PATH_TESTS+PATH_FILES, cnt.target+PATH_TESTS+PATH_FILES)
	utils.CopyDir(cnt.path+PATH_TESTS+PATH_ATTRIBUTES, cnt.target+PATH_TESTS+PATH_ATTRIBUTES)
	utils.CopyDir(cnt.path+PATH_TESTS+PATH_CONFD, cnt.target+PATH_TESTS+PATH_CONFD)
	utils.CopyDir(cnt.path+PATH_TESTS+PATH_RUNLEVELS, cnt.target+PATH_TESTS+PATH_RUNLEVELS)

	os.MkdirAll(cnt.target+PATH_TESTS+PATH_FILES+PATH_TESTS, 0777)
	for _, f := range files {
		if !f.IsDir() {
			if err := utils.CopyFile(cnt.path+PATH_TESTS+"/"+f.Name(), cnt.target+PATH_TESTS+PATH_FILES+PATH_TESTS+"/"+f.Name()); err != nil {
				log.Get().Panic(err)
			}
		}
	}

	ExecScript := strings.Replace(TEST_INIT_SCRIPT, "%%COMMAND%%", "'"+strings.Join(cnt.manifest.Aci.App.Exec, "' '")+"'", 1)

	ioutil.WriteFile(cnt.target+PATH_TESTS+PATH_FILES+"/init.sh", []byte(ExecScript), 0777)

	fullname, err := spec.NewACFullName(cnt.manifest.NameAndVersion.Name() + "_test:" + cnt.manifest.NameAndVersion.Version())
	if err != nil {
		log.Get().Panic(err)
	}

	resultMountName, _ := types.NewACName("result")
	testAci, err := NewAciWithManifest(cnt.target+PATH_TESTS, cnt.args, spec.AciManifest{
		Aci: spec.AciDefinition{
			App: &spec.CntApp{
				Exec:        []string{"/init.sh"},
				MountPoints: []types.MountPoint{{Path: PATH_RESULT, Name: *resultMountName}},
			},
			Dependencies: []spec.ACFullname{BATS_ACI, cnt.manifest.NameAndVersion},
		},
		NameAndVersion: *fullname,
	})
	if err != nil {
		log.Get().Panic("Cannot build test aci", err)
	}
	return testAci, nil
}
<|endoftext|>


<commit_before>// Copyright © 2016 Jip J. Dekker <jip@dekker.li>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package helpers

import (
	"errors"
	"os"
	"path/filepath"

	log "github.com/Sirupsen/logrus"
)

// CleanPath returns a cleaned path from a given string relative
// to the working directory unless explicitly absolute
func CleanPath(path string) (string, error) {
	if path == "" {
		// No given path, use working directory
		return os.Getwd()
	} else if filepath.IsAbs(path) {
		// Given path is absolute
		return filepath.Clean(path), nil
	} else {
		// Given path is a relative path
		dir, err := os.Getwd()
		if err == nil {
			return filepath.Join(dir, path), nil
		}
		return "", err
	}
}

// Exists checks if a file or directory exists.
func Exists(path string) bool {
	_, err := os.Stat(path)
	if err == nil {
		return true
	}
	if !os.IsNotExist(err) {
		log.WithFields(log.Fields{"error": err, "path": path}).
			Panic("Unable to check path")
	}
	return false
}

// FindFileDir returns the path of the
func FindFileDir(file string) (string, error) {
	wd, err := os.Getwd()
	if err != nil {
		return "", nil
	}
	for path := wd; path != "."; path = filepath.Dir(path) {
		if Exists(filepath.Join(path, file)) {
			return path, nil
		}
	}
	return "", &os.PathError{
		Path: wd,
		Err:  errors.New("directory containing " + file + " not found"),
	}
}
<commit_msg>Adds helper to generate absolute paths<commit_after>// Copyright © 2016 Jip J. Dekker <jip@dekker.li>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package helpers

import (
	"errors"
	"os"
	"path/filepath"

	log "github.com/Sirupsen/logrus"
)

// CleanPath returns a cleaned path from a given string relative
// to the working directory unless explicitly absolute
func CleanPath(path string) (string, error) {
	if path == "" {
		// No given path, use working directory
		return os.Getwd()
	} else if filepath.IsAbs(path) {
		// Given path is absolute
		return filepath.Clean(path), nil
	} else {
		// Given path is a relative path
		dir, err := os.Getwd()
		if err == nil {
			return filepath.Join(dir, path), nil
		}
		return "", err
	}
}

// Exists checks if a file or directory exists.
func Exists(path string) bool {
	_, err := os.Stat(path)
	if err == nil {
		return true
	}
	if !os.IsNotExist(err) {
		log.WithFields(log.Fields{"error": err, "path": path}).
			Panic("Unable to check path")
	}
	return false
}

// FindFileDir returns the path of the
func FindFileDir(file string) (string, error) {
	wd, err := os.Getwd()
	if err != nil {
		return "", nil
	}
	for path := wd; path != "."; path = filepath.Dir(path) {
		if Exists(filepath.Join(path, file)) {
			return path, nil
		}
	}
	return "", &os.PathError{
		Path: wd,
		Err:  errors.New("directory containing " + file + " not found"),
	}
}

// AbsolutePath returns an the path if it is absolute or
// otherwise filepath.Join(lib, path)
func AbsolutePath(path, lib string) string {
	if filepath.IsAbs(path) {
		return path
	}
	return filepath.Join(lib, path)
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package lease

import (
	"errors"
	"fmt"
	"io"
	"os"
	"sync"
)

// A read-write wrapper around a file. Unlike a read lease, this cannot be
// revoked.
//
// All methods are safe for concurrent access.
type ReadWriteLease interface {
	// Methods with semantics matching *os.File.
	io.ReadWriteSeeker
	io.ReaderAt
	io.WriterAt
	Truncate(size int64) (err error)

	// Return the current size of the underlying file.
	Size() (size int64, err error)

	// Downgrade to a read lease, releasing any resources pinned by this lease to
	// the pool that may be revoked, as with any read lease. After successfully
	// downgrading, this lease must not be used again.
	Downgrade() (rl ReadLease, err error)
}

////////////////////////////////////////////////////////////////////////
// Implementation
////////////////////////////////////////////////////////////////////////

type readWriteLease struct {
	mu sync.Mutex

	/////////////////////////
	// Dependencies
	/////////////////////////

	// The leaser that issued this lease.
	leaser *FileLeaser

	// The underlying file. We serialize access using the mutex.
	//
	// GUARDED_BY(mu)
	file *os.File

	/////////////////////////
	// Mutable state
	/////////////////////////

	// The cumulative number of bytes we have reported to the leaser using
	// FileLeaser.addReadWriteByteDelta. When the size changes, we report the
	// difference between the new size and this value.
	//
	// GUARDED_BY(mu)
	reportedSize int64
}

var _ ReadWriteLease = &readWriteLease{}

func newReadWriteLease(
	leaser *FileLeaser,
	file *os.File) (rwl *readWriteLease) {
	rwl = &readWriteLease{
		leaser: leaser,
		file:   file,
	}

	return
}

// LOCKS_EXCLUDED(rwl.mu)
func (rwl *readWriteLease) Read(p []byte) (n int, err error) {
	rwl.mu.Lock()
	defer rwl.mu.Unlock()

	n, err = rwl.file.Read(p)
	return
}

func (rwl *readWriteLease) Write(p []byte) (n int, err error) {
	err = errors.New("TODO")
	return
}

// LOCKS_EXCLUDED(rwl.mu)
func (rwl *readWriteLease) Seek(
	offset int64,
	whence int) (off int64, err error) {
	rwl.mu.Lock()
	defer rwl.mu.Unlock()

	off, err = rwl.file.Seek(offset, whence)
	return
}

// LOCKS_EXCLUDED(rwl.mu)
func (rwl *readWriteLease) ReadAt(p []byte, off int64) (n int, err error) {
	rwl.mu.Lock()
	defer rwl.mu.Unlock()

	n, err = rwl.file.ReadAt(p, off)
	return
}

func (rwl *readWriteLease) WriteAt(p []byte, off int64) (n int, err error) {
	err = errors.New("TODO")
	return
}

func (rwl *readWriteLease) Truncate(size int64) (err error) {
	err = errors.New("TODO")
	return
}

// LOCKS_EXCLUDED(rwl.mu)
func (rwl *readWriteLease) Size() (size int64, err error) {
	rwl.mu.Lock()
	defer rwl.mu.Unlock()

	// Stat the file to get its size.
	fi, err := rwl.file.Stat()
	if err != nil {
		err = fmt.Errorf("Stat: %v", err)
		return
	}

	size = fi.Size()
	return
}

func (rwl *readWriteLease) Downgrade() (rl ReadLease, err error) {
	err = errors.New("TODO")
	return
}
<commit_msg>More lock annotations.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package lease

import (
	"errors"
	"fmt"
	"io"
	"os"
	"sync"
)

// A read-write wrapper around a file. Unlike a read lease, this cannot be
// revoked.
//
// All methods are safe for concurrent access.
type ReadWriteLease interface {
	// Methods with semantics matching *os.File.
	io.ReadWriteSeeker
	io.ReaderAt
	io.WriterAt
	Truncate(size int64) (err error)

	// Return the current size of the underlying file.
	Size() (size int64, err error)

	// Downgrade to a read lease, releasing any resources pinned by this lease to
	// the pool that may be revoked, as with any read lease. After successfully
	// downgrading, this lease must not be used again.
	Downgrade() (rl ReadLease, err error)
}

////////////////////////////////////////////////////////////////////////
// Implementation
////////////////////////////////////////////////////////////////////////

type readWriteLease struct {
	mu sync.Mutex

	/////////////////////////
	// Dependencies
	/////////////////////////

	// The leaser that issued this lease.
	leaser *FileLeaser

	// The underlying file. We serialize access using the mutex.
	//
	// GUARDED_BY(mu)
	file *os.File

	/////////////////////////
	// Mutable state
	/////////////////////////

	// The cumulative number of bytes we have reported to the leaser using
	// FileLeaser.addReadWriteByteDelta. When the size changes, we report the
	// difference between the new size and this value.
	//
	// GUARDED_BY(mu)
	reportedSize int64
}

var _ ReadWriteLease = &readWriteLease{}

func newReadWriteLease(
	leaser *FileLeaser,
	file *os.File) (rwl *readWriteLease) {
	rwl = &readWriteLease{
		leaser: leaser,
		file:   file,
	}

	return
}

// LOCKS_EXCLUDED(rwl.mu)
func (rwl *readWriteLease) Read(p []byte) (n int, err error) {
	rwl.mu.Lock()
	defer rwl.mu.Unlock()

	n, err = rwl.file.Read(p)
	return
}

// LOCKS_EXCLUDED(rwl.mu)
func (rwl *readWriteLease) Write(p []byte) (n int, err error) {
	err = errors.New("TODO")
	return
}

// LOCKS_EXCLUDED(rwl.mu)
func (rwl *readWriteLease) Seek(
	offset int64,
	whence int) (off int64, err error) {
	rwl.mu.Lock()
	defer rwl.mu.Unlock()

	off, err = rwl.file.Seek(offset, whence)
	return
}

// LOCKS_EXCLUDED(rwl.mu)
func (rwl *readWriteLease) ReadAt(p []byte, off int64) (n int, err error) {
	rwl.mu.Lock()
	defer rwl.mu.Unlock()

	n, err = rwl.file.ReadAt(p, off)
	return
}

// LOCKS_EXCLUDED(rwl.mu)
func (rwl *readWriteLease) WriteAt(p []byte, off int64) (n int, err error) {
	err = errors.New("TODO")
	return
}

// LOCKS_EXCLUDED(rwl.mu)
func (rwl *readWriteLease) Truncate(size int64) (err error) {
	err = errors.New("TODO")
	return
}

// LOCKS_EXCLUDED(rwl.mu)
func (rwl *readWriteLease) Size() (size int64, err error) {
	rwl.mu.Lock()
	defer rwl.mu.Unlock()

	// Stat the file to get its size.
	fi, err := rwl.file.Stat()
	if err != nil {
		err = fmt.Errorf("Stat: %v", err)
		return
	}

	size = fi.Size()
	return
}

// LOCKS_EXCLUDED(rwl.mu)
func (rwl *readWriteLease) Downgrade() (rl ReadLease, err error) {
	err = errors.New("TODO")
	return
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"

	"math"
	"sort"

	"github.com/codegangsta/cli"
	"github.com/mattn/go-tty"
	"github.com/pkg/browser"
)

type ActionType int

const (
	LABEL_AS_POSITIVE ActionType = iota
	LABEL_AS_NEGATIVE
	SAVE
	HELP
	SKIP
	EXIT
)

func rune2ActionType(r rune) ActionType {
	switch r {
	case 'p':
		return LABEL_AS_POSITIVE
	case 'n':
		return LABEL_AS_NEGATIVE
	case 's':
		return SAVE
	case 'h':
		return HELP
	case 'e':
		return EXIT
	default:
		return SKIP
	}
}

func input2ActionType() (ActionType, error) {
	t, err := tty.Open()
	defer t.Close()
	if err != nil {
		return EXIT, err
	}
	var r rune
	for r == 0 {
		r, err = t.ReadRune()
		if err != nil {
			return SKIP, err
		}
	}
	return rune2ActionType(r), nil
}

func NextExampleToBeAnnotated(model *Model, examples Examples) *Example {
	unlabeledExamples := model.SortByScore(examples)
	if len(unlabeledExamples) == 0 {
		return nil
	}
	e := unlabeledExamples[0]
	if e == nil {
		return nil
	}
	return e
}

var ActionHelpDoc = `
p: Label this example as positive.
n: Label this example as negative.
s: Save additionally annotated examples in 'output-filename'.
h: Show this help.
e: Exit.
`

func doAnnotate(c *cli.Context) error {
	inputFilename := c.String("input-filename")
	outputFilename := c.String("output-filename")
	openUrl := c.Bool("open-url")
	filterStatusCodeOk := c.Bool("filter-status-code-ok")
	showActiveFeatures := c.Bool("show-active-features")

	if inputFilename == "" {
		_ = cli.ShowCommandHelp(c, "annotate")
		return cli.NewExitError("`input-filename` is a required field.", 1)
	}

	if outputFilename == "" {
		_ = cli.ShowCommandHelp(c, "annotate")
		return cli.NewExitError("`output-filename` is a required field.", 1)
	}

	cacheFilename := CacheFilename

	cache, err := LoadCache(cacheFilename)
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
	}

	examples, err := ReadExamples(inputFilename)
	if err != nil {
		return err
	}

	AttachMetaData(cache, examples)
	if filterStatusCodeOk {
		examples = FilterStatusCodeOkExamples(examples)
	}
	model := TrainedModel(examples)

annotationLoop:
	for {
		e := NextExampleToBeAnnotated(model, examples)
		fmt.Println("Label this example (Score: " + fmt.Sprintf("%+0.03f", e.Score) + "): " + e.Url + " (" + e.Title + ")")

		if openUrl {
			browser.OpenURL(e.Url)
		}
		if showActiveFeatures {
			ShowActiveFeatures(model, *e, 5)
		}

		act, err := input2ActionType()
		if err != nil {
			return err
		}
		switch act {
		case LABEL_AS_POSITIVE:
			fmt.Println("Labeled as positive")
			e.Annotate(POSITIVE)
		case LABEL_AS_NEGATIVE:
			fmt.Println("Labeled as negative")
			e.Annotate(NEGATIVE)
		case SKIP:
			fmt.Println("Skiped this example")
			continue
		case SAVE:
			fmt.Println("Saved labeld examples")
			WriteExamples(examples, outputFilename)
		case HELP:
			fmt.Println(ActionHelpDoc)
		case EXIT:
			fmt.Println("EXIT")
			break annotationLoop
		default:
			break annotationLoop
		}
		model = TrainedModel(examples)
	}

	WriteExamples(examples, outputFilename)
	cache.Save(cacheFilename)

	return nil
}

var commandAnnotate = cli.Command{
	Name:  "annotate",
	Usage: "Annotate URLs",
	Description: `
Annotate URLs using active learning.
`,
	Action: doAnnotate,
	Flags: []cli.Flag{
		cli.StringFlag{Name: "input-filename"},
		cli.StringFlag{Name: "output-filename"},
		cli.BoolFlag{Name: "open-url", Usage: "Open url in background"},
		cli.BoolFlag{Name: "filter-status-code-ok", Usage: "Use only examples with status code = 200"},
		cli.BoolFlag{Name: "show-active-features"},
	},
}

type FeatureWeightPair struct {
	Feature string
	Weight  float64
}

type FeatureWeightPairs []FeatureWeightPair

func ShowActiveFeatures(model *Model, example Example, n int) {
	result := FeatureWeightPairs{}
	for _, f := range example.Fv {
		result = append(result, FeatureWeightPair{f, model.GetAveragedWeight(f)})
	}
	sort.Sort(sort.Reverse(result))

	cnt := 0
	for _, pair := range result {
		if cnt >= n {
			break
		}
		if (example.Score > 0.0 && pair.Weight > 0.0) || (example.Score < 0.0 && pair.Weight < 0.0) {
			fmt.Println(fmt.Sprintf("%+0.1f %s", pair.Weight, pair.Feature))
			cnt++
		}
	}
}

func (slice FeatureWeightPairs) Len() int {
	return len(slice)
}

func (slice FeatureWeightPairs) Less(i, j int) bool {
	return math.Abs(slice[i].Weight) < math.Abs(slice[j].Weight)
}

func (slice FeatureWeightPairs) Swap(i, j int) {
	slice[i], slice[j] = slice[j], slice[i]
}
<commit_msg>SKIPしても現状同じ事例が出てくるので、ヘルプを出すようにする<commit_after>package main

import (
	"fmt"
	"os"

	"math"
	"sort"

	"github.com/codegangsta/cli"
	"github.com/mattn/go-tty"
	"github.com/pkg/browser"
)

type ActionType int

const (
	LABEL_AS_POSITIVE ActionType = iota
	LABEL_AS_NEGATIVE
	SAVE
	HELP
	SKIP
	EXIT
)

func rune2ActionType(r rune) ActionType {
	switch r {
	case 'p':
		return LABEL_AS_POSITIVE
	case 'n':
		return LABEL_AS_NEGATIVE
	case 's':
		return SAVE
	case 'h':
		return HELP
	case 'e':
		return EXIT
	default:
		return HELP
	}
}

func input2ActionType() (ActionType, error) {
	t, err := tty.Open()
	defer t.Close()
	if err != nil {
		return EXIT, err
	}
	var r rune
	for r == 0 {
		r, err = t.ReadRune()
		if err != nil {
			return HELP, err
		}
	}
	return rune2ActionType(r), nil
}

func NextExampleToBeAnnotated(model *Model, examples Examples) *Example {
	unlabeledExamples := model.SortByScore(examples)
	if len(unlabeledExamples) == 0 {
		return nil
	}
	e := unlabeledExamples[0]
	if e == nil {
		return nil
	}
	return e
}

var ActionHelpDoc = `
p: Label this example as positive.
n: Label this example as negative.
s: Save additionally annotated examples in 'output-filename'.
h: Show this help.
e: Exit.
`

func doAnnotate(c *cli.Context) error {
	inputFilename := c.String("input-filename")
	outputFilename := c.String("output-filename")
	openUrl := c.Bool("open-url")
	filterStatusCodeOk := c.Bool("filter-status-code-ok")
	showActiveFeatures := c.Bool("show-active-features")

	if inputFilename == "" {
		_ = cli.ShowCommandHelp(c, "annotate")
		return cli.NewExitError("`input-filename` is a required field.", 1)
	}

	if outputFilename == "" {
		_ = cli.ShowCommandHelp(c, "annotate")
		return cli.NewExitError("`output-filename` is a required field.", 1)
	}

	cacheFilename := CacheFilename

	cache, err := LoadCache(cacheFilename)
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
	}

	examples, err := ReadExamples(inputFilename)
	if err != nil {
		return err
	}

	AttachMetaData(cache, examples)
	if filterStatusCodeOk {
		examples = FilterStatusCodeOkExamples(examples)
	}
	model := TrainedModel(examples)

annotationLoop:
	for {
		e := NextExampleToBeAnnotated(model, examples)
		fmt.Println("Label this example (Score: " + fmt.Sprintf("%+0.03f", e.Score) + "): " + e.Url + " (" + e.Title + ")")

		if openUrl {
			browser.OpenURL(e.Url)
		}
		if showActiveFeatures {
			ShowActiveFeatures(model, *e, 5)
		}

		act, err := input2ActionType()
		if err != nil {
			return err
		}
		switch act {
		case LABEL_AS_POSITIVE:
			fmt.Println("Labeled as positive")
			e.Annotate(POSITIVE)
		case LABEL_AS_NEGATIVE:
			fmt.Println("Labeled as negative")
			e.Annotate(NEGATIVE)
		case SKIP:
			fmt.Println("Skiped this example")
			continue
		case SAVE:
			fmt.Println("Saved labeld examples")
			WriteExamples(examples, outputFilename)
		case HELP:
			fmt.Println(ActionHelpDoc)
		case EXIT:
			fmt.Println("EXIT")
			break annotationLoop
		default:
			break annotationLoop
		}
		model = TrainedModel(examples)
	}

	WriteExamples(examples, outputFilename)
	cache.Save(cacheFilename)

	return nil
}

var commandAnnotate = cli.Command{
	Name:  "annotate",
	Usage: "Annotate URLs",
	Description: `
Annotate URLs using active learning.
`,
	Action: doAnnotate,
	Flags: []cli.Flag{
		cli.StringFlag{Name: "input-filename"},
		cli.StringFlag{Name: "output-filename"},
		cli.BoolFlag{Name: "open-url", Usage: "Open url in background"},
		cli.BoolFlag{Name: "filter-status-code-ok", Usage: "Use only examples with status code = 200"},
		cli.BoolFlag{Name: "show-active-features"},
	},
}

type FeatureWeightPair struct {
	Feature string
	Weight  float64
}

type FeatureWeightPairs []FeatureWeightPair

func ShowActiveFeatures(model *Model, example Example, n int) {
	result := FeatureWeightPairs{}
	for _, f := range example.Fv {
		result = append(result, FeatureWeightPair{f, model.GetAveragedWeight(f)})
	}
	sort.Sort(sort.Reverse(result))

	cnt := 0
	for _, pair := range result {
		if cnt >= n {
			break
		}
		if (example.Score > 0.0 && pair.Weight > 0.0) || (example.Score < 0.0 && pair.Weight < 0.0) {
			fmt.Println(fmt.Sprintf("%+0.1f %s", pair.Weight, pair.Feature))
			cnt++
		}
	}
}

func (slice FeatureWeightPairs) Len() int {
	return len(slice)
}

func (slice FeatureWeightPairs) Less(i, j int) bool {
	return math.Abs(slice[i].Weight) < math.Abs(slice[j].Weight)
}

func (slice FeatureWeightPairs) Swap(i, j int) {
	slice[i], slice[j] = slice[j], slice[i]
}
<|endoftext|>


<commit_before>package lfsapi

import (
	"fmt"
	"net/url"
	"os"
	"path"
	"strings"
	"sync"

	"github.com/git-lfs/git-lfs/config"
	"github.com/git-lfs/git-lfs/git"
	"github.com/rubyist/tracerx"
)

type Access string

const (
	NoneAccess      Access = "none"
	BasicAccess     Access = "basic"
	PrivateAccess   Access = "private"
	NegotiateAccess Access = "negotiate"
	NTLMAccess      Access = "ntlm"
	emptyAccess     Access = ""
	defaultRemote          = "origin"
)

type EndpointFinder interface {
	NewEndpointFromCloneURL(rawurl string) Endpoint
	NewEndpoint(rawurl string) Endpoint
	Endpoint(operation, remote string) Endpoint
	RemoteEndpoint(operation, remote string) Endpoint
	GitRemoteURL(remote string, forpush bool) string
	AccessFor(rawurl string) Access
	SetAccess(rawurl string, access Access)
	GitProtocol() string
}

type endpointGitFinder struct {
	git         Env
	gitProtocol string

	aliasMu sync.Mutex
	aliases map[string]string

	accessMu  sync.Mutex
	urlAccess map[string]Access
	urlConfig *config.URLConfig
}

func NewEndpointFinder(git Env) EndpointFinder {
	e := &endpointGitFinder{
		gitProtocol: "https",
		aliases:     make(map[string]string),
		urlAccess:   make(map[string]Access),
	}

	if git != nil {
		e.git = git
		e.urlConfig = config.NewURLConfig(e.git)
		if v, ok := git.Get("lfs.gitprotocol"); ok {
			e.gitProtocol = v
		}
		initAliases(e, git)
	}

	return e
}

func (e *endpointGitFinder) Endpoint(operation, remote string) Endpoint {
	ep := e.getEndpoint(operation, remote)
	ep.Operation = operation
	return ep
}

func (e *endpointGitFinder) getEndpoint(operation, remote string) Endpoint {
	if e.git == nil {
		return Endpoint{}
	}

	if operation == "upload" {
		if url, ok := e.git.Get("lfs.pushurl"); ok {
			return e.NewEndpoint(url)
		}
	}

	if url, ok := e.git.Get("lfs.url"); ok {
		return e.NewEndpoint(url)
	}

	if len(remote) > 0 && remote != defaultRemote {
		if e := e.RemoteEndpoint(operation, remote); len(e.Url) > 0 {
			return e
		}
	}

	return e.RemoteEndpoint(operation, defaultRemote)
}

func (e *endpointGitFinder) RemoteEndpoint(operation, remote string) Endpoint {
	if e.git == nil {
		return Endpoint{}
	}

	if len(remote) == 0 {
		remote = defaultRemote
	}

	// Support separate push URL if specified and pushing
	if operation == "upload" {
		if url, ok := e.git.Get("remote." + remote + ".lfspushurl"); ok {
			return e.NewEndpoint(url)
		}
	}
	if url, ok := e.git.Get("remote." + remote + ".lfsurl"); ok {
		return e.NewEndpoint(url)
	}

	// finally fall back on git remote url (also supports pushurl)
	if url := e.GitRemoteURL(remote, operation == "upload"); url != "" {
		return e.NewEndpointFromCloneURL(url)
	}

	return Endpoint{}
}

func (e *endpointGitFinder) GitRemoteURL(remote string, forpush bool) string {
	if e.git != nil {
		if forpush {
			if u, ok := e.git.Get("remote." + remote + ".pushurl"); ok {
				return u
			}
		}

		if u, ok := e.git.Get("remote." + remote + ".url"); ok {
			return u
		}
	}

	if err := git.ValidateRemote(remote); err == nil {
		return remote
	}

	return ""
}

func (e *endpointGitFinder) NewEndpointFromCloneURL(rawurl string) Endpoint {
	ep := e.NewEndpoint(rawurl)
	if ep.Url == UrlUnknown {
		return ep
	}

	if strings.HasSuffix(rawurl, "/") {
		ep.Url = rawurl[0 : len(rawurl)-1]
	}

	// When using main remote URL for HTTP, append info/lfs
	if path.Ext(ep.Url) == ".git" {
		ep.Url += "/info/lfs"
	} else {
		ep.Url += ".git/info/lfs"
	}

	return ep
}

func (e *endpointGitFinder) NewEndpoint(rawurl string) Endpoint {
	rawurl = e.ReplaceUrlAlias(rawurl)
	u, err := url.Parse(rawurl)
	if err != nil {
		return endpointFromBareSshUrl(rawurl)
	}

	switch u.Scheme {
	case "ssh":
		return endpointFromSshUrl(u)
	case "http", "https":
		return endpointFromHttpUrl(u)
	case "git":
		return endpointFromGitUrl(u, e)
	case "":
		return endpointFromBareSshUrl(u.String())
	default:
		// Just passthrough to preserve
		return Endpoint{Url: rawurl}
	}
}

func (e *endpointGitFinder) AccessFor(rawurl string) Access {
	if e.git == nil {
		return NoneAccess
	}

	accessurl := urlWithoutAuth(rawurl)

	e.accessMu.Lock()
	defer e.accessMu.Unlock()

	if cached, ok := e.urlAccess[accessurl]; ok {
		return cached
	}

	key := fmt.Sprintf("lfs.%s.access", accessurl)
	e.urlAccess[accessurl] = e.fetchGitAccess(key)
	return e.urlAccess[accessurl]
}

func (e *endpointGitFinder) SetAccess(rawurl string, access Access) {
	accessurl := urlWithoutAuth(rawurl)
	key := fmt.Sprintf("lfs.%s.access", accessurl)
	tracerx.Printf("setting repository access to %s", access)

	e.accessMu.Lock()
	defer e.accessMu.Unlock()

	switch access {
	case emptyAccess, NoneAccess:
		git.Config.UnsetLocalKey("", key)
		e.urlAccess[accessurl] = NoneAccess
	default:
		git.Config.SetLocal("", key, string(access))
		e.urlAccess[accessurl] = access
	}
}

func urlWithoutAuth(rawurl string) string {
	if !strings.Contains(rawurl, "@") {
		return rawurl
	}

	u, err := url.Parse(rawurl)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error parsing URL %q: %s", rawurl, err)
		return rawurl
	}

	u.User = nil
	return u.String()
}

func (e *endpointGitFinder) fetchGitAccess(key string) Access {
	if v, _ := e.git.Get(key); len(v) > 0 {
		access := Access(strings.ToLower(v))
		if access == PrivateAccess {
			return BasicAccess
		}
		return access
	}
	return NoneAccess
}

func (e *endpointGitFinder) GitProtocol() string {
	return e.gitProtocol
}

// ReplaceUrlAlias returns a url with a prefix from a `url.*.insteadof` git
// config setting. If multiple aliases match, use the longest one.
// See https://git-scm.com/docs/git-config for Git's docs.
func (e *endpointGitFinder) ReplaceUrlAlias(rawurl string) string {
	e.aliasMu.Lock()
	defer e.aliasMu.Unlock()

	var longestalias string
	for alias, _ := range e.aliases {
		if !strings.HasPrefix(rawurl, alias) {
			continue
		}

		if longestalias < alias {
			longestalias = alias
		}
	}

	if len(longestalias) > 0 {
		return e.aliases[longestalias] + rawurl[len(longestalias):]
	}

	return rawurl
}

func initAliases(e *endpointGitFinder, git Env) {
	prefix := "url."
	suffix := ".insteadof"
	for gitkey, gitval := range git.All() {
		if len(gitval) == 0 || !(strings.HasPrefix(gitkey, prefix) && strings.HasSuffix(gitkey, suffix)) {
			continue
		}
		if _, ok := e.aliases[gitval[len(gitval)-1]]; ok {
			fmt.Fprintf(os.Stderr, "WARNING: Multiple 'url.*.insteadof' keys with the same alias: %q\n", gitval)
		}
		e.aliases[gitval[len(gitval)-1]] = gitkey[len(prefix) : len(gitkey)-len(suffix)]
	}
}
<commit_msg>lfsapi/endpoint_finder: use `*URLConfig` to do access lookups<commit_after>package lfsapi

import (
	"fmt"
	"net/url"
	"os"
	"path"
	"strings"
	"sync"

	"github.com/git-lfs/git-lfs/config"
	"github.com/git-lfs/git-lfs/git"
	"github.com/rubyist/tracerx"
)

type Access string

const (
	NoneAccess      Access = "none"
	BasicAccess     Access = "basic"
	PrivateAccess   Access = "private"
	NegotiateAccess Access = "negotiate"
	NTLMAccess      Access = "ntlm"
	emptyAccess     Access = ""
	defaultRemote          = "origin"
)

type EndpointFinder interface {
	NewEndpointFromCloneURL(rawurl string) Endpoint
	NewEndpoint(rawurl string) Endpoint
	Endpoint(operation, remote string) Endpoint
	RemoteEndpoint(operation, remote string) Endpoint
	GitRemoteURL(remote string, forpush bool) string
	AccessFor(rawurl string) Access
	SetAccess(rawurl string, access Access)
	GitProtocol() string
}

type endpointGitFinder struct {
	git         Env
	gitProtocol string

	aliasMu sync.Mutex
	aliases map[string]string

	accessMu  sync.Mutex
	urlAccess map[string]Access
	urlConfig *config.URLConfig
}

func NewEndpointFinder(git Env) EndpointFinder {
	e := &endpointGitFinder{
		gitProtocol: "https",
		aliases:     make(map[string]string),
		urlAccess:   make(map[string]Access),
	}

	if git != nil {
		e.git = git
		e.urlConfig = config.NewURLConfig(e.git)
		if v, ok := git.Get("lfs.gitprotocol"); ok {
			e.gitProtocol = v
		}
		initAliases(e, git)
	}

	return e
}

func (e *endpointGitFinder) Endpoint(operation, remote string) Endpoint {
	ep := e.getEndpoint(operation, remote)
	ep.Operation = operation
	return ep
}

func (e *endpointGitFinder) getEndpoint(operation, remote string) Endpoint {
	if e.git == nil {
		return Endpoint{}
	}

	if operation == "upload" {
		if url, ok := e.git.Get("lfs.pushurl"); ok {
			return e.NewEndpoint(url)
		}
	}

	if url, ok := e.git.Get("lfs.url"); ok {
		return e.NewEndpoint(url)
	}

	if len(remote) > 0 && remote != defaultRemote {
		if e := e.RemoteEndpoint(operation, remote); len(e.Url) > 0 {
			return e
		}
	}

	return e.RemoteEndpoint(operation, defaultRemote)
}

func (e *endpointGitFinder) RemoteEndpoint(operation, remote string) Endpoint {
	if e.git == nil {
		return Endpoint{}
	}

	if len(remote) == 0 {
		remote = defaultRemote
	}

	// Support separate push URL if specified and pushing
	if operation == "upload" {
		if url, ok := e.git.Get("remote." + remote + ".lfspushurl"); ok {
			return e.NewEndpoint(url)
		}
	}
	if url, ok := e.git.Get("remote." + remote + ".lfsurl"); ok {
		return e.NewEndpoint(url)
	}

	// finally fall back on git remote url (also supports pushurl)
	if url := e.GitRemoteURL(remote, operation == "upload"); url != "" {
		return e.NewEndpointFromCloneURL(url)
	}

	return Endpoint{}
}

func (e *endpointGitFinder) GitRemoteURL(remote string, forpush bool) string {
	if e.git != nil {
		if forpush {
			if u, ok := e.git.Get("remote." + remote + ".pushurl"); ok {
				return u
			}
		}

		if u, ok := e.git.Get("remote." + remote + ".url"); ok {
			return u
		}
	}

	if err := git.ValidateRemote(remote); err == nil {
		return remote
	}

	return ""
}

func (e *endpointGitFinder) NewEndpointFromCloneURL(rawurl string) Endpoint {
	ep := e.NewEndpoint(rawurl)
	if ep.Url == UrlUnknown {
		return ep
	}

	if strings.HasSuffix(rawurl, "/") {
		ep.Url = rawurl[0 : len(rawurl)-1]
	}

	// When using main remote URL for HTTP, append info/lfs
	if path.Ext(ep.Url) == ".git" {
		ep.Url += "/info/lfs"
	} else {
		ep.Url += ".git/info/lfs"
	}

	return ep
}

func (e *endpointGitFinder) NewEndpoint(rawurl string) Endpoint {
	rawurl = e.ReplaceUrlAlias(rawurl)
	u, err := url.Parse(rawurl)
	if err != nil {
		return endpointFromBareSshUrl(rawurl)
	}

	switch u.Scheme {
	case "ssh":
		return endpointFromSshUrl(u)
	case "http", "https":
		return endpointFromHttpUrl(u)
	case "git":
		return endpointFromGitUrl(u, e)
	case "":
		return endpointFromBareSshUrl(u.String())
	default:
		// Just passthrough to preserve
		return Endpoint{Url: rawurl}
	}
}

func (e *endpointGitFinder) AccessFor(rawurl string) Access {
	if e.git == nil {
		return NoneAccess
	}

	accessurl := urlWithoutAuth(rawurl)

	e.accessMu.Lock()
	defer e.accessMu.Unlock()

	if cached, ok := e.urlAccess[accessurl]; ok {
		return cached
	}

	e.urlAccess[accessurl] = e.fetchGitAccess(accessurl)
	return e.urlAccess[accessurl]
}

func (e *endpointGitFinder) SetAccess(rawurl string, access Access) {
	accessurl := urlWithoutAuth(rawurl)
	key := fmt.Sprintf("lfs.%s.access", accessurl)
	tracerx.Printf("setting repository access to %s", access)

	e.accessMu.Lock()
	defer e.accessMu.Unlock()

	switch access {
	case emptyAccess, NoneAccess:
		git.Config.UnsetLocalKey("", key)
		e.urlAccess[accessurl] = NoneAccess
	default:
		git.Config.SetLocal("", key, string(access))
		e.urlAccess[accessurl] = access
	}
}

func urlWithoutAuth(rawurl string) string {
	if !strings.Contains(rawurl, "@") {
		return rawurl
	}

	u, err := url.Parse(rawurl)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error parsing URL %q: %s", rawurl, err)
		return rawurl
	}

	u.User = nil
	return u.String()
}

func (e *endpointGitFinder) fetchGitAccess(rawurl string) Access {
	if v, _ := e.urlConfig.Get("lfs", rawurl, "access"); len(v) > 0 {
		access := Access(strings.ToLower(v))
		if access == PrivateAccess {
			return BasicAccess
		}
		return access
	}
	return NoneAccess
}

func (e *endpointGitFinder) GitProtocol() string {
	return e.gitProtocol
}

// ReplaceUrlAlias returns a url with a prefix from a `url.*.insteadof` git
// config setting. If multiple aliases match, use the longest one.
// See https://git-scm.com/docs/git-config for Git's docs.
func (e *endpointGitFinder) ReplaceUrlAlias(rawurl string) string {
	e.aliasMu.Lock()
	defer e.aliasMu.Unlock()

	var longestalias string
	for alias, _ := range e.aliases {
		if !strings.HasPrefix(rawurl, alias) {
			continue
		}

		if longestalias < alias {
			longestalias = alias
		}
	}

	if len(longestalias) > 0 {
		return e.aliases[longestalias] + rawurl[len(longestalias):]
	}

	return rawurl
}

func initAliases(e *endpointGitFinder, git Env) {
	prefix := "url."
	suffix := ".insteadof"
	for gitkey, gitval := range git.All() {
		if len(gitval) == 0 || !(strings.HasPrefix(gitkey, prefix) && strings.HasSuffix(gitkey, suffix)) {
			continue
		}
		if _, ok := e.aliases[gitval[len(gitval)-1]]; ok {
			fmt.Fprintf(os.Stderr, "WARNING: Multiple 'url.*.insteadof' keys with the same alias: %q\n", gitval)
		}
		e.aliases[gitval[len(gitval)-1]] = gitkey[len(prefix) : len(gitkey)-len(suffix)]
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"
)

type Validator struct {
	root string // work dir
	code []Code // list of submissions
}

func NewValidator(dir string, train bool) (Validator, error) {
	var err error
	v := Validator{
		root: dir,
		code: make([]Code, 0, 2),
	}

	// find all 'higgsml-pred' executables
	exes := make([]string, 0)
	err = filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error {
		if fi.IsDir() {
			return nil
		}
		if strings.Contains(strings.ToLower(path), runscript) {
			exes = append(exes, path)
		}
		return err
	})

	if len(exes) <= 0 {
		return v, fmt.Errorf("hml: could not find any suitable executable in zip-file")
	}

	if len(exes) > 2 {
		return v, fmt.Errorf("hml: too many higgsml-pred executables (got=%d, max=%d)", len(exes), 2)
	}

	for _, exe := range exes {
		dir := filepath.Dir(exe)
		code, err := NewCode(dir, train)
		if err != nil {
			return v, err
		}
		code.Name = dir[len(v.root):]
		if code.Name[0] == '/' {
			code.Name = code.Name[1:]
		}
		v.code = append(v.code, code)
	}

	return v, err
}

func (v Validator) Run() error {
	var err error

	for i, code := range v.code {
		start := time.Now()
		printf("\n=== code submission #%d/%d (%s)...\n", i+1, len(v.code), code.Name)
		err = code.run()
		if err != nil {
			return err
		}
		printf("=== code submission #%d/%d (%s)... [ok] (delta=%v)\n",
			i+1, len(v.code), code.Name,
			time.Since(start),
		)
	}

	return err
}

type Code struct {
	Root    string // directory containing sources/binaries
	Name    string // name of this code submission (e.g. team/code)
	Train   string // path to training executable
	Pred    string // path to prediction executable
	Trained string // path to trained data parameters

	License string // path to LICENSE file
	Readme  string // path to README file

	DoTraining bool

	trainfile string // path to training.csv file
	testfile  string // path to test.csv file
}

func NewCode(dir string, train bool) (Code, error) {
	var err error

	code := Code{
		Root:       dir,
		Name:       filepath.Base(dir),
		Trained:    "trained.dat",
		DoTraining: train,
		trainfile:  trainfile,
		testfile:   testfile,
	}

	// FIXME: handle multiple-submissions zipfiles
	//        presumably: 1 directory per submission.

	exes := make([]string, 0)
	// find executables
	err = filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error {
		if fi.IsDir() {
			return nil
		}
		if strings.Contains(strings.ToLower(path), "readme") {
			code.Readme = path
		}

		if strings.Contains(strings.ToLower(path), "license") {
			code.License = path
		}

		if strings.Contains(strings.ToLower(path), "trained.dat") {
			code.Trained = path
		}

		// FIXME: better way ?
		if !strings.Contains(fi.Mode().String(), "x") {
			return nil
		}
		exes = append(exes, path)
		// printf(">>> %s\n", path)
		if strings.HasSuffix(strings.ToLower(path), trainscript) {
			code.Train = path
		}
		if strings.HasSuffix(strings.ToLower(path), runscript) {
			code.Pred = path
		}
		return err
	})

	if len(exes) <= 0 {
		return code, fmt.Errorf("hml: could not find any suitable executable in zip-file")
	}

	if code.Train == "" && code.Pred == "" {
		// take first one
		code.Train = exes[0]
		code.Pred = exes[0]
	}

	if code.Train == "" && code.Pred != "" {
		code.Train = code.Pred
	}

	if code.Train != "" && code.Pred == "" {
		code.Pred = code.Train
	}

	if code.License == "" {
		return code, fmt.Errorf("hml: could not find a LICENSE file under [%s]", code.Root)
	}

	return code, err
}

func (code Code) run() error {
	var err error

	dir := filepath.Join(code.Root, ".higgsml-work")
	err = os.MkdirAll(dir, 0755)
	if err != nil {
		return err
	}

	if code.DoTraining {
		err = code.run_training(dir)
		if err != nil {
			return err
		}
		printf("\n")
	}

	err = code.run_pred(dir)
	if err != nil {
		return err
	}

	return err
}

func (code Code) run_training(dir string) error {
	var err error
	errch := make(chan error)

	printf("::: run training...\n")

	cmd := exec.Command(code.Train, code.trainfile, pdir(dir, "trained.dat"))
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	cmd.Stdin = os.Stdin
	cmd.Dir = code.Root

	start := time.Now()
	go func() {
		err = cmd.Start()
		if err != nil {
			errch <- err
			return
		}
		errch <- cmd.Wait()
	}()

	duration := 1 * time.Hour
	select {
	case <-time.After(duration):
		cmd.Process.Kill()
		return fmt.Errorf("hml: training timed out (%v)\n", duration)
	case err = <-errch:
		break
	}

	if err != nil {
		printf("::: run training... [ERR] (delta=%v)\n", time.Since(start))
		return err
	}

	printf("::: run training... [ok] (delta=%v)\n", time.Since(start))
	return err
}

func (code Code) run_pred(dir string) error {
	var err error
	errch := make(chan error)

	printf("::: run prediction...\n")

	trained := code.Trained
	if code.DoTraining {
		// if we ran the training, then use that file instead.
		trained = pdir(dir, "trained.dat")
	}

	cmd := exec.Command(code.Pred, code.testfile, trained, pdir(dir, "scores_test.csv"))
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	cmd.Stdin = os.Stdin
	cmd.Dir = code.Root

	start := time.Now()
	go func() {
		err = cmd.Start()
		if err != nil {
			errch <- err
			return
		}
		errch <- cmd.Wait()
	}()

	duration := 1 * time.Hour
	select {
	case <-time.After(duration):
		cmd.Process.Kill()
		return fmt.Errorf("hml: prediction timed out (%v)\n", duration)
	case err = <-errch:
		break
	}

	if err != nil {
		printf("::: run prediction... [ERR] (delta=%v)\n", time.Since(start))
		return err
	}

	printf("::: run prediction... [ok] (delta=%v)\n", time.Since(start))
	return err
}

func pdir(dirs ...string) string {
	return filepath.Join(dirs...)
}
<commit_msg>hml-validate: use constants<commit_after>package main

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"
)

type Validator struct {
	root string // work dir
	code []Code // list of submissions
}

func NewValidator(dir string, train bool) (Validator, error) {
	var err error
	v := Validator{
		root: dir,
		code: make([]Code, 0, 2),
	}

	// find all 'higgsml-pred' executables
	exes := make([]string, 0)
	err = filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error {
		if fi.IsDir() {
			return nil
		}
		if strings.Contains(strings.ToLower(path), runscript) {
			exes = append(exes, path)
		}
		return err
	})

	if len(exes) <= 0 {
		return v, fmt.Errorf("hml: could not find any suitable executable in zip-file")
	}

	if len(exes) > 2 {
		return v, fmt.Errorf("hml: too many higgsml-pred executables (got=%d, max=%d)", len(exes), 2)
	}

	for _, exe := range exes {
		dir := filepath.Dir(exe)
		code, err := NewCode(dir, train)
		if err != nil {
			return v, err
		}
		code.Name = dir[len(v.root):]
		if code.Name[0] == '/' {
			code.Name = code.Name[1:]
		}
		v.code = append(v.code, code)
	}

	return v, err
}

func (v Validator) Run() error {
	var err error

	for i, code := range v.code {
		start := time.Now()
		printf("\n=== code submission #%d/%d (%s)...\n", i+1, len(v.code), code.Name)
		err = code.run()
		if err != nil {
			return err
		}
		printf("=== code submission #%d/%d (%s)... [ok] (delta=%v)\n",
			i+1, len(v.code), code.Name,
			time.Since(start),
		)
	}

	return err
}

type Code struct {
	Root    string // directory containing sources/binaries
	Name    string // name of this code submission (e.g. team/code)
	Train   string // path to training executable
	Pred    string // path to prediction executable
	Trained string // path to trained data parameters

	License string // path to LICENSE file
	Readme  string // path to README file

	DoTraining bool

	trainfile string // path to training.csv file
	testfile  string // path to test.csv file
}

func NewCode(dir string, train bool) (Code, error) {
	var err error

	code := Code{
		Root:       dir,
		Name:       filepath.Base(dir),
		Trained:    trainedname,
		DoTraining: train,
		trainfile:  trainfile,
		testfile:   testfile,
	}

	// FIXME: handle multiple-submissions zipfiles
	//        presumably: 1 directory per submission.

	exes := make([]string, 0)
	// find executables
	err = filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error {
		if fi.IsDir() {
			return nil
		}
		if strings.Contains(strings.ToLower(path), "readme") {
			code.Readme = path
		}

		if strings.Contains(strings.ToLower(path), "license") {
			code.License = path
		}

		if strings.Contains(strings.ToLower(path), trainedname) {
			code.Trained = path
		}

		// FIXME: better way ?
		if !strings.Contains(fi.Mode().String(), "x") {
			return nil
		}
		exes = append(exes, path)
		// printf(">>> %s\n", path)
		if strings.HasSuffix(strings.ToLower(path), trainscript) {
			code.Train = path
		}
		if strings.HasSuffix(strings.ToLower(path), runscript) {
			code.Pred = path
		}
		return err
	})

	if len(exes) <= 0 {
		return code, fmt.Errorf("hml: could not find any suitable executable in zip-file")
	}

	if code.Train == "" && code.Pred == "" {
		// take first one
		code.Train = exes[0]
		code.Pred = exes[0]
	}

	if code.Train == "" && code.Pred != "" {
		code.Train = code.Pred
	}

	if code.Train != "" && code.Pred == "" {
		code.Pred = code.Train
	}

	if code.License == "" {
		return code, fmt.Errorf("hml: could not find a LICENSE file under [%s]", code.Root)
	}

	return code, err
}

func (code Code) run() error {
	var err error

	dir := filepath.Join(code.Root, ".higgsml-work")
	err = os.MkdirAll(dir, 0755)
	if err != nil {
		return err
	}

	if code.DoTraining {
		err = code.run_training(dir)
		if err != nil {
			return err
		}
		printf("\n")
	}

	err = code.run_pred(dir)
	if err != nil {
		return err
	}

	return err
}

func (code Code) run_training(dir string) error {
	var err error
	errch := make(chan error)

	printf("::: run training...\n")

	trained := pdir(dir, trainedname)

	cmd := exec.Command(code.Train, code.trainfile, trained)
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	cmd.Stdin = os.Stdin
	cmd.Dir = code.Root

	start := time.Now()
	go func() {
		err = cmd.Start()
		if err != nil {
			errch <- err
			return
		}
		errch <- cmd.Wait()
	}()

	duration := 1 * time.Hour
	select {
	case <-time.After(duration):
		cmd.Process.Kill()
		return fmt.Errorf("hml: training timed out (%v)\n", duration)
	case err = <-errch:
		break
	}

	if err != nil {
		printf("::: run training... [ERR] (delta=%v)\n", time.Since(start))
		return err
	}

	printf("::: run training... [ok] (delta=%v)\n", time.Since(start))
	return err
}

func (code Code) run_pred(dir string) error {
	var err error
	errch := make(chan error)

	printf("::: run prediction...\n")

	trained := code.Trained
	if code.DoTraining {
		// if we ran the training, then use that file instead.
		trained = pdir(dir, trainedname)
	}

	results := pdir(dir, csv_results)
	cmd := exec.Command(code.Pred, code.testfile, trained, results)
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	cmd.Stdin = os.Stdin
	cmd.Dir = code.Root

	start := time.Now()
	go func() {
		err = cmd.Start()
		if err != nil {
			errch <- err
			return
		}
		errch <- cmd.Wait()
	}()

	duration := 1 * time.Hour
	select {
	case <-time.After(duration):
		cmd.Process.Kill()
		return fmt.Errorf("hml: prediction timed out (%v)\n", duration)
	case err = <-errch:
		break
	}

	if err != nil {
		printf("::: run prediction... [ERR] (delta=%v)\n", time.Since(start))
		return err
	}

	printf("::: run prediction... [ok] (delta=%v)\n", time.Since(start))
	return err
}

func pdir(dirs ...string) string {
	return filepath.Join(dirs...)
}
<|endoftext|>