<commit_before>// +build e2e

/*
Copyright 2018 Knative Authors LLC
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package test

import (
	"fmt"
	"io/ioutil"
	"os"
	"testing"
	"time"

	knativetest "github.com/knative/pkg/test"
	"github.com/tektoncd/pipeline/pkg/apis/pipeline/v1alpha1"
	tb "github.com/tektoncd/pipeline/test/builder"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

const (
	helloworldResourceName    = "helloworldgit"
	addFileTaskName           = "add-file-to-resource-task"
	readFileTaskName          = "read-new-file-task"
	bucketTestPipelineName    = "bucket-test-pipeline"
	bucketTestPipelineRunName = "bucket-test-pipeline-run"
	systemNamespace           = "tekton-pipelines"
	bucketSecretName          = "bucket-secret"
	bucketSecretKey           = "bucket-secret-key"
)

// TestStorageBucketPipelineRun is an integration test that will verify a pipeline
// can use a bucket for temporary storage of artifacts shared between tasks
func TestStorageBucketPipelineRun(t *testing.T) {
	configFilePath := os.Getenv("GCP_SERVICE_ACCOUNT_KEY_PATH")
	if configFilePath == "" {
		t.Skip("GCP_SERVICE_ACCOUNT_KEY_PATH variable is not set.")
	}
	c, namespace := setup(t)
	t.Parallel()

	knativetest.CleanupOnInterrupt(func() { tearDown(t, c, namespace) }, t.Logf)
	defer tearDown(t, c, namespace)

	bucketName := fmt.Sprintf("build-pipeline-test-%s-%d", namespace, time.Now().Unix())

	t.Logf("Creating Secret %s", bucketSecretName)
	if _, err := c.KubeClient.Kube.CoreV1().Secrets(namespace).Create(getBucketSecret(t, configFilePath, namespace)); err != nil {
		t.Fatalf("Failed to create Secret `%s`: %s", bucketSecretName, err)
	}
	defer deleteBucketSecret(c, t, namespace)

	t.Logf("Creating GCS bucket %s", bucketName)
	createbuckettask := tb.Task("createbuckettask", namespace, tb.TaskSpec(
		tb.TaskVolume("bucket-secret-volume", tb.VolumeSource(corev1.VolumeSource{
			Secret: &corev1.SecretVolumeSource{
				SecretName: bucketSecretName,
			},
		})),
		tb.Step("step1", "google/cloud-sdk:alpine",
			tb.Command("/bin/bash"),
			tb.Args("-c", fmt.Sprintf("gcloud auth activate-service-account --key-file /var/secret/bucket-secret/bucket-secret-key && gsutil mb gs://%s", bucketName)),
			tb.VolumeMount("bucket-secret-volume", fmt.Sprintf("/var/secret/%s", bucketSecretName)),
			tb.EnvVar("CREDENTIALS", fmt.Sprintf("/var/secret/%s/%s", bucketSecretName, bucketSecretKey)),
		),
	),
	)

	t.Logf("Creating Task %s", "createbuckettask")
	if _, err := c.TaskClient.Create(createbuckettask); err != nil {
		t.Fatalf("Failed to create Task `%s`: %s", "createbuckettask", err)
	}

	createbuckettaskrun := tb.TaskRun("createbuckettaskrun", namespace,
		tb.TaskRunSpec(tb.TaskRunTaskRef("createbuckettask")))

	t.Logf("Creating TaskRun %s", "createbuckettaskrun")
	if _, err := c.TaskRunClient.Create(createbuckettaskrun); err != nil {
		t.Fatalf("Failed to create TaskRun `%s`: %s", "createbuckettaskrun", err)
	}

	if err := WaitForTaskRunState(c, "createbuckettaskrun", TaskRunSucceed("createbuckettaskrun"), "TaskRunSuccess"); err != nil {
		t.Errorf("Error waiting for TaskRun %s to finish: %s", "createbuckettaskrun", err)
	}

	defer runTaskToDeleteBucket(c, t, namespace, bucketName, bucketSecretName, bucketSecretKey)

	originalConfigMap, err := c.KubeClient.Kube.CoreV1().ConfigMaps(systemNamespace).Get(v1alpha1.BucketConfigName, metav1.GetOptions{})
	if err != nil {
		t.Fatalf("Failed to get ConfigMap `%s`: %s", v1alpha1.BucketConfigName, err)
	}
	originalConfigMapData := originalConfigMap.Data

	t.Logf("Creating ConfigMap %s", v1alpha1.BucketConfigName)
	configMapData := map[string]string{
		v1alpha1.BucketLocationKey:              fmt.Sprintf("gs://%s", bucketName),
		v1alpha1.BucketServiceAccountSecretName: bucketSecretName,
		v1alpha1.BucketServiceAccountSecretKey:  bucketSecretKey,
	}
	c.KubeClient.UpdateConfigMap(systemNamespace, v1alpha1.BucketConfigName, configMapData)
	defer resetConfigMap(c, systemNamespace, v1alpha1.BucketConfigName, originalConfigMapData)

	t.Logf("Creating Git PipelineResource %s", helloworldResourceName)
	helloworldResource := tb.PipelineResource(helloworldResourceName, namespace, tb.PipelineResourceSpec(
		v1alpha1.PipelineResourceTypeGit,
		tb.PipelineResourceSpecParam("Url", "https://github.com/pivotal-nader-ziada/gohelloworld"),
		tb.PipelineResourceSpecParam("Revision", "master"),
	),
	)
	if _, err := c.PipelineResourceClient.Create(helloworldResource); err != nil {
		t.Fatalf("Failed to create Pipeline Resource `%s`: %s", helloworldResourceName, err)
	}

	t.Logf("Creating Task %s", addFileTaskName)
	addFileTask := tb.Task(addFileTaskName, namespace, tb.TaskSpec(
		tb.TaskInputs(tb.InputsResource(helloworldResourceName, v1alpha1.PipelineResourceTypeGit)),
		tb.TaskOutputs(tb.OutputsResource(helloworldResourceName, v1alpha1.PipelineResourceTypeGit)),
		tb.Step("addfile", "ubuntu", tb.Command("/bin/bash"),
			tb.Args("-c", "echo stuff > /workspace/helloworldgit/newfile"),
		),
	))
	if _, err := c.TaskClient.Create(addFileTask); err != nil {
		t.Fatalf("Failed to create Task `%s`: %s", addFileTaskName, err)
	}

	t.Logf("Creating Task %s", readFileTaskName)
	readFileTask := tb.Task(readFileTaskName, namespace, tb.TaskSpec(
		tb.TaskInputs(tb.InputsResource(helloworldResourceName, v1alpha1.PipelineResourceTypeGit)),
		tb.Step("readfile", "ubuntu", tb.Command("/bin/bash"),
			tb.Args("-c", "cat /workspace/helloworldgit/newfile"),
		),
	))
	if _, err := c.TaskClient.Create(readFileTask); err != nil {
		t.Fatalf("Failed to create Task `%s`: %s", readFileTaskName, err)
	}

	t.Logf("Creating Pipeline %s", bucketTestPipelineName)
	bucketTestPipeline := tb.Pipeline(bucketTestPipelineName, namespace, tb.PipelineSpec(
		tb.PipelineDeclaredResource("source-repo", "git"),
		tb.PipelineTask("addfile", addFileTaskName,
			tb.PipelineTaskInputResource("helloworldgit", "source-repo"),
			tb.PipelineTaskOutputResource("helloworldgit", "source-repo"),
		),
		tb.PipelineTask("readfile", readFileTaskName,
			tb.PipelineTaskInputResource("helloworldgit", "source-repo", tb.From("addfile")),
		),
	))
	if _, err := c.PipelineClient.Create(bucketTestPipeline); err != nil {
		t.Fatalf("Failed to create Pipeline `%s`: %s", bucketTestPipelineName, err)
	}

	t.Logf("Creating PipelineRun %s", bucketTestPipelineRunName)
	bucketTestPipelineRun := tb.PipelineRun(bucketTestPipelineRunName, namespace, tb.PipelineRunSpec(
		bucketTestPipelineName,
		tb.PipelineRunResourceBinding("source-repo", tb.PipelineResourceBindingRef(helloworldResourceName)),
	))
	if _, err := c.PipelineRunClient.Create(bucketTestPipelineRun); err != nil {
		t.Fatalf("Failed to create PipelineRun `%s`: %s", bucketTestPipelineRunName, err)
	}

	// Verify status of PipelineRun (wait for it)
	if err := WaitForPipelineRunState(c, bucketTestPipelineRunName, timeout, PipelineRunSucceed(bucketTestPipelineRunName), "PipelineRunCompleted"); err != nil {
		t.Errorf("Error waiting for PipelineRun %s to finish: %s", bucketTestPipelineRunName, err)
		taskruns, err := c.TaskRunClient.List(metav1.ListOptions{})
		if err != nil {
			t.Errorf("Error getting TaskRun list for PipelineRun %s %s", bucketTestPipelineRunName, err)
		}
		for _, tr := range taskruns.Items {
			if tr.Status.PodName != "" {
				CollectBuildLogs(c, tr.Status.PodName, namespace, t.Logf)
			}
		}
		t.Fatalf("PipelineRun execution failed")
	}
}

func getBucketSecret(t *testing.T, configFilePath, namespace string) *corev1.Secret {
	t.Helper()
	f, err := ioutil.ReadFile(configFilePath)
	if err != nil {
		t.Fatalf("Failed to read json key file %s at path %s", err, configFilePath)
	}
	return &corev1.Secret{
		ObjectMeta: metav1.ObjectMeta{
			Namespace: namespace,
			Name:      bucketSecretName,
		},
		StringData: map[string]string{
			bucketSecretKey: string(f),
		},
	}
}

func deleteBucketSecret(c *clients, t *testing.T, namespace string) {
	if err := c.KubeClient.Kube.CoreV1().Secrets(namespace).Delete(bucketSecretName, &metav1.DeleteOptions{}); err != nil {
		t.Fatalf("Failed to delete Secret `%s`: %s", bucketSecretName, err)
	}
}

func resetConfigMap(c *clients, namespace, configName string, values map[string]string) error {
	return c.KubeClient.UpdateConfigMap(namespace, configName, values)
}

func runTaskToDeleteBucket(c *clients, t *testing.T, namespace, bucketName, bucketSecretName, bucketSecretKey string) {
	deletelbuckettask := tb.Task("deletelbuckettask", namespace, tb.TaskSpec(
		tb.TaskVolume("bucket-secret-volume", tb.VolumeSource(corev1.VolumeSource{
			Secret: &corev1.SecretVolumeSource{
				SecretName: bucketSecretName,
			},
		})),
		tb.Step("step1", "google/cloud-sdk:alpine",
			tb.Command("/bin/bash"),
			tb.Args("-c", fmt.Sprintf("gcloud auth activate-service-account --key-file /var/secret/bucket-secret/bucket-secret-key && gsutil rm -r gs://%s", bucketName)),
			tb.VolumeMount("bucket-secret-volume", fmt.Sprintf("/var/secret/%s", bucketSecretName)),
			tb.EnvVar("CREDENTIALS", fmt.Sprintf("/var/secret/%s/%s", bucketSecretName, bucketSecretKey)),
		),
	),
	)

	t.Logf("Creating Task %s", "deletelbuckettask")
	if _, err := c.TaskClient.Create(deletelbuckettask); err != nil {
		t.Fatalf("Failed to create Task `%s`: %s", "deletelbuckettask", err)
	}

	deletelbuckettaskrun := tb.TaskRun("deletelbuckettaskrun", namespace,
		tb.TaskRunSpec(tb.TaskRunTaskRef("deletelbuckettask")))

	t.Logf("Creating TaskRun %s", "deletelbuckettaskrun")
	if _, err := c.TaskRunClient.Create(deletelbuckettaskrun); err != nil {
		t.Fatalf("Failed to create TaskRun `%s`: %s", "deletelbuckettaskrun", err)
	}

	if err := WaitForTaskRunState(c, "deletelbuckettaskrun", TaskRunSucceed("deletelbuckettaskrun"), "TaskRunSuccess"); err != nil {
		t.Errorf("Error waiting for TaskRun %s to finish: %s", "deletelbuckettaskrun", err)
	}
}
<commit_msg>Fix TestStorageBucketPipelineRun to actually, y'know, run.<commit_after>// +build e2e

/*
Copyright 2018 Knative Authors LLC
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package test

import (
	"fmt"
	"io/ioutil"
	"os"
	"testing"
	"time"

	knativetest "github.com/knative/pkg/test"
	"github.com/tektoncd/pipeline/pkg/apis/pipeline/v1alpha1"
	tb "github.com/tektoncd/pipeline/test/builder"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

const (
	helloworldResourceName    = "helloworldgit"
	addFileTaskName           = "add-file-to-resource-task"
	readFileTaskName          = "read-new-file-task"
	bucketTestPipelineName    = "bucket-test-pipeline"
	bucketTestPipelineRunName = "bucket-test-pipeline-run"
	systemNamespace           = "tekton-pipelines"
	bucketSecretName          = "bucket-secret"
	bucketSecretKey           = "bucket-secret-key"
)

// TestStorageBucketPipelineRun is an integration test that will verify a pipeline
// can use a bucket for temporary storage of artifacts shared between tasks
func TestStorageBucketPipelineRun(t *testing.T) {
	configFilePath := os.Getenv("GCP_SERVICE_ACCOUNT_KEY_PATH")
	if configFilePath == "" {
		t.Skip("GCP_SERVICE_ACCOUNT_KEY_PATH variable is not set.")
	}
	c, namespace := setup(t)
	t.Parallel()

	knativetest.CleanupOnInterrupt(func() { tearDown(t, c, namespace) }, t.Logf)
	defer tearDown(t, c, namespace)

	bucketName := fmt.Sprintf("build-pipeline-test-%s-%d", namespace, time.Now().Unix())

	t.Logf("Creating Secret %s", bucketSecretName)
	if _, err := c.KubeClient.Kube.CoreV1().Secrets(namespace).Create(getBucketSecret(t, configFilePath, namespace)); err != nil {
		t.Fatalf("Failed to create Secret `%s`: %s", bucketSecretName, err)
	}
	defer deleteBucketSecret(c, t, namespace)

	t.Logf("Creating GCS bucket %s", bucketName)
	createbuckettask := tb.Task("createbuckettask", namespace, tb.TaskSpec(
		tb.TaskVolume("bucket-secret-volume", tb.VolumeSource(corev1.VolumeSource{
			Secret: &corev1.SecretVolumeSource{
				SecretName: bucketSecretName,
			},
		})),
		tb.Step("step1", "google/cloud-sdk:alpine",
			tb.Command("/bin/bash"),
			tb.Args("-c", fmt.Sprintf("gcloud auth activate-service-account --key-file /var/secret/bucket-secret/bucket-secret-key && gsutil mb gs://%s", bucketName)),
			tb.VolumeMount("bucket-secret-volume", fmt.Sprintf("/var/secret/%s", bucketSecretName)),
			tb.EnvVar("CREDENTIALS", fmt.Sprintf("/var/secret/%s/%s", bucketSecretName, bucketSecretKey)),
		),
	),
	)

	t.Logf("Creating Task %s", "createbuckettask")
	if _, err := c.TaskClient.Create(createbuckettask); err != nil {
		t.Fatalf("Failed to create Task `%s`: %s", "createbuckettask", err)
	}

	createbuckettaskrun := tb.TaskRun("createbuckettaskrun", namespace,
		tb.TaskRunSpec(tb.TaskRunTaskRef("createbuckettask")))

	t.Logf("Creating TaskRun %s", "createbuckettaskrun")
	if _, err := c.TaskRunClient.Create(createbuckettaskrun); err != nil {
		t.Fatalf("Failed to create TaskRun `%s`: %s", "createbuckettaskrun", err)
	}

	if err := WaitForTaskRunState(c, "createbuckettaskrun", TaskRunSucceed("createbuckettaskrun"), "TaskRunSuccess"); err != nil {
		t.Errorf("Error waiting for TaskRun %s to finish: %s", "createbuckettaskrun", err)
	}

	defer runTaskToDeleteBucket(c, t, namespace, bucketName, bucketSecretName, bucketSecretKey)

	originalConfigMap, err := c.KubeClient.Kube.CoreV1().ConfigMaps(systemNamespace).Get(v1alpha1.BucketConfigName, metav1.GetOptions{})
	if err != nil {
		t.Fatalf("Failed to get ConfigMap `%s`: %s", v1alpha1.BucketConfigName, err)
	}
	originalConfigMapData := originalConfigMap.Data

	t.Logf("Creating ConfigMap %s", v1alpha1.BucketConfigName)
	configMapData := map[string]string{
		v1alpha1.BucketLocationKey:              fmt.Sprintf("gs://%s", bucketName),
		v1alpha1.BucketServiceAccountSecretName: bucketSecretName,
		v1alpha1.BucketServiceAccountSecretKey:  bucketSecretKey,
	}
	updateConfigMap(c.KubeClient, systemNamespace, v1alpha1.BucketConfigName, configMapData)
	defer resetConfigMap(c, systemNamespace, v1alpha1.BucketConfigName, originalConfigMapData)

	t.Logf("Creating Git PipelineResource %s", helloworldResourceName)
	helloworldResource := tb.PipelineResource(helloworldResourceName, namespace, tb.PipelineResourceSpec(
		v1alpha1.PipelineResourceTypeGit,
		tb.PipelineResourceSpecParam("Url", "https://github.com/pivotal-nader-ziada/gohelloworld"),
		tb.PipelineResourceSpecParam("Revision", "master"),
	),
	)
	if _, err := c.PipelineResourceClient.Create(helloworldResource); err != nil {
		t.Fatalf("Failed to create Pipeline Resource `%s`: %s", helloworldResourceName, err)
	}

	t.Logf("Creating Task %s", addFileTaskName)
	addFileTask := tb.Task(addFileTaskName, namespace, tb.TaskSpec(
		tb.TaskInputs(tb.InputsResource(helloworldResourceName, v1alpha1.PipelineResourceTypeGit)),
		tb.TaskOutputs(tb.OutputsResource(helloworldResourceName, v1alpha1.PipelineResourceTypeGit)),
		tb.Step("addfile", "ubuntu", tb.Command("/bin/bash"),
			tb.Args("-c", "echo stuff > /workspace/helloworldgit/newfile"),
		),
	))
	if _, err := c.TaskClient.Create(addFileTask); err != nil {
		t.Fatalf("Failed to create Task `%s`: %s", addFileTaskName, err)
	}

	t.Logf("Creating Task %s", readFileTaskName)
	readFileTask := tb.Task(readFileTaskName, namespace, tb.TaskSpec(
		tb.TaskInputs(tb.InputsResource(helloworldResourceName, v1alpha1.PipelineResourceTypeGit)),
		tb.Step("readfile", "ubuntu", tb.Command("/bin/bash"),
			tb.Args("-c", "cat /workspace/helloworldgit/newfile"),
		),
	))
	if _, err := c.TaskClient.Create(readFileTask); err != nil {
		t.Fatalf("Failed to create Task `%s`: %s", readFileTaskName, err)
	}

	t.Logf("Creating Pipeline %s", bucketTestPipelineName)
	bucketTestPipeline := tb.Pipeline(bucketTestPipelineName, namespace, tb.PipelineSpec(
		tb.PipelineDeclaredResource("source-repo", "git"),
		tb.PipelineTask("addfile", addFileTaskName,
			tb.PipelineTaskInputResource("helloworldgit", "source-repo"),
			tb.PipelineTaskOutputResource("helloworldgit", "source-repo"),
		),
		tb.PipelineTask("readfile", readFileTaskName,
			tb.PipelineTaskInputResource("helloworldgit", "source-repo", tb.From("addfile")),
		),
	))
	if _, err := c.PipelineClient.Create(bucketTestPipeline); err != nil {
		t.Fatalf("Failed to create Pipeline `%s`: %s", bucketTestPipelineName, err)
	}

	t.Logf("Creating PipelineRun %s", bucketTestPipelineRunName)
	bucketTestPipelineRun := tb.PipelineRun(bucketTestPipelineRunName, namespace, tb.PipelineRunSpec(
		bucketTestPipelineName,
		tb.PipelineRunResourceBinding("source-repo", tb.PipelineResourceBindingRef(helloworldResourceName)),
	))
	if _, err := c.PipelineRunClient.Create(bucketTestPipelineRun); err != nil {
		t.Fatalf("Failed to create PipelineRun `%s`: %s", bucketTestPipelineRunName, err)
	}

	// Verify status of PipelineRun (wait for it)
	if err := WaitForPipelineRunState(c, bucketTestPipelineRunName, timeout, PipelineRunSucceed(bucketTestPipelineRunName), "PipelineRunCompleted"); err != nil {
		t.Errorf("Error waiting for PipelineRun %s to finish: %s", bucketTestPipelineRunName, err)
		taskruns, err := c.TaskRunClient.List(metav1.ListOptions{})
		if err != nil {
			t.Errorf("Error getting TaskRun list for PipelineRun %s %s", bucketTestPipelineRunName, err)
		}
		for _, tr := range taskruns.Items {
			if tr.Status.PodName != "" {
				CollectBuildLogs(c, tr.Status.PodName, namespace, t.Logf)
			}
		}
		t.Fatalf("PipelineRun execution failed")
	}
}

// updateConfigMap updates the config map for specified @name with values. We can't use the one from knativetest because
// it assumes that Data is already a non-nil map, and by default, it isn't!
func updateConfigMap(client *knativetest.KubeClient, name string, configName string, values map[string]string) error {
	configMap, err := client.GetConfigMap(name).Get(configName, metav1.GetOptions{})
	if err != nil {
		return err
	}

	if configMap.Data == nil {
		configMap.Data = make(map[string]string)
	}

	for key, value := range values {
		configMap.Data[key] = value
	}

	_, err = client.GetConfigMap(name).Update(configMap)
	return err
}

func getBucketSecret(t *testing.T, configFilePath, namespace string) *corev1.Secret {
	t.Helper()
	f, err := ioutil.ReadFile(configFilePath)
	if err != nil {
		t.Fatalf("Failed to read json key file %s at path %s", err, configFilePath)
	}
	return &corev1.Secret{
		ObjectMeta: metav1.ObjectMeta{
			Namespace: namespace,
			Name:      bucketSecretName,
		},
		StringData: map[string]string{
			bucketSecretKey: string(f),
		},
	}
}

func deleteBucketSecret(c *clients, t *testing.T, namespace string) {
	if err := c.KubeClient.Kube.CoreV1().Secrets(namespace).Delete(bucketSecretName, &metav1.DeleteOptions{}); err != nil {
		t.Fatalf("Failed to delete Secret `%s`: %s", bucketSecretName, err)
	}
}

func resetConfigMap(c *clients, namespace, configName string, values map[string]string) error {
	return updateConfigMap(c.KubeClient, namespace, configName, values)
}

func runTaskToDeleteBucket(c *clients, t *testing.T, namespace, bucketName, bucketSecretName, bucketSecretKey string) {
	deletelbuckettask := tb.Task("deletelbuckettask", namespace, tb.TaskSpec(
		tb.TaskVolume("bucket-secret-volume", tb.VolumeSource(corev1.VolumeSource{
			Secret: &corev1.SecretVolumeSource{
				SecretName: bucketSecretName,
			},
		})),
		tb.Step("step1", "google/cloud-sdk:alpine",
			tb.Command("/bin/bash"),
			tb.Args("-c", fmt.Sprintf("gcloud auth activate-service-account --key-file /var/secret/bucket-secret/bucket-secret-key && gsutil rm -r gs://%s", bucketName)),
			tb.VolumeMount("bucket-secret-volume", fmt.Sprintf("/var/secret/%s", bucketSecretName)),
			tb.EnvVar("CREDENTIALS", fmt.Sprintf("/var/secret/%s/%s", bucketSecretName, bucketSecretKey)),
		),
	),
	)

	t.Logf("Creating Task %s", "deletelbuckettask")
	if _, err := c.TaskClient.Create(deletelbuckettask); err != nil {
		t.Fatalf("Failed to create Task `%s`: %s", "deletelbuckettask", err)
	}

	deletelbuckettaskrun := tb.TaskRun("deletelbuckettaskrun", namespace,
		tb.TaskRunSpec(tb.TaskRunTaskRef("deletelbuckettask")))

	t.Logf("Creating TaskRun %s", "deletelbuckettaskrun")
	if _, err := c.TaskRunClient.Create(deletelbuckettaskrun); err != nil {
		t.Fatalf("Failed to create TaskRun `%s`: %s", "deletelbuckettaskrun", err)
	}

	if err := WaitForTaskRunState(c, "deletelbuckettaskrun", TaskRunSucceed("deletelbuckettaskrun"), "TaskRunSuccess"); err != nil {
		t.Errorf("Error waiting for TaskRun %s to finish: %s", "deletelbuckettaskrun", err)
	}
}
<|endoftext|>


<commit_before>package buf

import (
	"os"
	"strconv"
	"sync"
)

// Pool provides functionality to generate and recycle buffers on demand.
type Pool interface {
	// Allocate either returns a unused buffer from the pool, or generates a new one from system.
	Allocate() *Buffer
	// Free recycles the given buffer.
	Free(*Buffer)
}

// SyncPool is a buffer pool based on sync.Pool
type SyncPool struct {
	allocator *sync.Pool
}

// NewSyncPool creates a SyncPool with given buffer size.
func NewSyncPool(bufferSize uint32) *SyncPool {
	pool := &SyncPool{
		allocator: &sync.Pool{
			New: func() interface{} { return make([]byte, bufferSize) },
		},
	}
	return pool
}

// Allocate implements Pool.Allocate().
func (p *SyncPool) Allocate() *Buffer {
	return &Buffer{
		v:    p.allocator.Get().([]byte),
		pool: p,
	}
}

// Free implements Pool.Free().
func (p *SyncPool) Free(buffer *Buffer) {
	rawBuffer := buffer.v
	if rawBuffer == nil {
		return
	}
	p.allocator.Put(rawBuffer)
}

// BufferPool is a Pool that utilizes an internal cache.
type BufferPool struct {
	chain     chan []byte
	allocator *sync.Pool
}

// NewBufferPool creates a new BufferPool with given buffer size, and internal cache size.
func NewBufferPool(bufferSize, poolSize uint32) *BufferPool {
	pool := &BufferPool{
		chain: make(chan []byte, poolSize),
		allocator: &sync.Pool{
			New: func() interface{} { return make([]byte, bufferSize) },
		},
	}
	for i := uint32(0); i < poolSize; i++ {
		pool.chain <- make([]byte, bufferSize)
	}
	return pool
}

// Allocate implements Pool.Allocate().
func (p *BufferPool) Allocate() *Buffer {
	var b []byte
	select {
	case b = <-p.chain:
	default:
		b = p.allocator.Get().([]byte)
	}
	return &Buffer{
		v:    b,
		pool: p,
	}
}

// Free implements Pool.Free().
func (p *BufferPool) Free(buffer *Buffer) {
	rawBuffer := buffer.v
	if rawBuffer == nil {
		return
	}
	select {
	case p.chain <- rawBuffer:
	default:
		p.allocator.Put(rawBuffer)
	}
}

const (
	// Size of a regular buffer.
	Size = 8 * 1024
	// SizeSmall is the size of a small buffer.
	SizeSmall = 2 * 1024

	poolSizeEnvKey = "v2ray.buffer.size"
)

var (
	mediumPool Pool
	smallPool  = NewSyncPool(SizeSmall)
)

func init() {
	var size uint32 = 20
	sizeStr := os.Getenv(poolSizeEnvKey)
	if len(sizeStr) > 0 {
		customSize, err := strconv.ParseUint(sizeStr, 10, 32)
		if err == nil {
			size = uint32(customSize)
		}
	}
	if size > 0 {
		totalByteSize := size * 1024 * 1024
		mediumPool = NewBufferPool(Size, totalByteSize/Size)
	} else {
		mediumPool = NewSyncPool(Size)
	}
}
<commit_msg>reduce default pool size for non x86 platform to 5MB.<commit_after>package buf

import (
	"os"
	"runtime"
	"strconv"
	"sync"
)

// Pool provides functionality to generate and recycle buffers on demand.
type Pool interface {
	// Allocate either returns a unused buffer from the pool, or generates a new one from system.
	Allocate() *Buffer
	// Free recycles the given buffer.
	Free(*Buffer)
}

// SyncPool is a buffer pool based on sync.Pool
type SyncPool struct {
	allocator *sync.Pool
}

// NewSyncPool creates a SyncPool with given buffer size.
func NewSyncPool(bufferSize uint32) *SyncPool {
	pool := &SyncPool{
		allocator: &sync.Pool{
			New: func() interface{} { return make([]byte, bufferSize) },
		},
	}
	return pool
}

// Allocate implements Pool.Allocate().
func (p *SyncPool) Allocate() *Buffer {
	return &Buffer{
		v:    p.allocator.Get().([]byte),
		pool: p,
	}
}

// Free implements Pool.Free().
func (p *SyncPool) Free(buffer *Buffer) {
	rawBuffer := buffer.v
	if rawBuffer == nil {
		return
	}
	p.allocator.Put(rawBuffer)
}

// BufferPool is a Pool that utilizes an internal cache.
type BufferPool struct {
	chain     chan []byte
	allocator *sync.Pool
}

// NewBufferPool creates a new BufferPool with given buffer size, and internal cache size.
func NewBufferPool(bufferSize, poolSize uint32) *BufferPool {
	pool := &BufferPool{
		chain: make(chan []byte, poolSize),
		allocator: &sync.Pool{
			New: func() interface{} { return make([]byte, bufferSize) },
		},
	}
	for i := uint32(0); i < poolSize; i++ {
		pool.chain <- make([]byte, bufferSize)
	}
	return pool
}

// Allocate implements Pool.Allocate().
func (p *BufferPool) Allocate() *Buffer {
	var b []byte
	select {
	case b = <-p.chain:
	default:
		b = p.allocator.Get().([]byte)
	}
	return &Buffer{
		v:    b,
		pool: p,
	}
}

// Free implements Pool.Free().
func (p *BufferPool) Free(buffer *Buffer) {
	rawBuffer := buffer.v
	if rawBuffer == nil {
		return
	}
	select {
	case p.chain <- rawBuffer:
	default:
		p.allocator.Put(rawBuffer)
	}
}

const (
	// Size of a regular buffer.
	Size = 8 * 1024
	// SizeSmall is the size of a small buffer.
	SizeSmall = 2 * 1024

	poolSizeEnvKey = "v2ray.buffer.size"
)

var (
	mediumPool Pool
	smallPool  = NewSyncPool(SizeSmall)
)

func getDefaultPoolSize() uint32 {
	switch runtime.GOARCH {
	case "amd64", "386":
		return 20
	default:
		return 5
	}
}

func init() {
	var size uint32 = getDefaultPoolSize()
	sizeStr := os.Getenv(poolSizeEnvKey)
	if len(sizeStr) > 0 {
		customSize, err := strconv.ParseUint(sizeStr, 10, 32)
		if err == nil {
			size = uint32(customSize)
		}
	}
	if size > 0 {
		totalByteSize := size * 1024 * 1024
		mediumPool = NewBufferPool(Size, totalByteSize/Size)
	} else {
		mediumPool = NewSyncPool(Size)
	}
}
<|endoftext|>


<commit_before>package sync

import (
	"strings"
)

// Database is a local representation of a Firebase database.
type Database struct {
	root *Node
}

// NewDB creates a new instance of a Database.
func NewDB() *Database {
	return &Database{
		root: &Node{
			Children: map[string]*Node{},
		},
	}
}

// Add puts a Node into the database.
func (d *Database) Add(path string, n *Node) {
	if path == "" {
		d.root = n
		return
	}

	rabbitHole := strings.Split(path, "/")
	current := d.root
	for i := 0; i < len(rabbitHole)-1; i++ {
		step := rabbitHole[i]
		next, ok := current.Children[step]
		if !ok {
			next = &Node{
				Parent:   current,
				Key:      step,
				Children: map[string]*Node{},
			}
			current.Children[step] = next
		}
		next.Value = nil // no long has a value since it now has a child
		current, next = next, nil
	}

	lastPath := rabbitHole[len(rabbitHole)-1]
	current.Children[lastPath] = n
	n.Parent = current
}

// Update merges the current node with the given node.
func (d *Database) Update(path string, n *Node) {
	current := d.root
	rabbitHole := strings.Split(path, "/")

	for i := 0; i < len(rabbitHole); i++ {
		step := rabbitHole[i]
		if step == "" {
			// prevent against empty strings due to strings.Split
			continue
		}
		next, ok := current.Children[step]
		if !ok {
			next = &Node{
				Parent:   current,
				Key:      step,
				Children: map[string]*Node{},
			}
			current.Children[step] = next
		}
		next.Value = nil // no long has a value since it now has a child
		current, next = next, nil
	}

	current.merge(n)

}

// Del removes the node at the given path.
func (d *Database) Del(path string) {
	if path == "" {
		d.root = &Node{
			Children: map[string]*Node{},
		}
		return
	}

	rabbitHole := strings.Split(path, "/")
	current := d.root

	// traverse to target node's parent
	var delIdx int
	for ; delIdx < len(rabbitHole)-1; delIdx++ {
		next, ok := current.Children[rabbitHole[delIdx]]
		if !ok {
			// item does not exist, no need to do anything
			return
		}

		current = next
	}

	endNode := current
	leafPath := rabbitHole[len(rabbitHole)-1]
	delete(endNode.Children, leafPath)

	for tmp := endNode.prune(); tmp != nil; tmp = tmp.prune() {
		delIdx--
		endNode = tmp
	}

	if endNode != nil {
		delete(endNode.Children, rabbitHole[delIdx])
	}
}

// Get fetches a node at a given path.
func (d *Database) Get(path string) *Node {
	current := d.root
	if path == "" {
		return current
	}

	rabbitHole := strings.Split(path, "/")
	for i := 0; i < len(rabbitHole); i++ {
		var ok bool
		current, ok = current.Children[rabbitHole[i]]
		if !ok {
			return nil
		}
	}
	return current
}
<commit_msg>making the database object thread safe<commit_after>package sync

import (
	"strings"
	"sync"
)

// Database is a local representation of a Firebase database.
type Database struct {
	root *Node

	mtx sync.RWMutex
}

// NewDB creates a new instance of a Database.
func NewDB() *Database {
	return &Database{
		root: &Node{
			Children: map[string]*Node{},
		},
	}
}

// Add puts a Node into the database.
func (d *Database) Add(path string, n *Node) {
	d.mtx.Lock()
	defer d.mtx.Unlock()

	if path == "" {
		d.root = n
		return
	}

	rabbitHole := strings.Split(path, "/")
	current := d.root
	for i := 0; i < len(rabbitHole)-1; i++ {
		step := rabbitHole[i]
		next, ok := current.Children[step]
		if !ok {
			next = &Node{
				Parent:   current,
				Key:      step,
				Children: map[string]*Node{},
			}
			current.Children[step] = next
		}
		next.Value = nil // no long has a value since it now has a child
		current, next = next, nil
	}

	lastPath := rabbitHole[len(rabbitHole)-1]
	current.Children[lastPath] = n
	n.Parent = current
}

// Update merges the current node with the given node.
func (d *Database) Update(path string, n *Node) {
	d.mtx.Lock()
	defer d.mtx.Unlock()

	current := d.root
	rabbitHole := strings.Split(path, "/")

	for i := 0; i < len(rabbitHole); i++ {
		step := rabbitHole[i]
		if step == "" {
			// prevent against empty strings due to strings.Split
			continue
		}
		next, ok := current.Children[step]
		if !ok {
			next = &Node{
				Parent:   current,
				Key:      step,
				Children: map[string]*Node{},
			}
			current.Children[step] = next
		}
		next.Value = nil // no long has a value since it now has a child
		current, next = next, nil
	}

	current.merge(n)

}

// Del removes the node at the given path.
func (d *Database) Del(path string) {
	d.mtx.Lock()
	defer d.mtx.Unlock()

	if path == "" {
		d.root = &Node{
			Children: map[string]*Node{},
		}
		return
	}

	rabbitHole := strings.Split(path, "/")
	current := d.root

	// traverse to target node's parent
	var delIdx int
	for ; delIdx < len(rabbitHole)-1; delIdx++ {
		next, ok := current.Children[rabbitHole[delIdx]]
		if !ok {
			// item does not exist, no need to do anything
			return
		}

		current = next
	}

	endNode := current
	leafPath := rabbitHole[len(rabbitHole)-1]
	delete(endNode.Children, leafPath)

	for tmp := endNode.prune(); tmp != nil; tmp = tmp.prune() {
		delIdx--
		endNode = tmp
	}

	if endNode != nil {
		delete(endNode.Children, rabbitHole[delIdx])
	}
}

// Get fetches a node at a given path.
func (d *Database) Get(path string) *Node {
	d.mtx.RLock()
	defer d.mtx.RUnlock()

	current := d.root
	if path == "" {
		return current
	}

	rabbitHole := strings.Split(path, "/")
	for i := 0; i < len(rabbitHole); i++ {
		var ok bool
		current, ok = current.Children[rabbitHole[i]]
		if !ok {
			return nil
		}
	}
	return current
}
<|endoftext|>


<commit_before>package taggolib

import (
	"bytes"
	"io/ioutil"
	"log"
	"reflect"
	"testing"
)

var (
	// Read in test files
	flacFile = func() []byte {
		file, err := ioutil.ReadFile("./test/tone16bit.flac")
		if err != nil {
			log.Fatalf("Could not open test FLAC: %v", err)
		}

		return file
	}()
	mp3File = func() []byte {
		file, err := ioutil.ReadFile("./test/tone16bit.mp3")
		if err != nil {
			log.Fatalf("Could not open test MP3: %v", err)
		}

		return file
	}()
)

// TestNew verifies that New creates the proper parser for an example input stream
func TestNew(t *testing.T) {
	// Table of tests
	var tests = []struct {
		stream     []byte
		parser     Parser
		err        error
		encoder    string
		tags       []string
		properties []int
	}{
		// Check for FLAC file, with hardcoded expected tags and properties
		{flacFile, &FLACParser{}, nil, "reference libFLAC 1.1.4 20070213", []string{"Artist", "Album", "Title"}, []int{5, 202, 16, 44100}},

		// Check for MP3 file, with hardcoded expected tags and properties
		{mp3File, &MP3Parser{}, nil, "MP3FS", []string{"Artist", "Album", "Title"}, []int{5, 320, 16, 44100}},

		// Check for an unknown format
		{[]byte("nonsense"), nil, ErrUnknownFormat, "", nil, nil},
	}

	// Iterate all tests
	for _, test := range tests {
		// Generate a io.ReadSeeker
		reader := bytes.NewReader(test.stream)

		// Attempt to create a parser for the reader
		parser, err := New(reader)
		if err != nil {
			// If an error occurred, check if it was expected
			if err != test.err {
				t.Fatalf("unexpected error: %v", err)
			}
		}

		// Verify that the proper parser type was created
		if reflect.TypeOf(parser) != reflect.TypeOf(test.parser) {
			t.Fatalf("unexpected parser type: %v", reflect.TypeOf(parser))
		}

		// Discard nil parser
		if parser == nil {
			continue
		}

		// Check for valid encoder
		if parser.Encoder() != test.encoder {
			t.Fatalf("mismatched Encoder: %v != %v", parser.Encoder(), test.encoder)
		}

		// Check for valid tags
		if test.tags != nil {
			// Artist
			if parser.Artist() != test.tags[0] {
				t.Fatalf("mismatched tag Artist: %v != %v", parser.Artist(), test.tags[0])
			}

			// Album
			if parser.Album() != test.tags[1] {
				t.Fatalf("mismatched tag Album: %v != %v", parser.Album(), test.tags[1])
			}

			// Title
			if parser.Title() != test.tags[2] {
				t.Fatalf("mismatched tag Title: %v != %v", parser.Title(), test.tags[2])
			}
		}

		// Check for valid properties
		if test.properties != nil {
			// Duration
			if int(parser.Duration().Seconds()) != test.properties[0] {
				t.Fatalf("mismatched property Duration: %v != %v", parser.Duration().Seconds(), test.properties[0])
			}

			// Bitrate
			if parser.Bitrate() != test.properties[1] {
				t.Fatalf("mismatched property Bitrate: %v != %v", parser.Bitrate(), test.properties[1])
			}

			// BitDepth
			if parser.BitDepth() != test.properties[2] {
				t.Fatalf("mismatched property BitDepth: %v != %v", parser.BitDepth(), test.properties[2])
			}

			// SampleRate
			if parser.SampleRate() != test.properties[3] {
				t.Fatalf("mismatched property SampleRate: %v != %v", parser.SampleRate(), test.properties[3])
			}
		}
	}
}

// BenchmarkNew checks the performance of the New() function
func BenchmarkNew(b *testing.B) {
	for i := 0; i < b.N; i++ {
		New(bytes.NewReader(flacFile))
	}
}
<commit_msg>Add benchmark tests for FLAC and MP3<commit_after>package taggolib

import (
	"bytes"
	"io/ioutil"
	"log"
	"reflect"
	"testing"
)

var (
	// Read in test files
	flacFile = func() []byte {
		file, err := ioutil.ReadFile("./test/tone16bit.flac")
		if err != nil {
			log.Fatalf("Could not open test FLAC: %v", err)
		}

		return file
	}()
	mp3File = func() []byte {
		file, err := ioutil.ReadFile("./test/tone16bit.mp3")
		if err != nil {
			log.Fatalf("Could not open test MP3: %v", err)
		}

		return file
	}()
)

// TestNew verifies that New creates the proper parser for an example input stream
func TestNew(t *testing.T) {
	// Table of tests
	var tests = []struct {
		stream     []byte
		parser     Parser
		err        error
		encoder    string
		tags       []string
		properties []int
	}{
		// Check for FLAC file, with hardcoded expected tags and properties
		{flacFile, &FLACParser{}, nil, "reference libFLAC 1.1.4 20070213", []string{"Artist", "Album", "Title"}, []int{5, 202, 16, 44100}},

		// Check for MP3 file, with hardcoded expected tags and properties
		{mp3File, &MP3Parser{}, nil, "MP3FS", []string{"Artist", "Album", "Title"}, []int{5, 320, 16, 44100}},

		// Check for an unknown format
		{[]byte("nonsense"), nil, ErrUnknownFormat, "", nil, nil},
	}

	// Iterate all tests
	for _, test := range tests {
		// Generate a io.ReadSeeker
		reader := bytes.NewReader(test.stream)

		// Attempt to create a parser for the reader
		parser, err := New(reader)
		if err != nil {
			// If an error occurred, check if it was expected
			if err != test.err {
				t.Fatalf("unexpected error: %v", err)
			}
		}

		// Verify that the proper parser type was created
		if reflect.TypeOf(parser) != reflect.TypeOf(test.parser) {
			t.Fatalf("unexpected parser type: %v", reflect.TypeOf(parser))
		}

		// Discard nil parser
		if parser == nil {
			continue
		}

		// Check for valid encoder
		if parser.Encoder() != test.encoder {
			t.Fatalf("mismatched Encoder: %v != %v", parser.Encoder(), test.encoder)
		}

		// Check for valid tags
		if test.tags != nil {
			// Artist
			if parser.Artist() != test.tags[0] {
				t.Fatalf("mismatched tag Artist: %v != %v", parser.Artist(), test.tags[0])
			}

			// Album
			if parser.Album() != test.tags[1] {
				t.Fatalf("mismatched tag Album: %v != %v", parser.Album(), test.tags[1])
			}

			// Title
			if parser.Title() != test.tags[2] {
				t.Fatalf("mismatched tag Title: %v != %v", parser.Title(), test.tags[2])
			}
		}

		// Check for valid properties
		if test.properties != nil {
			// Duration
			if int(parser.Duration().Seconds()) != test.properties[0] {
				t.Fatalf("mismatched property Duration: %v != %v", parser.Duration().Seconds(), test.properties[0])
			}

			// Bitrate
			if parser.Bitrate() != test.properties[1] {
				t.Fatalf("mismatched property Bitrate: %v != %v", parser.Bitrate(), test.properties[1])
			}

			// BitDepth
			if parser.BitDepth() != test.properties[2] {
				t.Fatalf("mismatched property BitDepth: %v != %v", parser.BitDepth(), test.properties[2])
			}

			// SampleRate
			if parser.SampleRate() != test.properties[3] {
				t.Fatalf("mismatched property SampleRate: %v != %v", parser.SampleRate(), test.properties[3])
			}
		}
	}
}

// BenchmarkNewFLAC checks the performance of the New() function with a FLAC file
func BenchmarkNewFLAC(b *testing.B) {
	for i := 0; i < b.N; i++ {
		New(bytes.NewReader(flacFile))
	}
}

// BenchmarkNewMP3 checks the performance of the New() function with a MP3 file
func BenchmarkNewMP3(b *testing.B) {
	for i := 0; i < b.N; i++ {
		New(bytes.NewReader(mp3File))
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"reflect"
	"testing"
)

func TestParseNoRule(t *testing.T) {
	g := &grammar{}
	p := newParser("", []byte(""))
	_, err := p.parse(g)
	if err == nil {
		t.Fatal("want error, got nil")
	}
	el, ok := err.(errList)
	if !ok {
		t.Fatalf("want error type %T, got %T", errList{}, err)
	}
	if len(el) != 1 {
		t.Fatalf("want 1 error, got %d", len(el))
	}
	pe, ok := el[0].(*parserError)
	if !ok {
		t.Fatalf("want single error type %T, got %T", &parserError{}, el[0])
	}
	if pe.Inner != errNoRule {
		t.Fatalf("want error %v, got %v", errNoRule, el[0])
	}
}

func TestParseAnyMatcher(t *testing.T) {
	cases := []struct {
		in    string
		out   []byte
		match bool
	}{
		{"", nil, false},
		{"a", []byte("a"), true},
		{"\u2190", []byte("\u2190"), true},
		{"ab", []byte("a"), true},
		{"\u2190\U00001100", []byte("\u2190"), true},
		{"\x0d", []byte("\x0d"), true},
		{"\xfa", nil, false},
		{"\nab", []byte("\n"), true},
	}

	for _, tc := range cases {
		p := newParser("", []byte(tc.in))

		// advance to the first rune
		p.read()

		var want interface{}
		if tc.out != nil {
			want = tc.out
		}
		got, ok := p.parseAnyMatcher(&anyMatcher{})
		if !reflect.DeepEqual(got, want) {
			t.Errorf("%q: want %v, got %v", tc.in, tc.out, got)
		}
		if ok != tc.match {
			t.Errorf("%q: want match? %t, got %t", tc.in, tc.match, ok)
		}
		if p.pt.offset != len(tc.out) {
			t.Errorf("%q: want offset %d, got %d", tc.in, len(tc.out), p.pt.offset)
		}
	}
}

func TestParseLitMatcher(t *testing.T) {
	cases := []struct {
		in    string
		lit   string
		ic    bool
		out   []byte
		match bool
	}{
		{"", "", false, []byte{}, true}, // empty literal always matches
		{"", "", true, []byte{}, true},  // empty literal always matches
		{"a", "", false, []byte{}, true},
		{"a", "", true, []byte{}, true},
		{"a", "a", false, []byte("a"), true},
		{"a", "a", true, []byte("a"), true},
		{"a", "A", false, nil, false},
		{"a", "a", true, []byte("a"), true}, // ignored case literal is always generated lowercase
		{"A", "a", true, []byte("A"), true},
		{"b", "a", false, nil, false},
		{"b", "a", true, nil, false},
		{"abc", "ab", false, []byte("ab"), true},
		{"abc", "ab", true, []byte("ab"), true},
		{"ab", "abc", false, nil, false},
		{"ab", "abc", true, nil, false},
		{"\u2190a", "\u2190", false, []byte("\u2190"), true},
		{"\u2190a", "\u2190", true, []byte("\u2190"), true},
	}

	for _, tc := range cases {
		p := newParser("", []byte(tc.in))

		// advance to the first rune
		p.read()

		var want interface{}
		if tc.out != nil {
			want = tc.out
		}
		lbl := fmt.Sprintf("%q (%t): %q", tc.lit, tc.ic, tc.in)

		got, ok := p.parseLitMatcher(&litMatcher{val: tc.lit, ignoreCase: tc.ic})
		if !reflect.DeepEqual(got, want) {
			t.Errorf("%s: want %v, got %v", lbl, tc.out, got)
		}
		if ok != tc.match {
			t.Errorf("%s: want match? %t, got %t", lbl, tc.match, ok)
		}
		if p.pt.offset != len(tc.out) {
			t.Errorf("%s: want offset %d, got %d", lbl, len(tc.out), p.pt.offset)
		}
	}
}
<commit_msg>test: CharClassMatcher<commit_after>package main

import (
	"fmt"
	"reflect"
	"testing"
	"unicode"
)

func TestParseNoRule(t *testing.T) {
	g := &grammar{}
	p := newParser("", []byte(""))
	_, err := p.parse(g)
	if err == nil {
		t.Fatal("want error, got nil")
	}
	el, ok := err.(errList)
	if !ok {
		t.Fatalf("want error type %T, got %T", errList{}, err)
	}
	if len(el) != 1 {
		t.Fatalf("want 1 error, got %d", len(el))
	}
	pe, ok := el[0].(*parserError)
	if !ok {
		t.Fatalf("want single error type %T, got %T", &parserError{}, el[0])
	}
	if pe.Inner != errNoRule {
		t.Fatalf("want error %v, got %v", errNoRule, el[0])
	}
}

func TestParseAnyMatcher(t *testing.T) {
	cases := []struct {
		in    string
		out   []byte
		match bool
	}{
		{"", nil, false},
		{"a", []byte("a"), true},
		{"\u2190", []byte("\u2190"), true},
		{"ab", []byte("a"), true},
		{"\u2190\U00001100", []byte("\u2190"), true},
		{"\x0d", []byte("\x0d"), true},
		{"\xfa", nil, false},
		{"\nab", []byte("\n"), true},
	}

	for _, tc := range cases {
		p := newParser("", []byte(tc.in))

		// advance to the first rune
		p.read()

		var want interface{}
		if tc.out != nil {
			want = tc.out
		}
		got, ok := p.parseAnyMatcher(&anyMatcher{})
		if !reflect.DeepEqual(got, want) {
			t.Errorf("%q: want %v, got %v", tc.in, tc.out, got)
		}
		if ok != tc.match {
			t.Errorf("%q: want match? %t, got %t", tc.in, tc.match, ok)
		}
		if p.pt.offset != len(tc.out) {
			t.Errorf("%q: want offset %d, got %d", tc.in, len(tc.out), p.pt.offset)
		}
	}
}

func TestParseLitMatcher(t *testing.T) {
	cases := []struct {
		in    string
		lit   string
		ic    bool
		out   []byte
		match bool
	}{
		{"", "", false, []byte{}, true}, // empty literal always matches
		{"", "", true, []byte{}, true},  // empty literal always matches
		{"a", "", false, []byte{}, true},
		{"a", "", true, []byte{}, true},
		{"a", "a", false, []byte("a"), true},
		{"a", "a", true, []byte("a"), true},
		{"a", "A", false, nil, false},
		{"a", "a", true, []byte("a"), true}, // ignored case literal is always generated lowercase
		{"A", "a", true, []byte("A"), true},
		{"b", "a", false, nil, false},
		{"b", "a", true, nil, false},
		{"abc", "ab", false, []byte("ab"), true},
		{"abc", "ab", true, []byte("ab"), true},
		{"ab", "abc", false, nil, false},
		{"ab", "abc", true, nil, false},
		{"\u2190a", "\u2190", false, []byte("\u2190"), true},
		{"\u2190a", "\u2190", true, []byte("\u2190"), true},
		{"\n", "\n", false, []byte("\n"), true},
		{"\n", "\n", true, []byte("\n"), true},
		{"\na", "\n", false, []byte("\n"), true},
		{"\na", "\n", true, []byte("\n"), true},
	}

	for _, tc := range cases {
		p := newParser("", []byte(tc.in))

		// advance to the first rune
		p.read()

		var want interface{}
		if tc.out != nil {
			want = tc.out
		}
		lbl := fmt.Sprintf("%q (%t): %q", tc.lit, tc.ic, tc.in)

		got, ok := p.parseLitMatcher(&litMatcher{val: tc.lit, ignoreCase: tc.ic})
		if !reflect.DeepEqual(got, want) {
			t.Errorf("%s: want %v, got %v", lbl, tc.out, got)
		}
		if ok != tc.match {
			t.Errorf("%s: want match? %t, got %t", lbl, tc.match, ok)
		}
		if p.pt.offset != len(tc.out) {
			t.Errorf("%s: want offset %d, got %d", lbl, len(tc.out), p.pt.offset)
		}
	}
}

func TestParseCharClassMatcher(t *testing.T) {
	cases := []struct {
		in      string
		val     string
		chars   []rune
		ranges  []rune
		classes []string
		ic      bool
		iv      bool
		out     []byte
	}{
		{in: "", val: "[]", out: nil}, // empty char class means no char matches
		{in: "", val: "[^]", iv: true, out: []byte{}},
		{in: "", val: "[]i", ic: true, out: nil},
		{in: "", val: "[^]i", ic: true, iv: true, out: []byte{}},
		{in: "a", val: "[]", out: nil},
		{in: "a", val: "[^]", iv: true, out: []byte("a")},
		{in: "a", val: "[]i", ic: true, out: nil},
		{in: "a", val: "[^]i", ic: true, iv: true, out: []byte("a")},

		{in: "a", val: "[a]", chars: []rune{'a'}, out: []byte("a")},
		{in: "a", val: "[a]i", ic: true, chars: []rune{'a'}, out: []byte("a")},
		{in: "A", val: "[a]i", ic: true, chars: []rune{'a'}, out: []byte("A")},
		{in: "a", val: "[^a]", chars: []rune{'a'}, iv: true, out: nil},
		{in: "A", val: "[^a]i", iv: true, ic: true, chars: []rune{'a'}, out: nil},

		{in: "b", val: "[a]", chars: []rune{'a'}, out: nil},
		{in: "b", val: "[a]i", ic: true, chars: []rune{'a'}, out: nil},
		{in: "B", val: "[a]i", ic: true, chars: []rune{'a'}, out: nil},
		{in: "b", val: "[^a]", chars: []rune{'a'}, iv: true, out: []byte("b")},
		{in: "b", val: "[^a]i", iv: true, ic: true, chars: []rune{'a'}, out: []byte("b")},
		{in: "B", val: "[^a]i", iv: true, ic: true, chars: []rune{'a'}, out: []byte("B")},

		{in: "b", val: "[a-c]", ranges: []rune{'a', 'c'}, out: []byte("b")},
		{in: "B", val: "[a-c]", ranges: []rune{'a', 'c'}, out: nil},
		{in: "b", val: "[a-c]i", ic: true, ranges: []rune{'a', 'c'}, out: []byte("b")},
		{in: "B", val: "[a-c]i", ic: true, ranges: []rune{'a', 'c'}, out: []byte("B")},
		{in: "b", val: "[^a-c]", ranges: []rune{'a', 'c'}, iv: true, out: nil},
		{in: "B", val: "[^a-c]", ranges: []rune{'a', 'c'}, iv: true, out: []byte("B")},
		{in: "b", val: "[^a-c]i", iv: true, ic: true, ranges: []rune{'a', 'c'}, out: nil},
		{in: "B", val: "[^a-c]i", iv: true, ic: true, ranges: []rune{'a', 'c'}, out: nil},
		{in: "z", val: "[^a-c]i", iv: true, ic: true, chars: []rune{'a', 'c'}, out: []byte("z")},

		{in: "b", val: "[c-a]", ranges: []rune{'c', 'a'}, out: nil},
		{in: "B", val: "[c-a]i", ic: true, ranges: []rune{'c', 'a'}, out: nil},
		{in: "B", val: "[^c-a]", iv: true, ranges: []rune{'c', 'a'}, out: []byte("B")},
		{in: "B", val: "[^c-a]i", ic: true, iv: true, ranges: []rune{'c', 'a'}, out: []byte("B")},
	}

	for _, tc := range cases {
		p := newParser("", []byte(tc.in))

		// advance to the first rune
		p.read()

		var want interface{}
		var match bool
		if tc.out != nil {
			want = tc.out
			match = true
		}
		lbl := fmt.Sprintf("%q (%t-%t): %q", tc.val, tc.ic, tc.iv, tc.in)

		classes := make([]*unicode.RangeTable, len(tc.classes))
		for i, c := range tc.classes {
			classes[i] = rangeTable(c)
		}

		got, ok := p.parseCharClassMatcher(&charClassMatcher{
			val:        tc.val,
			chars:      tc.chars,
			ranges:     tc.ranges,
			classes:    classes,
			ignoreCase: tc.ic,
			inverted:   tc.iv,
		})
		if !reflect.DeepEqual(got, want) {
			t.Errorf("%s: want %v, got %v", lbl, tc.out, got)
		}
		if ok != match {
			t.Errorf("%s: want match? %t, got %t", lbl, match, ok)
		}
		if p.pt.offset != len(tc.out) {
			t.Errorf("%s: want offset %d, got %d", lbl, len(tc.out), p.pt.offset)
		}
	}
}
<|endoftext|>


<commit_before>package multi

import (
	"github.com/thehivecorporation/log"
	"github.com/thehivecorporation/log/telemetry"
)

type telemetryImpl struct {
	telemetry.Common
	impls []log.Telemetry
}

func New(ts ...log.Telemetry) log.Telemetry {
	return &telemetryImpl{
		impls: ts,
	}
}

func (t *telemetryImpl) WithTags(s log.Tags) log.Telemetry {
	for _, i := range t.impls {
		i.WithTags(s)
	}

	return t
}

func (t *telemetryImpl) WithTag(k, v string) log.Telemetry {
	for _, i := range t.impls {
		i.WithTag(k, v)
	}

	return t
}

func (t *telemetryImpl) Inc(name string, value float64, extra ...interface{}) log.Logger {
	for _, i := range t.impls {
		i.Inc(name, value, extra)
	}

	return t.Logger
}

func (t *telemetryImpl) Gauge(name string, value float64, extra ...interface{}) log.Logger {
	for _, i := range t.impls {
		i.Gauge(name, value, extra)
	}

	return t.Logger
}

func (t *telemetryImpl) Fix(name string, value float64, extra ...interface{}) log.Logger {
	for _, i := range t.impls {
		i.Fix(name, value, extra)
	}

	return t.Logger
}

func (t *telemetryImpl) Histogram(name string, value float64, extra ...interface{}) log.Logger {
	for _, i := range t.impls {
		i.Histogram(name, value, extra)
	}

	return t.Logger
}

func (t *telemetryImpl) Summary(name string, value float64, extra ...interface{}) log.Logger {
	for _, i := range t.impls {
		i.Summary(name, value, extra)
	}

	return t.Logger
}

func (t telemetryImpl) Clone() log.Telemetry {
	t.impls = make([]log.Telemetry, len(t.impls))

	for i, v := range t.impls {
		t.impls[i] = v.Clone()
	}

	t.Tags = make([]string, 0)

	return &t
}

func (t *telemetryImpl) SetLogger(l log.Logger) {
	t.Logger = l
}
<commit_msg>fix small stuff in telemetry<commit_after>package multi

import (
	"github.com/thehivecorporation/log"
	"github.com/thehivecorporation/log/telemetry"
)

type telemetryImpl struct {
	telemetry.Common
	impls []log.Telemetry
}

func New(ts ...log.Telemetry) log.Telemetry {
	return &telemetryImpl{
		impls: ts,
	}
}

func (t *telemetryImpl) WithTags(s log.Tags) log.Telemetry {
	for _, i := range t.impls {
		i.WithTags(s)
	}

	return t
}

func (t *telemetryImpl) WithTag(k, v string) log.Telemetry {
	for _, i := range t.impls {
		i.WithTag(k, v)
	}

	return t
}

func (t *telemetryImpl) Inc(name string, value float64, extra ...interface{}) log.Logger {
	for _, i := range t.impls {
		i.Inc(name, value, extra)
	}

	return t.Logger
}

func (t *telemetryImpl) Gauge(name string, value float64, extra ...interface{}) log.Logger {
	for _, i := range t.impls {
		i.Gauge(name, value, extra)
	}

	return t.Logger
}

func (t *telemetryImpl) Fix(name string, value float64, extra ...interface{}) log.Logger {
	for _, i := range t.impls {
		i.Fix(name, value, extra)
	}

	return t.Logger
}

func (t *telemetryImpl) Histogram(name string, value float64, extra ...interface{}) log.Logger {
	for _, i := range t.impls {
		i.Histogram(name, value, extra)
	}

	return t.Logger
}

func (t *telemetryImpl) Summary(name string, value float64, extra ...interface{}) log.Logger {
	for _, i := range t.impls {
		i.Summary(name, value, extra)
	}

	return t.Logger
}

func (t telemetryImpl) Clone() log.Telemetry {
	t.impls = make([]log.Telemetry, len(t.impls))

	for i, v := range t.impls {
		t.impls[i] = v.Clone()
	}

	t.Tags = make(map[string]string, 0)

	return &t
}

func (t *telemetryImpl) SetLogger(l log.Logger) {
	t.Logger = l
}
<|endoftext|>


<commit_before>package solidproxy

import (
	"crypto/tls"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"time"
)

const (
	MaxIdleConnections int = 20
	RequestTimeout     int = 5
)

type Proxy struct {
	HttpClient      *http.Client
	HttpAgentClient *http.Client
	Log             *log.Logger
	Agent           *Agent
}

func NewProxy(agent *Agent, skip bool) *Proxy {
	proxy := &Proxy{
		HttpClient: NewClient(skip),
		Agent:      agent,
		Log:        InitLogger(false),
	}

	if agent.Cert != nil {
		proxy.HttpAgentClient = agent.NewAgentClient(skip)
	}

	return proxy
}

func (p *Proxy) Handler(w http.ResponseWriter, req *http.Request) {
	p.Log.Println("New request from:", req.RemoteAddr, "for URI:", req.URL.String())
	// Log the time it takes to finish the request (for debugging)
	defer TimeTrack(time.Now(), req.Method+" operation", p.Log)

	uri := req.FormValue("uri")
	if len(uri) == 0 {
		msg := "HTTP 400 - Bad Request. Please provide a URI to the proxy."
		p.Log.Println(msg, req.URL.String())
		w.WriteHeader(400)
		w.Write([]byte(msg))
		return
	}

	resource, err := url.ParseRequestURI(uri)
	if err != nil {
		p.Log.Println("Error parsing URL:", req.URL, err.Error())
		w.WriteHeader(400)
		w.Write([]byte("HTTP 400 - Bad Request. You must provide a valid URI: " + req.URL.String()))
		return
	}
	// rewrite URL
	req.URL = resource
	req.Host = resource.Host
	req.RequestURI = resource.RequestURI()
	// get user
	user := req.Header.Get("User")

	p.Log.Println("Proxying request for URI:", req.URL, "and user:", user, "using Agent:", p.Agent.WebID)

	// build new response
	// no error should exist at this point, it was caught earlier
	// by url.Parse and the server handler
	plain, _ := http.NewRequest(req.Method, req.URL.String(), req.Body)
	// copy headers
	CopyHeaders(req.Header, plain.Header)
	// overwrite User Agent
	plain.Header.Set("User-Agent", GetServerFullName())

	r, err := p.HttpClient.Do(plain)
	if err != nil {
		p.Log.Println("Request execution error:", err)
		w.WriteHeader(500)
		w.Write([]byte(err.Error()))
		return
	}

	// Retry with server credentials if authentication is required
	if (r.StatusCode == 401 || r.StatusCode == 403) && (len(user) > 0 && p.HttpAgentClient != nil) {
		// Close the previous response to reuse the connection
		r.Body.Close()

		// build new response
		authenticated, err := http.NewRequest(req.Method, req.URL.String(), req.Body)
		// copy headers
		CopyHeaders(req.Header, authenticated.Header)
		// overwrite our specific ones
		authenticated.Header.Set("User-Agent", GetServerFullName())
		authenticated.Header.Set("On-Behalf-Of", user)

		var solutionMsg string
		// Retry the request
		if len(cookies[user]) > 0 { // Use existing cookie
			solutionMsg = "Retrying with cookies"
			authenticated.AddCookie(cookies[user][req.Host][0])
			// Create the client
			r, err = p.HttpClient.Do(authenticated)
		} else { // Using WebIDTLS client
			solutionMsg = "Retrying with WebID-TLS"
			r, err = p.HttpAgentClient.Do(authenticated)
		}

		if err != nil {
			p.Log.Println("Request execution error on auth retry:", err)
			w.WriteHeader(500)
			w.Write([]byte(err.Error()))
			return
		}
		// Close the response to reuse the connection
		defer r.Body.Close()

		// Store cookies per user and request host
		if len(r.Cookies()) > 0 {
			cookiesL.Lock()
			// TODO: should store cookies based on domain value AND path from cookie
			cookies[user] = map[string][]*http.Cookie{}
			cookies[user][req.Host] = r.Cookies()
			p.Log.Printf("Cookies: %+v\n", cookies)
			cookiesL.Unlock()
		}
		p.Log.Println("Resource "+authenticated.URL.String(),
			"requires authentication (HTTP 401).", solutionMsg,
			"resulted in HTTP", r.StatusCode)

		p.Log.Println("Got authenticated response code:", r.StatusCode)
		w.Header().Set("Authenticated-Request", "1")
	}

	// Write data back
	// CORS
	w.Header().Set("Access-Control-Allow-Credentials", "true")
	w.Header().Set("Access-Control-Expose-Headers", "User, Triples, Location, Origin, Link, Vary, Last-Modified, Content-Length")
	w.Header().Set("Access-Control-Max-Age", "60")
	origin := req.Header.Get("Origin")
	if len(origin) > 0 {
		w.Header().Set("Access-Control-Allow-Origin", origin)
	}

	// copy headers
	CopyHeaders(r.Header, w.Header())

	w.WriteHeader(r.StatusCode)
	// r.Body will be empty at worst, so it should never trigger an error
	body, _ := ioutil.ReadAll(r.Body)
	w.Write(body)

	p.Log.Println("Received public data with status HTTP", r.StatusCode)
	return
}

func CopyHeaders(from http.Header, to http.Header) {
	for key, values := range from {
		for _, value := range values {
			if key != "User" || key != "Cookie" {
				to.Set(key, value)
			}
		}
	}
}

func NewClient(skip bool) *http.Client {
	return &http.Client{
		Transport: &http.Transport{
			MaxIdleConnsPerHost: MaxIdleConnections,
			TLSClientConfig: &tls.Config{
				InsecureSkipVerify: skip,
			},
		},
		Timeout: time.Duration(RequestTimeout) * time.Second,
	}
}

func (agent *Agent) NewAgentClient(skip bool) *http.Client {
	return &http.Client{
		Transport: &http.Transport{
			MaxIdleConnsPerHost: MaxIdleConnections,
			TLSClientConfig: &tls.Config{
				Certificates:       []tls.Certificate{*agent.Cert},
				InsecureSkipVerify: skip,
			},
		},
		Timeout: time.Duration(RequestTimeout) * time.Second,
	}
}

func TimeTrack(start time.Time, name string, logger *log.Logger) {
	elapsed := time.Since(start)
	logger.Printf("%s finished in %s", name, elapsed)
}
<commit_msg>send client cert in case the cookie expired<commit_after>package solidproxy

import (
	"crypto/tls"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"time"
)

const (
	MaxIdleConnections int = 20
	RequestTimeout     int = 5
)

type Proxy struct {
	HttpClient      *http.Client
	HttpAgentClient *http.Client
	Log             *log.Logger
	Agent           *Agent
}

func NewProxy(agent *Agent, skip bool) *Proxy {
	proxy := &Proxy{
		HttpClient: NewClient(skip),
		Agent:      agent,
		Log:        InitLogger(false),
	}

	if agent.Cert != nil {
		proxy.HttpAgentClient = agent.NewAgentClient(skip)
	}

	return proxy
}

func (p *Proxy) Handler(w http.ResponseWriter, req *http.Request) {
	p.Log.Println("New request from:", req.RemoteAddr, "for URI:", req.URL.String())
	// Log the time it takes to finish the request (for debugging)
	defer TimeTrack(time.Now(), req.Method+" operation", p.Log)

	uri := req.FormValue("uri")
	if len(uri) == 0 {
		msg := "HTTP 400 - Bad Request. Please provide a URI to the proxy."
		p.Log.Println(msg, req.URL.String())
		w.WriteHeader(400)
		w.Write([]byte(msg))
		return
	}

	resource, err := url.ParseRequestURI(uri)
	if err != nil {
		p.Log.Println("Error parsing URL:", req.URL, err.Error())
		w.WriteHeader(400)
		w.Write([]byte("HTTP 400 - Bad Request. You must provide a valid URI: " + req.URL.String()))
		return
	}
	// rewrite URL
	req.URL = resource
	req.Host = resource.Host
	req.RequestURI = resource.RequestURI()
	// get user
	user := req.Header.Get("User")

	p.Log.Println("Proxying request for URI:", req.URL, "and user:", user, "using Agent:", p.Agent.WebID)

	// build new response
	// no error should exist at this point, it was caught earlier
	// by url.Parse and the server handler
	plain, _ := http.NewRequest(req.Method, req.URL.String(), req.Body)
	// copy headers
	CopyHeaders(req.Header, plain.Header)
	// overwrite User Agent
	plain.Header.Set("User-Agent", GetServerFullName())

	r, err := p.HttpClient.Do(plain)
	if err != nil {
		p.Log.Println("Request execution error:", err)
		w.WriteHeader(500)
		w.Write([]byte(err.Error()))
		return
	}

	// Retry with server credentials if authentication is required
	if (r.StatusCode == 401 || r.StatusCode == 403) && (len(user) > 0 && p.HttpAgentClient != nil) {
		// Close the previous response to reuse the connection
		r.Body.Close()

		// build new response
		authenticated, err := http.NewRequest(req.Method, req.URL.String(), req.Body)
		// copy headers
		CopyHeaders(req.Header, authenticated.Header)
		// overwrite our specific ones
		authenticated.Header.Set("User-Agent", GetServerFullName())
		authenticated.Header.Set("On-Behalf-Of", user)

		solutionMsg := "Retrying with WebID-TLS"
		// Retry the request
		if len(cookies[user]) > 0 { // Use existing cookie
			solutionMsg = "Retrying with cookies"
			authenticated.AddCookie(cookies[user][req.Host][0])
		}
		// Create the client
		r, err = p.HttpClient.Do(authenticated)
		if err != nil {
			p.Log.Println("Request execution error on auth retry:", err)
			w.WriteHeader(500)
			w.Write([]byte(err.Error()))
			return
		}
		// Close the response to reuse the connection
		defer r.Body.Close()

		// Store cookies per user and request host
		if len(r.Cookies()) > 0 {
			cookiesL.Lock()
			// TODO: should store cookies based on domain value AND path from cookie
			cookies[user] = map[string][]*http.Cookie{}
			cookies[user][req.Host] = r.Cookies()
			p.Log.Printf("Cookies: %+v\n", cookies)
			cookiesL.Unlock()
		}
		p.Log.Println("Resource "+authenticated.URL.String(),
			"requires authentication (HTTP 401).", solutionMsg,
			"resulted in HTTP", r.StatusCode)

		p.Log.Println("Got authenticated response code:", r.StatusCode)
		w.Header().Set("Authenticated-Request", "1")
	}

	// Write data back
	// CORS
	w.Header().Set("Access-Control-Allow-Credentials", "true")
	w.Header().Set("Access-Control-Expose-Headers", "User, Triples, Location, Origin, Link, Vary, Last-Modified, Content-Length")
	w.Header().Set("Access-Control-Max-Age", "60")
	origin := req.Header.Get("Origin")
	if len(origin) > 0 {
		w.Header().Set("Access-Control-Allow-Origin", origin)
	}

	// copy headers
	CopyHeaders(r.Header, w.Header())

	w.WriteHeader(r.StatusCode)
	// r.Body will be empty at worst, so it should never trigger an error
	body, _ := ioutil.ReadAll(r.Body)
	w.Write(body)

	p.Log.Println("Received public data with status HTTP", r.StatusCode)
	return
}

func CopyHeaders(from http.Header, to http.Header) {
	for key, values := range from {
		for _, value := range values {
			if key != "User" || key != "Cookie" {
				to.Set(key, value)
			}
		}
	}
}

func NewClient(skip bool) *http.Client {
	return &http.Client{
		Transport: &http.Transport{
			MaxIdleConnsPerHost: MaxIdleConnections,
			TLSClientConfig: &tls.Config{
				InsecureSkipVerify: skip,
			},
		},
		Timeout: time.Duration(RequestTimeout) * time.Second,
	}
}

func (agent *Agent) NewAgentClient(skip bool) *http.Client {
	return &http.Client{
		Transport: &http.Transport{
			MaxIdleConnsPerHost: MaxIdleConnections,
			TLSClientConfig: &tls.Config{
				Certificates:       []tls.Certificate{*agent.Cert},
				InsecureSkipVerify: skip,
			},
		},
		Timeout: time.Duration(RequestTimeout) * time.Second,
	}
}

func TimeTrack(start time.Time, name string, logger *log.Logger) {
	elapsed := time.Since(start)
	logger.Printf("%s finished in %s", name, elapsed)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/http/httputil"
	"strings"
	"sync"
	"time"

	"context"
	"github.com/abbot/go-http-auth"
)

const (
	browserStarted int = iota
	browserFailed
	seleniumError
)

const (
	pingPath  string = "/ping"
	errPath   string = "/err"
	routePath string = "/wd/hub/session"
	proxyPath string = routePath + "/"
	head      int    = len(proxyPath)
	tail      int    = head + 32
	sessPart  int    = 4 // /wd/hub/session/{various length session}
)

var (
	quota           = make(map[string]Browsers)
	routes   Routes = make(Routes)
	num      uint64
	numLock  sync.Mutex
	confLock sync.RWMutex
)

// Routes - an MD5 to host map
type Routes map[string]*Host

type caps map[string]interface{}

func (c *caps) capability(k string) string {
	dc := (*c)["desiredCapabilities"]
	switch dc.(type) {
	case map[string]interface{}:
		v := dc.(map[string]interface{})
		switch v[k].(type) {
		case string:
			return v[k].(string)
		}
	}
	return ""
}

func (c *caps) setCapability(k string, v string) {
	(*c)["desiredCapabilities"].(map[string]interface{})[k] = v
}

func (c *caps) browser() string {
	return c.capability("browserName")
}

func (c *caps) version() string {
	return c.capability("version")
}

func (c *caps) setVersion(version string) {
	c.setCapability("version", version)
}

func (h *Host) session(ctx context.Context, c caps) (map[string]interface{}, int) {
	b, _ := json.Marshal(c)
	req, err := http.NewRequest(http.MethodPost, h.sessionURL(), bytes.NewReader(b))
	if err != nil {
		return nil, seleniumError
	}
	ctx, _ = context.WithTimeout(ctx, timeout)
	req = req.WithContext(ctx)
	req.Header.Set("Content-Type", "application/json")
	resp, err := http.DefaultClient.Do(req)
	if resp != nil {
		defer resp.Body.Close()
	}
	if err != nil {
		return nil, seleniumError
	}
	var reply map[string]interface{}
	err = json.NewDecoder(resp.Body).Decode(&reply)
	if err != nil {
		return nil, seleniumError
	}
	if resp.StatusCode != http.StatusOK {
		return reply, browserFailed
	}
	return reply, browserStarted
}

func reply(w http.ResponseWriter, msg map[string]interface{}) {
	reply, _ := json.Marshal(msg)
	w.Header().Set("Content-Type", "application/json")
	w.Write(reply)
}

func serial() uint64 {
	numLock.Lock()
	defer numLock.Unlock()
	id := num
	num++
	return id
}

func info(r *http.Request) (user, remote string) {
	user = "unknown"
	if u, _, ok := r.BasicAuth(); ok {
		user = u
	}
	remote = r.Header.Get("X-Forwarded-For")
	if remote != "" {
		return
	}
	remote, _, _ = net.SplitHostPort(r.RemoteAddr)
	return
}

func fmtBrowser(browser, version string) string {
	if version != "" {
		return fmt.Sprintf("%s-%s", browser, version)
	}
	return browser
}

func browserErrMsg(js map[string]interface{}) string {
	if js == nil {
		return ""
	}
	val, ok := js["value"].(map[string]interface{})
	if !ok {
		return ""
	}
	msg, ok := val["message"].(string)
	if !ok {
		return ""
	}
	return msg
}

func jsonErrMsg(msg string) string {
	message := make(map[string]string)
	message["message"] = msg
	value := make(map[string]interface{})
	value["value"] = message
	value["status"] = 13
	result, _ := json.Marshal(value)
	return string(result)
}

func route(w http.ResponseWriter, r *http.Request) {
	start := time.Now()
	id := serial()
	user, remote := info(r)
	var c caps
	err := json.NewDecoder(r.Body).Decode(&c)
	if err != nil {
		http.Error(w, fmt.Sprintf("bad json format: %s", err.Error()), http.StatusBadRequest)
		log.Printf("[%d] [BAD_JSON] [%s] [%s] [%v]\n", id, user, remote, err)
		return
	}
	browser, version := c.browser(), c.version()
	if browser == "" {
		http.Error(w, "browser not set", http.StatusBadRequest)
		log.Printf("[%d] [BROWSER_NOT_SET] [%s] [%s]\n", id, user, remote)
		return
	}
	count := 0
loop:
	for {
		confLock.RLock()
		browsers := quota[user]
		hosts, version := browsers.find(browser, version)
		confLock.RUnlock()
		if len(hosts) == 0 {
			http.Error(w, fmt.Sprintf("unsupported browser: %s", fmtBrowser(browser, version)), http.StatusNotFound)
			log.Printf("[%d] [UNSUPPORTED_BROWSER] [%s] [%s] [%s]\n", id, user, remote, fmtBrowser(browser, version))
			return
		}
		for h, i := hosts.choose(); ; h, i = hosts.choose() {
			count++
			if h == nil {
				break loop
			}
			log.Printf("[%d] [SESSION_ATTEMPTED] [%s] [%s] [%s] [%s] [%d]\n", id, user, remote, fmtBrowser(browser, version), h.net(), count)
			excludes := make([]string, 0)
			c.setVersion(version)
			resp, status := h.session(r.Context(), c)
			select {
			case <-r.Context().Done():
				log.Printf("[%d] [%.2fs] [CLIENT_DISCONNECTED] [%s] [%s] [%s] [%s] [%d]\n", id, float64(time.Now().Sub(start).Seconds()), user, remote, fmtBrowser(browser, version), h.net(), count)
				return
			default:
			}
			switch status {
			case browserStarted:
				sess := resp["sessionId"].(string)
				resp["sessionId"] = h.sum() + sess
				reply(w, resp)
				log.Printf("[%d] [%.2fs] [SESSION_CREATED] [%s] [%s] [%s] [%s] [%s] [%d]\n", id, float64(time.Now().Sub(start).Seconds()), user, remote, fmtBrowser(browser, version), h.net(), sess, count)
				return
			case browserFailed:
				hosts = append(hosts[:i], hosts[i+1:]...)
			case seleniumError:
				excludes = append(excludes, h.region)
				hosts, version = browsers.find(browser, version, excludes...)
			}
			log.Printf("[%d] [SESSION_FAILED] [%s] [%s] [%s] [%s] %s\n", id, user, remote, fmtBrowser(browser, version), h.net(), browserErrMsg(resp))
			if len(hosts) == 0 {
				break loop
			}
		}
	}
	http.Error(w, jsonErrMsg(fmt.Sprintf("cannot create session %s on any hosts after %d attempt(s)", fmtBrowser(browser, version), count)), http.StatusInternalServerError)
	log.Printf("[%d] [SESSION_NOT_CREATED] [%s] [%s] [%s]\n", id, user, remote, fmtBrowser(browser, version))
}

func proxy(r *http.Request) {
	_, remote := info(r)
	r.URL.Scheme = "http"
	if len(r.URL.Path) > tail {
		sum := r.URL.Path[head:tail]
		proxyPath := r.URL.Path[:head] + r.URL.Path[tail:]
		if h, ok := routes[sum]; ok {
			if body, err := ioutil.ReadAll(r.Body); err == nil {
				r.Body.Close()
				var msg map[string]interface{}
				if err := json.Unmarshal(body, &msg); err == nil {
					delete(msg, "sessionId")
					body, _ = json.Marshal(msg)
					r.ContentLength = int64(len(body))
				}
				r.Body = ioutil.NopCloser(bytes.NewReader(body))
			}
			r.URL.Host = h.net()
			r.URL.Path = proxyPath
			if r.Method == "DELETE" {
				sess := strings.Split(proxyPath, "/")[sessPart]
				log.Printf("[SESSION_DELETED] [%s] [%s] [%s]\n", remote, h.net(), sess)
			}
			return
		}
		log.Printf("[ROUTE_NOT_FOUND] [%s] [%s]\n", remote, proxyPath)
	} else {
		log.Printf("[INVALID_URL] [%s] [%s]\n", remote, r.URL.Path)
	}
	r.URL.Host = listen
	r.URL.Path = errPath
}

func ping(w http.ResponseWriter, r *http.Request) {
	w.Write([]byte("Ok\n"))
}

func err(w http.ResponseWriter, r *http.Request) {
	http.Error(w, "route not found", http.StatusNotFound)
}

func postOnly(handler http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		if r.Method != http.MethodPost {
			http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
			return
		}
		handler(w, r)
	}
}

func withCloseNotifier(handler http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		ctx, cancel := context.WithCancel(r.Context())
		go func() {
			handler(w, r.WithContext(ctx))
			cancel()
		}()
		select {
		case <-w.(http.CloseNotifier).CloseNotify():
			cancel()
		case <-ctx.Done():
		}
	}
}

func readConfig(fn string, browsers *Browsers) error {
	file, err := ioutil.ReadFile(fn)
	if err != nil {
		return fmt.Errorf("error reading configuration file %s: %v", fn, err)
	}
	if err := xml.Unmarshal(file, browsers); err != nil {
		return fmt.Errorf("error parsing configuration file %s: %v", fn, err)
	}
	return nil
}

func appendRoutes(routes Routes, config *Browsers) Routes {
	for _, b := range config.Browsers {
		for _, v := range b.Versions {
			for _, r := range v.Regions {
				for i, h := range r.Hosts {
					r.Hosts[i].region = r.Name
					routes[h.sum()] = &r.Hosts[i]
				}
			}
		}
	}
	return routes
}

func requireBasicAuth(authenticator *auth.BasicAuth, handler func(http.ResponseWriter, *http.Request)) func(http.ResponseWriter, *http.Request) {
	return authenticator.Wrap(func(w http.ResponseWriter, r *auth.AuthenticatedRequest) {
		handler(w, &r.Request)
	})
}

func mux() http.Handler {
	mux := http.NewServeMux()
	authenticator := auth.NewBasicAuthenticator(
		"Selenium Grid Router",
		auth.HtpasswdFileProvider(users),
	)
	mux.HandleFunc(pingPath, ping)
	mux.HandleFunc(errPath, err)
	mux.HandleFunc(routePath, requireBasicAuth(authenticator, postOnly(withCloseNotifier(route))))
	mux.Handle(proxyPath, &httputil.ReverseProxy{Director: proxy})
	return mux
}
<commit_msg>Call cancel to release any allocated resources.<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/http/httputil"
	"strings"
	"sync"
	"time"

	"context"
	"github.com/abbot/go-http-auth"
)

const (
	browserStarted int = iota
	browserFailed
	seleniumError
)

const (
	pingPath  string = "/ping"
	errPath   string = "/err"
	routePath string = "/wd/hub/session"
	proxyPath string = routePath + "/"
	head      int    = len(proxyPath)
	tail      int    = head + 32
	sessPart  int    = 4 // /wd/hub/session/{various length session}
)

var (
	quota           = make(map[string]Browsers)
	routes   Routes = make(Routes)
	num      uint64
	numLock  sync.Mutex
	confLock sync.RWMutex
)

// Routes - an MD5 to host map
type Routes map[string]*Host

type caps map[string]interface{}

func (c *caps) capability(k string) string {
	dc := (*c)["desiredCapabilities"]
	switch dc.(type) {
	case map[string]interface{}:
		v := dc.(map[string]interface{})
		switch v[k].(type) {
		case string:
			return v[k].(string)
		}
	}
	return ""
}

func (c *caps) setCapability(k string, v string) {
	(*c)["desiredCapabilities"].(map[string]interface{})[k] = v
}

func (c *caps) browser() string {
	return c.capability("browserName")
}

func (c *caps) version() string {
	return c.capability("version")
}

func (c *caps) setVersion(version string) {
	c.setCapability("version", version)
}

func (h *Host) session(ctx context.Context, c caps) (map[string]interface{}, int) {
	b, _ := json.Marshal(c)
	req, err := http.NewRequest(http.MethodPost, h.sessionURL(), bytes.NewReader(b))
	if err != nil {
		return nil, seleniumError
	}
	ctx, cancel := context.WithTimeout(ctx, timeout)
	defer cancel()
	req = req.WithContext(ctx)
	req.Header.Set("Content-Type", "application/json")
	resp, err := http.DefaultClient.Do(req)
	if resp != nil {
		defer resp.Body.Close()
	}
	if err != nil {
		return nil, seleniumError
	}
	var reply map[string]interface{}
	err = json.NewDecoder(resp.Body).Decode(&reply)
	if err != nil {
		return nil, seleniumError
	}
	if resp.StatusCode != http.StatusOK {
		return reply, browserFailed
	}
	return reply, browserStarted
}

func reply(w http.ResponseWriter, msg map[string]interface{}) {
	reply, _ := json.Marshal(msg)
	w.Header().Set("Content-Type", "application/json")
	w.Write(reply)
}

func serial() uint64 {
	numLock.Lock()
	defer numLock.Unlock()
	id := num
	num++
	return id
}

func info(r *http.Request) (user, remote string) {
	user = "unknown"
	if u, _, ok := r.BasicAuth(); ok {
		user = u
	}
	remote = r.Header.Get("X-Forwarded-For")
	if remote != "" {
		return
	}
	remote, _, _ = net.SplitHostPort(r.RemoteAddr)
	return
}

func fmtBrowser(browser, version string) string {
	if version != "" {
		return fmt.Sprintf("%s-%s", browser, version)
	}
	return browser
}

func browserErrMsg(js map[string]interface{}) string {
	if js == nil {
		return ""
	}
	val, ok := js["value"].(map[string]interface{})
	if !ok {
		return ""
	}
	msg, ok := val["message"].(string)
	if !ok {
		return ""
	}
	return msg
}

func jsonErrMsg(msg string) string {
	message := make(map[string]string)
	message["message"] = msg
	value := make(map[string]interface{})
	value["value"] = message
	value["status"] = 13
	result, _ := json.Marshal(value)
	return string(result)
}

func route(w http.ResponseWriter, r *http.Request) {
	start := time.Now()
	id := serial()
	user, remote := info(r)
	var c caps
	err := json.NewDecoder(r.Body).Decode(&c)
	if err != nil {
		http.Error(w, fmt.Sprintf("bad json format: %s", err.Error()), http.StatusBadRequest)
		log.Printf("[%d] [BAD_JSON] [%s] [%s] [%v]\n", id, user, remote, err)
		return
	}
	browser, version := c.browser(), c.version()
	if browser == "" {
		http.Error(w, "browser not set", http.StatusBadRequest)
		log.Printf("[%d] [BROWSER_NOT_SET] [%s] [%s]\n", id, user, remote)
		return
	}
	count := 0
loop:
	for {
		confLock.RLock()
		browsers := quota[user]
		hosts, version := browsers.find(browser, version)
		confLock.RUnlock()
		if len(hosts) == 0 {
			http.Error(w, fmt.Sprintf("unsupported browser: %s", fmtBrowser(browser, version)), http.StatusNotFound)
			log.Printf("[%d] [UNSUPPORTED_BROWSER] [%s] [%s] [%s]\n", id, user, remote, fmtBrowser(browser, version))
			return
		}
		for h, i := hosts.choose(); ; h, i = hosts.choose() {
			count++
			if h == nil {
				break loop
			}
			log.Printf("[%d] [SESSION_ATTEMPTED] [%s] [%s] [%s] [%s] [%d]\n", id, user, remote, fmtBrowser(browser, version), h.net(), count)
			excludes := make([]string, 0)
			c.setVersion(version)
			resp, status := h.session(r.Context(), c)
			select {
			case <-r.Context().Done():
				log.Printf("[%d] [%.2fs] [CLIENT_DISCONNECTED] [%s] [%s] [%s] [%s] [%d]\n", id, float64(time.Now().Sub(start).Seconds()), user, remote, fmtBrowser(browser, version), h.net(), count)
				return
			default:
			}
			switch status {
			case browserStarted:
				sess := resp["sessionId"].(string)
				resp["sessionId"] = h.sum() + sess
				reply(w, resp)
				log.Printf("[%d] [%.2fs] [SESSION_CREATED] [%s] [%s] [%s] [%s] [%s] [%d]\n", id, float64(time.Now().Sub(start).Seconds()), user, remote, fmtBrowser(browser, version), h.net(), sess, count)
				return
			case browserFailed:
				hosts = append(hosts[:i], hosts[i+1:]...)
			case seleniumError:
				excludes = append(excludes, h.region)
				hosts, version = browsers.find(browser, version, excludes...)
			}
			log.Printf("[%d] [SESSION_FAILED] [%s] [%s] [%s] [%s] %s\n", id, user, remote, fmtBrowser(browser, version), h.net(), browserErrMsg(resp))
			if len(hosts) == 0 {
				break loop
			}
		}
	}
	http.Error(w, jsonErrMsg(fmt.Sprintf("cannot create session %s on any hosts after %d attempt(s)", fmtBrowser(browser, version), count)), http.StatusInternalServerError)
	log.Printf("[%d] [SESSION_NOT_CREATED] [%s] [%s] [%s]\n", id, user, remote, fmtBrowser(browser, version))
}

func proxy(r *http.Request) {
	_, remote := info(r)
	r.URL.Scheme = "http"
	if len(r.URL.Path) > tail {
		sum := r.URL.Path[head:tail]
		proxyPath := r.URL.Path[:head] + r.URL.Path[tail:]
		if h, ok := routes[sum]; ok {
			if body, err := ioutil.ReadAll(r.Body); err == nil {
				r.Body.Close()
				var msg map[string]interface{}
				if err := json.Unmarshal(body, &msg); err == nil {
					delete(msg, "sessionId")
					body, _ = json.Marshal(msg)
					r.ContentLength = int64(len(body))
				}
				r.Body = ioutil.NopCloser(bytes.NewReader(body))
			}
			r.URL.Host = h.net()
			r.URL.Path = proxyPath
			if r.Method == "DELETE" {
				sess := strings.Split(proxyPath, "/")[sessPart]
				log.Printf("[SESSION_DELETED] [%s] [%s] [%s]\n", remote, h.net(), sess)
			}
			return
		}
		log.Printf("[ROUTE_NOT_FOUND] [%s] [%s]\n", remote, proxyPath)
	} else {
		log.Printf("[INVALID_URL] [%s] [%s]\n", remote, r.URL.Path)
	}
	r.URL.Host = listen
	r.URL.Path = errPath
}

func ping(w http.ResponseWriter, r *http.Request) {
	w.Write([]byte("Ok\n"))
}

func err(w http.ResponseWriter, r *http.Request) {
	http.Error(w, "route not found", http.StatusNotFound)
}

func postOnly(handler http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		if r.Method != http.MethodPost {
			http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
			return
		}
		handler(w, r)
	}
}

func withCloseNotifier(handler http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		ctx, cancel := context.WithCancel(r.Context())
		go func() {
			handler(w, r.WithContext(ctx))
			cancel()
		}()
		select {
		case <-w.(http.CloseNotifier).CloseNotify():
			cancel()
		case <-ctx.Done():
		}
	}
}

func readConfig(fn string, browsers *Browsers) error {
	file, err := ioutil.ReadFile(fn)
	if err != nil {
		return fmt.Errorf("error reading configuration file %s: %v", fn, err)
	}
	if err := xml.Unmarshal(file, browsers); err != nil {
		return fmt.Errorf("error parsing configuration file %s: %v", fn, err)
	}
	return nil
}

func appendRoutes(routes Routes, config *Browsers) Routes {
	for _, b := range config.Browsers {
		for _, v := range b.Versions {
			for _, r := range v.Regions {
				for i, h := range r.Hosts {
					r.Hosts[i].region = r.Name
					routes[h.sum()] = &r.Hosts[i]
				}
			}
		}
	}
	return routes
}

func requireBasicAuth(authenticator *auth.BasicAuth, handler func(http.ResponseWriter, *http.Request)) func(http.ResponseWriter, *http.Request) {
	return authenticator.Wrap(func(w http.ResponseWriter, r *auth.AuthenticatedRequest) {
		handler(w, &r.Request)
	})
}

func mux() http.Handler {
	mux := http.NewServeMux()
	authenticator := auth.NewBasicAuthenticator(
		"Selenium Grid Router",
		auth.HtpasswdFileProvider(users),
	)
	mux.HandleFunc(pingPath, ping)
	mux.HandleFunc(errPath, err)
	mux.HandleFunc(routePath, requireBasicAuth(authenticator, postOnly(withCloseNotifier(route))))
	mux.Handle(proxyPath, &httputil.ReverseProxy{Director: proxy})
	return mux
}
<|endoftext|>


<commit_before>package main_test

import (
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	"github.com/onsi/gomega/gexec"
	"io/ioutil"
	"os"
	"os/exec"
)

var _ = Describe("Verify_multidigest", func() {
	var session *gexec.Session
	var act func(arg ...string)
	var tempFile *os.File

	BeforeEach(func() {
		var err error
		tempFile, err = ioutil.TempFile("", "multi-digest-test")
		Expect(err).ToNot(HaveOccurred())
		_, err = tempFile.WriteString("sample content")
		Expect(err).ToNot(HaveOccurred())

		act = func(argCommands ...string) {
			var err error
			command := exec.Command(pathToBoshUtils, argCommands...)
			session, err = gexec.Start(command, GinkgoWriter, GinkgoWriter)
			Expect(err).ShouldNot(HaveOccurred())
		}
	})

	AfterEach(func() {
		os.Remove(tempFile.Name())
	})

	Describe("version option", func() {
		It("has a version flag", func() {
			act("--version")
			Eventually(session).Should(gexec.Exit(0))
			Eventually(session.Out).Should(gbytes.Say("version \\[DEV BUILD\\]"))
		})
	})

	Context("verification", func() {
		Context("when correct args are passed to verify-multi-digest command", func() {
			It("exits 0", func() {
				act("verify-multi-digest", tempFile.Name(), "c4f246e2d6f84ee61a699d68a4bd1a2e43ec40f6")
				Eventually(session).Should(gexec.Exit(0))
			})
		})

		Context("when passing incorrect args", func() {
			It("exits 1 when digest does not match", func() {
				act("verify-multi-digest", tempFile.Name(), "incorrect-digest")
				Eventually(session).Should(gexec.Exit(1))
				Eventually(session.Err).Should(gbytes.Say("Expected stream to have digest 'incorrect-digest' but was 'c4f246e2d6f84ee61a699d68a4bd1a2e43ec40f6'"))
			})

			It("exits 1 when file does not exist", func() {
				act("verify-multi-digest", "potato", "c4f246e2d6f84ee61a699d68a4bd1a2e43ec40f6")
				Eventually(session).Should(gexec.Exit(1))
				Eventually(session.Err).Should(gbytes.Say("open potato:"))
			})
		})
	})

	Context("digest creation", func() {
		Context("when correct args are passed to create-multi-digest command", func() {
			It("sha1", func() {
				act("create-multi-digest", "sha1", tempFile.Name())
				Eventually(session).Should(gexec.Exit(0))
				Eventually(session).Should(gbytes.Say("c4f246e2d6f84ee61a699d68a4bd1a2e43ec40f6"))
			})

			It("sha256", func() {
				act("create-multi-digest", "sha256", tempFile.Name())
				Eventually(session).Should(gexec.Exit(0))
				Eventually(session).Should(gbytes.Say("sha256:571ca3b4ef92a81f8c062f2c2437b9116435d1575589a7b64a5c607d058fde0d"))
			})

			It("sha512", func() {
				act("create-multi-digest", "sha512", tempFile.Name())
				Eventually(session).Should(gexec.Exit(0))
				Eventually(session).Should(gbytes.Say("sha512:bd9686023e9b5ddca02fe00ca0fcfe4dccbee6470ff90795aa005809c374b3a9f00cde7eba1a8266b715a0789041d08650d5cc4182856091ed93cfd3dd1195c8"))
			})

			It("sha1,sha256", func() {
				act("create-multi-digest", "sha1,sha256", tempFile.Name())
				Eventually(session).Should(gexec.Exit(0))
				Eventually(session).Should(gbytes.Say("c4f246e2d6f84ee61a699d68a4bd1a2e43ec40f6;sha256:571ca3b4ef92a81f8c062f2c2437b9116435d1575589a7b64a5c607d058fde0d"))
			})
		})

		Context("when passing incorrect args", func() {
			It("exits 1 when file does not exist", func() {
				act("create-multi-digest", "sha1", "potato")
				Eventually(session).Should(gexec.Exit(1))
				Eventually(session.Err).Should(gbytes.Say("Calculating digest of 'potato': open potato: no such file or directory"))
			})

			It("exits 1 when the algorithm is unknown", func() {
				act("create-multi-digest", "potoato", tempFile.Name())
				Eventually(session).Should(gexec.Exit(1))
				Eventually(session.Err).Should(gbytes.Say("unknown algorithm 'potoato'"))
			})
		})
	})

})
<commit_msg>Relax expectation for missing file<commit_after>package main_test

import (
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	"github.com/onsi/gomega/gexec"
	"io/ioutil"
	"os"
	"os/exec"
)

var _ = Describe("Verify_multidigest", func() {
	var session *gexec.Session
	var act func(arg ...string)
	var tempFile *os.File

	BeforeEach(func() {
		var err error
		tempFile, err = ioutil.TempFile("", "multi-digest-test")
		Expect(err).ToNot(HaveOccurred())
		_, err = tempFile.WriteString("sample content")
		Expect(err).ToNot(HaveOccurred())

		act = func(argCommands ...string) {
			var err error
			command := exec.Command(pathToBoshUtils, argCommands...)
			session, err = gexec.Start(command, GinkgoWriter, GinkgoWriter)
			Expect(err).ShouldNot(HaveOccurred())
		}
	})

	AfterEach(func() {
		os.Remove(tempFile.Name())
	})

	Describe("version option", func() {
		It("has a version flag", func() {
			act("--version")
			Eventually(session).Should(gexec.Exit(0))
			Eventually(session.Out).Should(gbytes.Say("version \\[DEV BUILD\\]"))
		})
	})

	Context("verification", func() {
		Context("when correct args are passed to verify-multi-digest command", func() {
			It("exits 0", func() {
				act("verify-multi-digest", tempFile.Name(), "c4f246e2d6f84ee61a699d68a4bd1a2e43ec40f6")
				Eventually(session).Should(gexec.Exit(0))
			})
		})

		Context("when passing incorrect args", func() {
			It("exits 1 when digest does not match", func() {
				act("verify-multi-digest", tempFile.Name(), "incorrect-digest")
				Eventually(session).Should(gexec.Exit(1))
				Eventually(session.Err).Should(gbytes.Say("Expected stream to have digest 'incorrect-digest' but was 'c4f246e2d6f84ee61a699d68a4bd1a2e43ec40f6'"))
			})

			It("exits 1 when file does not exist", func() {
				act("verify-multi-digest", "potato", "c4f246e2d6f84ee61a699d68a4bd1a2e43ec40f6")
				Eventually(session).Should(gexec.Exit(1))
				Eventually(session.Err).Should(gbytes.Say("open potato:"))
			})
		})
	})

	Context("digest creation", func() {
		Context("when correct args are passed to create-multi-digest command", func() {
			It("sha1", func() {
				act("create-multi-digest", "sha1", tempFile.Name())
				Eventually(session).Should(gexec.Exit(0))
				Eventually(session).Should(gbytes.Say("c4f246e2d6f84ee61a699d68a4bd1a2e43ec40f6"))
			})

			It("sha256", func() {
				act("create-multi-digest", "sha256", tempFile.Name())
				Eventually(session).Should(gexec.Exit(0))
				Eventually(session).Should(gbytes.Say("sha256:571ca3b4ef92a81f8c062f2c2437b9116435d1575589a7b64a5c607d058fde0d"))
			})

			It("sha512", func() {
				act("create-multi-digest", "sha512", tempFile.Name())
				Eventually(session).Should(gexec.Exit(0))
				Eventually(session).Should(gbytes.Say("sha512:bd9686023e9b5ddca02fe00ca0fcfe4dccbee6470ff90795aa005809c374b3a9f00cde7eba1a8266b715a0789041d08650d5cc4182856091ed93cfd3dd1195c8"))
			})

			It("sha1,sha256", func() {
				act("create-multi-digest", "sha1,sha256", tempFile.Name())
				Eventually(session).Should(gexec.Exit(0))
				Eventually(session).Should(gbytes.Say("c4f246e2d6f84ee61a699d68a4bd1a2e43ec40f6;sha256:571ca3b4ef92a81f8c062f2c2437b9116435d1575589a7b64a5c607d058fde0d"))
			})
		})

		Context("when passing incorrect args", func() {
			It("exits 1 when file does not exist", func() {
				act("create-multi-digest", "sha1", "potato")
				Eventually(session).Should(gexec.Exit(1))
				Eventually(session.Err).Should(gbytes.Say("Calculating digest of 'potato': open potato:"))
			})

			It("exits 1 when the algorithm is unknown", func() {
				act("create-multi-digest", "potoato", tempFile.Name())
				Eventually(session).Should(gexec.Exit(1))
				Eventually(session.Err).Should(gbytes.Say("unknown algorithm 'potoato'"))
			})
		})
	})

})
<|endoftext|>


<commit_before>//package main //for test
package GoHPACK

import (
	"fmt"
	"github.com/ami-GS/GoHPACK/huffman"
)

func PackIntRepresentation(I uint32, N byte) (buf []byte) {
	if I < uint32(1<<N)-1 {
		return []byte{byte(I)}
	} else {
		buf = []byte{byte(1<<N) - 1}
		I -= uint32(1<<N) - 1
		for I >= 0x80 {
			buf = append(buf, byte(I)&0x7f|0x80)
			I = (I >> 7)
		}
		buf = append(buf, byte(I))
		return buf
	}
}

func PackContent(content string, toHuffman bool) []byte {
	if len(content) == 0 {
		if toHuffman {
			return []byte{0x80}
		} else {
			return []byte{0x00}
		}
	}

	var Wire []byte
	if toHuffman {

		encoded, length := huffman.Root.Encode(content)
		intRep := PackIntRepresentation(uint32(length), 7)
		intRep[0] |= 0x80

		//Wire += hex.EncodeToString(*intRep) + strings.Trim(hex.EncodeToString(b), "00") // + encoded
		Wire = append(append(Wire, intRep...), encoded...)
	} else {
		intRep := PackIntRepresentation(uint32(len(content)), 7)
		Wire = append(append(Wire, intRep...), []byte(content)...)
	}
	return Wire
}

func Encode(Headers []Header, fromStaticTable, fromDynamicTable, toHuffman bool, table *Table, dynamicTableSize int) (Wire []byte) {
	if dynamicTableSize != -1 {
		intRep := PackIntRepresentation(uint32(dynamicTableSize), 5)
		intRep[0] |= 0x20
		Wire = append(Wire, intRep...)
	}

	for _, header := range Headers {
		match, index := table.FindHeader(header)
		if fromStaticTable && match {
			var indexLen, mask byte
			var content []byte
			if fromDynamicTable {
				indexLen = 7
				mask = 0x80
				content = []byte{}
			} else {
				indexLen = 4
				mask = 0x00
				content = PackContent(header.Value, toHuffman)
			}
			intRep := PackIntRepresentation(uint32(index), indexLen)
			intRep[0] |= mask
			Wire = append(append(Wire, intRep...), content...)
		} else if fromStaticTable && !match && index > 0 {
			var indexLen, mask byte
			if fromDynamicTable {
				indexLen = 6
				mask = 0x40
				table.AddHeader(header)
			} else {
				indexLen = 4
				mask = 0x00
			}
			intRep := PackIntRepresentation(uint32(index), indexLen)
			intRep[0] |= mask
			Wire = append(append(Wire, intRep...), PackContent(header.Value, toHuffman)...)
		} else {
			var prefix []byte
			if fromDynamicTable {
				prefix = []byte{0x40}
				table.AddHeader(header)
			} else {
				prefix = []byte{0x00}
			}
			content := append(PackContent(header.Name, toHuffman), PackContent(header.Value, toHuffman)...)
			Wire = append(append(Wire, prefix...), content...)
		}
	}

	return
}

func ParseIntRepresentation(buf []byte, N byte) (I, cursor uint32) {
	I = uint32(buf[0] & ((1 << N) - 1)) // byte could be used as byte
	cursor = 1
	if I < ((1 << N) - 1) {
		return I, cursor
	} else {
		var M byte = 0
		for (buf[cursor] & 0x80) > 0 {
			I += uint32(buf[cursor]&0x7f) * (1 << M)
			M += 7
			cursor += 1
		}
		I += uint32(buf[cursor]&0x7f) * (1 << M)
		return I, cursor + 1
	}
}

func ParseFromByte(buf []byte) (content string, cursor uint32) {
	length, cursor := ParseIntRepresentation(buf, 7)

	if buf[0]&0x80 > 0 {
		content = huffman.Root.Decode(buf[cursor:], length)
	} else {
		content = string(buf[cursor : cursor+length])
	}

	cursor += length
	return
}

func ParseHeader(index uint32, buf []byte, isIndexed bool, table *Table) (name, value string, cursor uint32) {
	if c := uint32(0); !isIndexed {
		if index == 0 {
			name, c = ParseFromByte(buf[cursor:])
			cursor += c
		}
		value, c = ParseFromByte(buf[cursor:])
		cursor += c
	}

	if index > 0 {
		header := table.GetHeader(index)

		name = header.Name
		if len(value) == 0 {
			value = header.Value
		}
	}
	return
}

func Decode(buf []byte, table *Table) (Headers []Header) {
	var cursor uint32 = 0
	for cursor < uint32(len(buf)) {
		isIndexed := false
		isIncremental := false
		var index, c uint32
		if buf[cursor]&0xe0 == 0x20 {
			// 7.3 Header Table Size Update
			size, c := ParseIntRepresentation(buf[cursor:], 5)
			table.SetDynamicTableSize(size)
			cursor += c
		}

		if (buf[cursor] & 0x80) > 0 {
			// 7.1 Indexed Header Field
			if (buf[cursor] & 0x7f) == 0 {
				panic('a')
			}
			index, c = ParseIntRepresentation(buf[cursor:], 7)
			isIndexed = true
		} else {
			if buf[cursor]&0xc0 == 0x40 {
				// 7.2.1 Literal Header Field with Incremental Indexing
				index, c = ParseIntRepresentation(buf[cursor:], 6)
				isIncremental = true
			} else if buf[cursor]&0xf0 == 0xf0 {
				// 7.2.3 Literal Header Field never Indexed
				index, c = ParseIntRepresentation(buf[cursor:], 4)
			} else {
				// 7.2.2 Literal Header Field without Indexing
				index, c = ParseIntRepresentation(buf[cursor:], 4)
			}
		}
		cursor += c

		name, value, c := ParseHeader(index, buf[cursor:], isIndexed, table)
		cursor += c

		header := Header{name, value}
		if isIncremental {
			table.AddHeader(header)
		}
		Headers = append(Headers, header)
	}

	return
}

func main() {
	//nums, _ := hex.DecodeString(string("1FA18DB701"))
	//fmt.Println(nums)
	//fmt.Println(ParseIntRepresentation(nums, 5))
	//decode("ff80000111")
	//fmt.Println(Decode("00073a6d6574686f640347455400073a736368656d650468747470000a3a617574686f726974790f7777772e7961686f6f2e636f2e6a7000053a70617468012f"))
	fmt.Println(huffman.HUFFMAN_TABLE)
}
<commit_msg>remove nonsense append<commit_after>//package main //for test
package GoHPACK

import (
	"fmt"
	"github.com/ami-GS/GoHPACK/huffman"
)

func PackIntRepresentation(I uint32, N byte) (buf []byte) {
	if I < uint32(1<<N)-1 {
		return []byte{byte(I)}
	} else {
		buf = []byte{byte(1<<N) - 1}
		I -= uint32(1<<N) - 1
		for I >= 0x80 {
			buf = append(buf, byte(I)&0x7f|0x80)
			I = (I >> 7)
		}
		buf = append(buf, byte(I))
		return buf
	}
}

func PackContent(content string, toHuffman bool) []byte {
	if len(content) == 0 {
		if toHuffman {
			return []byte{0x80}
		} else {
			return []byte{0x00}
		}
	}

	var Wire []byte
	if toHuffman {

		encoded, length := huffman.Root.Encode(content)
		intRep := PackIntRepresentation(uint32(length), 7)
		intRep[0] |= 0x80

		//Wire += hex.EncodeToString(*intRep) + strings.Trim(hex.EncodeToString(b), "00") // + encoded
		Wire = append(append(Wire, intRep...), encoded...)
	} else {
		intRep := PackIntRepresentation(uint32(len(content)), 7)
		Wire = append(append(Wire, intRep...), []byte(content)...)
	}
	return Wire
}

func Encode(Headers []Header, fromStaticTable, fromDynamicTable, toHuffman bool, table *Table, dynamicTableSize int) (Wire []byte) {
	if dynamicTableSize != -1 {
		intRep := PackIntRepresentation(uint32(dynamicTableSize), 5)
		intRep[0] |= 0x20
		Wire = intRep
	}

	for _, header := range Headers {
		match, index := table.FindHeader(header)
		if fromStaticTable && match {
			var indexLen, mask byte
			var content []byte
			if fromDynamicTable {
				indexLen = 7
				mask = 0x80
				content = []byte{}
			} else {
				indexLen = 4
				mask = 0x00
				content = PackContent(header.Value, toHuffman)
			}
			intRep := PackIntRepresentation(uint32(index), indexLen)
			intRep[0] |= mask
			Wire = append(append(Wire, intRep...), content...)
		} else if fromStaticTable && !match && index > 0 {
			var indexLen, mask byte
			if fromDynamicTable {
				indexLen = 6
				mask = 0x40
				table.AddHeader(header)
			} else {
				indexLen = 4
				mask = 0x00
			}
			intRep := PackIntRepresentation(uint32(index), indexLen)
			intRep[0] |= mask
			Wire = append(append(Wire, intRep...), PackContent(header.Value, toHuffman)...)
		} else {
			var prefix []byte
			if fromDynamicTable {
				prefix = []byte{0x40}
				table.AddHeader(header)
			} else {
				prefix = []byte{0x00}
			}
			content := append(PackContent(header.Name, toHuffman), PackContent(header.Value, toHuffman)...)
			Wire = append(append(Wire, prefix...), content...)
		}
	}

	return
}

func ParseIntRepresentation(buf []byte, N byte) (I, cursor uint32) {
	I = uint32(buf[0] & ((1 << N) - 1)) // byte could be used as byte
	cursor = 1
	if I < ((1 << N) - 1) {
		return I, cursor
	} else {
		var M byte = 0
		for (buf[cursor] & 0x80) > 0 {
			I += uint32(buf[cursor]&0x7f) * (1 << M)
			M += 7
			cursor += 1
		}
		I += uint32(buf[cursor]&0x7f) * (1 << M)
		return I, cursor + 1
	}
}

func ParseFromByte(buf []byte) (content string, cursor uint32) {
	length, cursor := ParseIntRepresentation(buf, 7)

	if buf[0]&0x80 > 0 {
		content = huffman.Root.Decode(buf[cursor:], length)
	} else {
		content = string(buf[cursor : cursor+length])
	}

	cursor += length
	return
}

func ParseHeader(index uint32, buf []byte, isIndexed bool, table *Table) (name, value string, cursor uint32) {
	if c := uint32(0); !isIndexed {
		if index == 0 {
			name, c = ParseFromByte(buf[cursor:])
			cursor += c
		}
		value, c = ParseFromByte(buf[cursor:])
		cursor += c
	}

	if index > 0 {
		header := table.GetHeader(index)

		name = header.Name
		if len(value) == 0 {
			value = header.Value
		}
	}
	return
}

func Decode(buf []byte, table *Table) (Headers []Header) {
	var cursor uint32 = 0
	for cursor < uint32(len(buf)) {
		isIndexed := false
		isIncremental := false
		var index, c uint32
		if buf[cursor]&0xe0 == 0x20 {
			// 7.3 Header Table Size Update
			size, c := ParseIntRepresentation(buf[cursor:], 5)
			table.SetDynamicTableSize(size)
			cursor += c
		}

		if (buf[cursor] & 0x80) > 0 {
			// 7.1 Indexed Header Field
			if (buf[cursor] & 0x7f) == 0 {
				panic('a')
			}
			index, c = ParseIntRepresentation(buf[cursor:], 7)
			isIndexed = true
		} else {
			if buf[cursor]&0xc0 == 0x40 {
				// 7.2.1 Literal Header Field with Incremental Indexing
				index, c = ParseIntRepresentation(buf[cursor:], 6)
				isIncremental = true
			} else if buf[cursor]&0xf0 == 0xf0 {
				// 7.2.3 Literal Header Field never Indexed
				index, c = ParseIntRepresentation(buf[cursor:], 4)
			} else {
				// 7.2.2 Literal Header Field without Indexing
				index, c = ParseIntRepresentation(buf[cursor:], 4)
			}
		}
		cursor += c

		name, value, c := ParseHeader(index, buf[cursor:], isIndexed, table)
		cursor += c

		header := Header{name, value}
		if isIncremental {
			table.AddHeader(header)
		}
		Headers = append(Headers, header)
	}

	return
}

func main() {
	//nums, _ := hex.DecodeString(string("1FA18DB701"))
	//fmt.Println(nums)
	//fmt.Println(ParseIntRepresentation(nums, 5))
	//decode("ff80000111")
	//fmt.Println(Decode("00073a6d6574686f640347455400073a736368656d650468747470000a3a617574686f726974790f7777772e7961686f6f2e636f2e6a7000053a70617468012f"))
	fmt.Println(huffman.HUFFMAN_TABLE)
}
<|endoftext|>