<commit_before>/*
Copyright 2021 The cert-manager Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cmapichecker

import (
	"context"
	"errors"
	"fmt"
	"testing"

	"k8s.io/apimachinery/pkg/runtime"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/client/fake"

	cmapi "github.com/jetstack/cert-manager/pkg/apis/certmanager/v1"
)

type fakeErrorClient struct {
	client.Client

	createError error
}

func (cl *fakeErrorClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error {
	if cl.createError != nil {
		return cl.createError
	}

	return cl.Client.Create(ctx, obj, opts...)
}

func newFakeCmapiChecker() (*fakeErrorClient, Interface, error) {
	scheme := runtime.NewScheme()
	if err := cmapi.AddToScheme(scheme); err != nil {
		return nil, nil, err
	}
	cl := fake.NewClientBuilder().WithScheme(scheme).Build()
	errorClient := &fakeErrorClient{
		Client:      cl,
		createError: nil,
	}

	return errorClient, &cmapiChecker{
		client: errorClient,
	}, nil
}

const (
	errCertManagerCRDsMapping  = `error finding the scope of the object: failed to get restmapping: no matches for kind "Certificate" in group "cert-manager.io"`
	errCertManagerCRDsNotFound = `the server could not find the requested resource (post certificates.cert-manager.io)`

	errMutatingWebhookServiceFailure     = `Internal error occurred: failed calling webhook "webhook.cert-manager.io": Post "https://cert-manager-webhook.cert-manager.svc:443/mutate?timeout=10s": service "cert-manager-webhook" not found`
	errMutatingWebhookDeploymentFailure  = `Internal error occurred: failed calling webhook "webhook.cert-manager.io": Post "https://cert-manager-webhook.cert-manager.svc:443/mutate?timeout=10s": dial tcp 10.96.38.90:443: connect: connection refused`
	errMutatingWebhookCertificateFailure = `Internal error occurred: failed calling webhook "webhook.cert-manager.io": Post "https://cert-manager-webhook.cert-manager.svc:443/mutate?timeout=10s": x509: certificate signed by unknown authority (possibly because of "x509: ECDSA verification failure" while trying to verify candidate authority certificate "cert-manager-webhook-ca"`

	errConversionWebhookServiceFailure     = `conversion webhook for cert-manager.io/v1alpha2, Kind=Certificate failed: Post "https://cert-manager-webhook.cert-manager.svc:443/convert?timeout=30s": service "cert-manager-webhook" not found`
	errConversionWebhookDeploymentFailure  = `conversion webhook for cert-manager.io/v1alpha2, Kind=Certificate failed: Post "https://cert-manager-webhook.cert-manager.svc:443/convert?timeout=30s": dial tcp 10.96.38.90:443: connect: connection refused`
	errConversionWebhookCertificateFailure = `conversion webhook for cert-manager.io/v1alpha2, Kind=Certificate failed: Post "https://cert-manager-webhook.cert-manager.svc:443/convert?timeout=30s": x509: certificate signed by unknown authority`
)

func TestCmapiChecker(t *testing.T) {
	tests := map[string]testT{
		"check API without errors": {
			createError: nil,

			expectedSimpleError:  "",
			expectedVerboseError: "",
		},
		"check API without CRDs installed 1": {
			createError: errors.New(errCertManagerCRDsMapping),

			expectedSimpleError:  ErrCertManagerCRDsNotFound.Error(),
			expectedVerboseError: fmt.Sprintf("%s (%s)", ErrCertManagerCRDsNotFound.Error(), errCertManagerCRDsMapping),
		},
		"check API without CRDs installed 2": {
			createError: errors.New(errCertManagerCRDsNotFound),

			expectedSimpleError:  ErrCertManagerCRDsNotFound.Error(),
			expectedVerboseError: fmt.Sprintf("%s (%s)", ErrCertManagerCRDsNotFound.Error(), errCertManagerCRDsNotFound),
		},

		"check API with mutating webhook service not ready": {
			createError: errors.New(errMutatingWebhookServiceFailure),

			expectedSimpleError:  ErrWebhookServiceFailure.Error(),
			expectedVerboseError: fmt.Sprintf("%s (%s)", ErrWebhookServiceFailure.Error(), errMutatingWebhookServiceFailure),
		},
		"check API with conversion webhook service not ready": {
			createError: errors.New(errConversionWebhookServiceFailure),

			expectedSimpleError:  ErrWebhookServiceFailure.Error(),
			expectedVerboseError: fmt.Sprintf("%s (%s)", ErrWebhookServiceFailure.Error(), errConversionWebhookServiceFailure),
		},

		"check API with mutating webhook pod not accepting connections": {
			createError: errors.New(errMutatingWebhookDeploymentFailure),

			expectedSimpleError:  ErrWebhookDeploymentFailure.Error(),
			expectedVerboseError: fmt.Sprintf("%s (%s)", ErrWebhookDeploymentFailure.Error(), errMutatingWebhookDeploymentFailure),
		},
		"check API with conversion webhook pod not accepting connections": {
			createError: errors.New(errConversionWebhookDeploymentFailure),

			expectedSimpleError:  ErrWebhookDeploymentFailure.Error(),
			expectedVerboseError: fmt.Sprintf("%s (%s)", ErrWebhookDeploymentFailure.Error(), errConversionWebhookDeploymentFailure),
		},

		"check API with webhook certificate not updated in mutation webhook resource definitions": {
			createError: errors.New(errMutatingWebhookCertificateFailure),

			expectedSimpleError:  ErrWebhookCertificateFailure.Error(),
			expectedVerboseError: fmt.Sprintf("%s (%s)", ErrWebhookCertificateFailure.Error(), errMutatingWebhookCertificateFailure),
		},
		"check API with webhook certificate not updated in conversion webhook resource definitions": {
			createError: errors.New(errConversionWebhookCertificateFailure),

			expectedSimpleError:  ErrWebhookCertificateFailure.Error(),
			expectedVerboseError: fmt.Sprintf("%s (%s)", ErrWebhookCertificateFailure.Error(), errConversionWebhookCertificateFailure),
		},
		"unexpected error": {
			createError: errors.New("unexpected error"),

			expectedSimpleError:  "",
			expectedVerboseError: "unexpected error",
		},
	}

	for n, test := range tests {
		t.Run(n, func(t *testing.T) {
			runTest(t, test)
		})
	}
}

type testT struct {
	createError error

	expectedSimpleError  string
	expectedVerboseError string
}

func runTest(t *testing.T, test testT) {
	errorClient, checker, err := newFakeCmapiChecker()
	if err != nil {
		t.Error(err)
	}

	errorClient.createError = test.createError

	var unwrappedErr error
	err = checker.Check(context.TODO())
	if err != nil {
		if err.Error() != test.expectedVerboseError {
			t.Errorf("error differs from expected error:\n%s\n vs \n%s", err.Error(), test.expectedVerboseError)
		}

		unwrappedErr = errors.Unwrap(err)
	} else {
		if test.expectedVerboseError != "" {
			t.Errorf("expected error did not occure:\n%s", test.expectedVerboseError)
		}
	}

	if unwrappedErr != nil {
		if unwrappedErr.Error() != test.expectedSimpleError {
			t.Errorf("simple error differs from expected error:\n%s\n vs \n%s", unwrappedErr.Error(), test.expectedSimpleError)
		}
	} else {
		if test.expectedSimpleError != "" {
			t.Errorf("expected simple error did not occure:\n%s", test.expectedSimpleError)
		}
	}
}
<commit_msg>A note about the relevance of conversion webhook unit-tests<commit_after>/*
Copyright 2021 The cert-manager Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cmapichecker

import (
	"context"
	"errors"
	"fmt"
	"testing"

	"k8s.io/apimachinery/pkg/runtime"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/client/fake"

	cmapi "github.com/jetstack/cert-manager/pkg/apis/certmanager/v1"
)

type fakeErrorClient struct {
	client.Client

	createError error
}

func (cl *fakeErrorClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error {
	if cl.createError != nil {
		return cl.createError
	}

	return cl.Client.Create(ctx, obj, opts...)
}

func newFakeCmapiChecker() (*fakeErrorClient, Interface, error) {
	scheme := runtime.NewScheme()
	if err := cmapi.AddToScheme(scheme); err != nil {
		return nil, nil, err
	}
	cl := fake.NewClientBuilder().WithScheme(scheme).Build()
	errorClient := &fakeErrorClient{
		Client:      cl,
		createError: nil,
	}

	return errorClient, &cmapiChecker{
		client: errorClient,
	}, nil
}

const (
	errCertManagerCRDsMapping  = `error finding the scope of the object: failed to get restmapping: no matches for kind "Certificate" in group "cert-manager.io"`
	errCertManagerCRDsNotFound = `the server could not find the requested resource (post certificates.cert-manager.io)`

	errMutatingWebhookServiceFailure     = `Internal error occurred: failed calling webhook "webhook.cert-manager.io": Post "https://cert-manager-webhook.cert-manager.svc:443/mutate?timeout=10s": service "cert-manager-webhook" not found`
	errMutatingWebhookDeploymentFailure  = `Internal error occurred: failed calling webhook "webhook.cert-manager.io": Post "https://cert-manager-webhook.cert-manager.svc:443/mutate?timeout=10s": dial tcp 10.96.38.90:443: connect: connection refused`
	errMutatingWebhookCertificateFailure = `Internal error occurred: failed calling webhook "webhook.cert-manager.io": Post "https://cert-manager-webhook.cert-manager.svc:443/mutate?timeout=10s": x509: certificate signed by unknown authority (possibly because of "x509: ECDSA verification failure" while trying to verify candidate authority certificate "cert-manager-webhook-ca"`

	// These /convert error examples test that we can correctly parse errors
	// while connecting to the conversion webhook,
	// but as of cert-manager 1.6 the conversion webhook will no-longer be used
	// because legacy CRD versions will no longer be "served"
	// and in 1.7 the conversion webhook may be removed at which point these can
	// be removed too.
	errConversionWebhookServiceFailure     = `conversion webhook for cert-manager.io/v1alpha2, Kind=Certificate failed: Post "https://cert-manager-webhook.cert-manager.svc:443/convert?timeout=30s": service "cert-manager-webhook" not found`
	errConversionWebhookDeploymentFailure  = `conversion webhook for cert-manager.io/v1alpha2, Kind=Certificate failed: Post "https://cert-manager-webhook.cert-manager.svc:443/convert?timeout=30s": dial tcp 10.96.38.90:443: connect: connection refused`
	errConversionWebhookCertificateFailure = `conversion webhook for cert-manager.io/v1alpha2, Kind=Certificate failed: Post "https://cert-manager-webhook.cert-manager.svc:443/convert?timeout=30s": x509: certificate signed by unknown authority`
)

func TestCmapiChecker(t *testing.T) {
	tests := map[string]testT{
		"check API without errors": {
			createError: nil,

			expectedSimpleError:  "",
			expectedVerboseError: "",
		},
		"check API without CRDs installed 1": {
			createError: errors.New(errCertManagerCRDsMapping),

			expectedSimpleError:  ErrCertManagerCRDsNotFound.Error(),
			expectedVerboseError: fmt.Sprintf("%s (%s)", ErrCertManagerCRDsNotFound.Error(), errCertManagerCRDsMapping),
		},
		"check API without CRDs installed 2": {
			createError: errors.New(errCertManagerCRDsNotFound),

			expectedSimpleError:  ErrCertManagerCRDsNotFound.Error(),
			expectedVerboseError: fmt.Sprintf("%s (%s)", ErrCertManagerCRDsNotFound.Error(), errCertManagerCRDsNotFound),
		},

		"check API with mutating webhook service not ready": {
			createError: errors.New(errMutatingWebhookServiceFailure),

			expectedSimpleError:  ErrWebhookServiceFailure.Error(),
			expectedVerboseError: fmt.Sprintf("%s (%s)", ErrWebhookServiceFailure.Error(), errMutatingWebhookServiceFailure),
		},
		"check API with conversion webhook service not ready": {
			createError: errors.New(errConversionWebhookServiceFailure),

			expectedSimpleError:  ErrWebhookServiceFailure.Error(),
			expectedVerboseError: fmt.Sprintf("%s (%s)", ErrWebhookServiceFailure.Error(), errConversionWebhookServiceFailure),
		},

		"check API with mutating webhook pod not accepting connections": {
			createError: errors.New(errMutatingWebhookDeploymentFailure),

			expectedSimpleError:  ErrWebhookDeploymentFailure.Error(),
			expectedVerboseError: fmt.Sprintf("%s (%s)", ErrWebhookDeploymentFailure.Error(), errMutatingWebhookDeploymentFailure),
		},
		"check API with conversion webhook pod not accepting connections": {
			createError: errors.New(errConversionWebhookDeploymentFailure),

			expectedSimpleError:  ErrWebhookDeploymentFailure.Error(),
			expectedVerboseError: fmt.Sprintf("%s (%s)", ErrWebhookDeploymentFailure.Error(), errConversionWebhookDeploymentFailure),
		},

		"check API with webhook certificate not updated in mutation webhook resource definitions": {
			createError: errors.New(errMutatingWebhookCertificateFailure),

			expectedSimpleError:  ErrWebhookCertificateFailure.Error(),
			expectedVerboseError: fmt.Sprintf("%s (%s)", ErrWebhookCertificateFailure.Error(), errMutatingWebhookCertificateFailure),
		},
		"check API with webhook certificate not updated in conversion webhook resource definitions": {
			createError: errors.New(errConversionWebhookCertificateFailure),

			expectedSimpleError:  ErrWebhookCertificateFailure.Error(),
			expectedVerboseError: fmt.Sprintf("%s (%s)", ErrWebhookCertificateFailure.Error(), errConversionWebhookCertificateFailure),
		},
		"unexpected error": {
			createError: errors.New("unexpected error"),

			expectedSimpleError:  "",
			expectedVerboseError: "unexpected error",
		},
	}

	for n, test := range tests {
		t.Run(n, func(t *testing.T) {
			runTest(t, test)
		})
	}
}

type testT struct {
	createError error

	expectedSimpleError  string
	expectedVerboseError string
}

func runTest(t *testing.T, test testT) {
	errorClient, checker, err := newFakeCmapiChecker()
	if err != nil {
		t.Error(err)
	}

	errorClient.createError = test.createError

	var unwrappedErr error
	err = checker.Check(context.TODO())
	if err != nil {
		if err.Error() != test.expectedVerboseError {
			t.Errorf("error differs from expected error:\n%s\n vs \n%s", err.Error(), test.expectedVerboseError)
		}

		unwrappedErr = errors.Unwrap(err)
	} else {
		if test.expectedVerboseError != "" {
			t.Errorf("expected error did not occure:\n%s", test.expectedVerboseError)
		}
	}

	if unwrappedErr != nil {
		if unwrappedErr.Error() != test.expectedSimpleError {
			t.Errorf("simple error differs from expected error:\n%s\n vs \n%s", unwrappedErr.Error(), test.expectedSimpleError)
		}
	} else {
		if test.expectedSimpleError != "" {
			t.Errorf("expected simple error did not occure:\n%s", test.expectedSimpleError)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2014, Google Inc.  All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tao

import (
	"net"
	"os"
	"syscall"

	"github.com/jlmucb/cloudproxy/go/tao/auth"
	"github.com/jlmucb/cloudproxy/go/util"
	"github.com/jlmucb/cloudproxy/go/util/protorpc"
)

func NewAuthenticatedFileSocket(sock *net.UnixListener) (error) {
	sockFile, err = sock.File()
	if err != nil {
		return err
		}
	err = syscall.SetsockoptInt(int(sockFile.Fd()), syscall.SOL_SOCKET, syscall.SO_PASSCRED, 1)
        sockFile.Close()
	return err
}
<commit_msg>Sorryn there was an error in linux_host_admin_rpc_linux.go caused by the last checkin.<commit_after>// Copyright (c) 2014, Google Inc.  All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tao

import (
	"net"
	"syscall"
)

func NewAuthenticatedFileSocket(sock *net.UnixListener) (error) {
	sockFile, err := sock.File()
	if err != nil {
		return err
		}
	err = syscall.SetsockoptInt(int(sockFile.Fd()), syscall.SOL_SOCKET, syscall.SO_PASSCRED, 1)
        sockFile.Close()
	return err
}
<|endoftext|>


<commit_before>package builder

import (
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"

	s2iapi "github.com/openshift/source-to-image/pkg/api"
)

var (
	// procCGroupPattern is a regular expression that parses the entries in /proc/self/cgroup
	procCGroupPattern = regexp.MustCompile(`\d+:([a-z_,]+):/.*/(docker-|)([a-z0-9]+).*`)
)

// readNetClsCGroup parses /proc/self/cgroup in order to determine the container id that can be used
// the network namespace that this process is running on.
func readNetClsCGroup(reader io.Reader) string {
	cgroups := make(map[string]string)

	scanner := bufio.NewScanner(reader)
	for scanner.Scan() {
		if match := procCGroupPattern.FindStringSubmatch(scanner.Text()); match != nil {
			list := strings.Split(match[1], ",")
			containerId := match[3]
			if len(list) > 0 {
				for _, key := range list {
					cgroups[key] = containerId
				}
			} else {
				cgroups[match[1]] = containerId
			}
		}
	}

	names := []string{"net_cls", "cpu"}
	for _, group := range names {
		if value, ok := cgroups[group]; ok {
			return value
		}
	}

	return ""
}

// getDockerNetworkMode determines whether the builder is running as a container
// by examining /proc/self/cgroup. This contenxt is then passed to source-to-image.
func getDockerNetworkMode() s2iapi.DockerNetworkMode {
	file, err := os.Open("/proc/self/cgroup")
	if err != nil {
		return ""
	}
	defer file.Close()

	if id := readNetClsCGroup(file); id != "" {
		return s2iapi.NewDockerNetworkModeContainer(id)
	}
	return ""
}

// GetCGroupLimits returns a struct populated with cgroup limit values gathered
// from the local /sys/fs/cgroup filesystem.  Overflow values are set to
// math.MaxInt64.
func GetCGroupLimits() (*s2iapi.CGroupLimits, error) {
	byteLimit, err := readInt64("/sys/fs/cgroup/memory/memory.limit_in_bytes")
	if err != nil {
		return nil, fmt.Errorf("cannot determine cgroup limits: %v", err)
	}

	// different docker versions seem to use different cgroup directories,
	// check for all of them.

	// seen on rhel systems
	cpuDir := "/sys/fs/cgroup/cpuacct,cpu"

	// seen on fedora systems with docker 1.9
	// note that in this case there is also a /sys/fs/cgroup/cpu that symlinks
	// to /sys/fs/cgroup/cpu,cpuacct, so technically the next check
	// would be sufficient, but it seems better to rely on the real directory
	// rather than a symlink.
	if _, err := os.Stat("/sys/fs/cgroup/cpu,cpuacct"); err == nil {
		cpuDir = "/sys/fs/cgroup/cpu,cpuacct"
	}

	// seen on debian systems with docker 1.10
	if _, err := os.Stat("/sys/fs/cgroup/cpu"); err == nil {
		cpuDir = "/sys/fs/cgroup/cpu"
	}

	cpuQuota, err := readInt64(filepath.Join(cpuDir, "cpu.cfs_quota_us"))
	if err != nil {
		return nil, fmt.Errorf("cannot determine cgroup limits: %v", err)
	}

	cpuPeriod, err := readInt64(filepath.Join(cpuDir, "cpu.cfs_period_us"))
	if err != nil {
		return nil, fmt.Errorf("cannot determine cgroup limits: %v", err)
	}

	cpuShares, err := readInt64(filepath.Join(cpuDir, "cpu.shares"))
	if err != nil {
		return nil, fmt.Errorf("cannot determine cgroup limits: %v", err)
	}

	return &s2iapi.CGroupLimits{
		CPUShares:        cpuShares,
		CPUPeriod:        cpuPeriod,
		CPUQuota:         cpuQuota,
		MemoryLimitBytes: byteLimit,
		// Set memoryswap==memorylimit, this ensures no swapping occurs.
		// see: https://docs.docker.com/engine/reference/run/#runtime-constraints-on-cpu-and-memory
		MemorySwap: byteLimit,
	}, nil
}

func readInt64(filePath string) (int64, error) {
	data, err := ioutil.ReadFile(filePath)
	if err != nil {
		return -1, err
	}
	s := strings.TrimSpace(string(data))
	val, err := strconv.ParseInt(s, 10, 64)
	// overflow errors are ok, we'll get return a math.MaxInt64 value which is more
	// than enough anyway.  For underflow we'll return MinInt64 and the error.
	if err != nil && err.(*strconv.NumError).Err == strconv.ErrRange {
		if s[0] == '-' {
			return math.MinInt64, err
		}
		return math.MaxInt64, nil
	} else if err != nil {
		return -1, err
	}
	return val, nil
}

// MergeEnv will take an existing environment and merge it with a new set of
// variables. For variables with the same name in both, only the one in the
// new environment will be kept.
func MergeEnv(oldEnv, newEnv []string) []string {
	key := func(e string) string {
		i := strings.Index(e, "=")
		if i == -1 {
			return e
		}
		return e[:i]
	}
	result := []string{}
	newVars := map[string]struct{}{}
	for _, e := range newEnv {
		newVars[key(e)] = struct{}{}
	}
	result = append(result, newEnv...)
	for _, e := range oldEnv {
		if _, exists := newVars[key(e)]; exists {
			continue
		}
		result = append(result, e)
	}
	return result
}
<commit_msg>use a max value of 92233720368547 for cgroup values<commit_after>package builder

import (
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"

	s2iapi "github.com/openshift/source-to-image/pkg/api"
)

var (
	// procCGroupPattern is a regular expression that parses the entries in /proc/self/cgroup
	procCGroupPattern = regexp.MustCompile(`\d+:([a-z_,]+):/.*/(docker-|)([a-z0-9]+).*`)
)

// readNetClsCGroup parses /proc/self/cgroup in order to determine the container id that can be used
// the network namespace that this process is running on.
func readNetClsCGroup(reader io.Reader) string {
	cgroups := make(map[string]string)

	scanner := bufio.NewScanner(reader)
	for scanner.Scan() {
		if match := procCGroupPattern.FindStringSubmatch(scanner.Text()); match != nil {
			list := strings.Split(match[1], ",")
			containerId := match[3]
			if len(list) > 0 {
				for _, key := range list {
					cgroups[key] = containerId
				}
			} else {
				cgroups[match[1]] = containerId
			}
		}
	}

	names := []string{"net_cls", "cpu"}
	for _, group := range names {
		if value, ok := cgroups[group]; ok {
			return value
		}
	}

	return ""
}

// getDockerNetworkMode determines whether the builder is running as a container
// by examining /proc/self/cgroup. This contenxt is then passed to source-to-image.
func getDockerNetworkMode() s2iapi.DockerNetworkMode {
	file, err := os.Open("/proc/self/cgroup")
	if err != nil {
		return ""
	}
	defer file.Close()

	if id := readNetClsCGroup(file); id != "" {
		return s2iapi.NewDockerNetworkModeContainer(id)
	}
	return ""
}

// GetCGroupLimits returns a struct populated with cgroup limit values gathered
// from the local /sys/fs/cgroup filesystem.  Overflow values are set to
// math.MaxInt64.
func GetCGroupLimits() (*s2iapi.CGroupLimits, error) {
	byteLimit, err := readInt64("/sys/fs/cgroup/memory/memory.limit_in_bytes")
	if err != nil {
		return nil, fmt.Errorf("cannot determine cgroup limits: %v", err)
	}
	// math.MaxInt64 seems to give cgroups trouble, this value is
	// still 92 terabytes, so it ought to be sufficiently large for
	// our purposes.
	if byteLimit > 92233720368547 {
		byteLimit = 92233720368547
	}

	// different docker versions seem to use different cgroup directories,
	// check for all of them.

	// seen on rhel systems
	cpuDir := "/sys/fs/cgroup/cpuacct,cpu"

	// seen on fedora systems with docker 1.9
	// note that in this case there is also a /sys/fs/cgroup/cpu that symlinks
	// to /sys/fs/cgroup/cpu,cpuacct, so technically the next check
	// would be sufficient, but it seems better to rely on the real directory
	// rather than a symlink.
	if _, err := os.Stat("/sys/fs/cgroup/cpu,cpuacct"); err == nil {
		cpuDir = "/sys/fs/cgroup/cpu,cpuacct"
	}

	// seen on debian systems with docker 1.10
	if _, err := os.Stat("/sys/fs/cgroup/cpu"); err == nil {
		cpuDir = "/sys/fs/cgroup/cpu"
	}

	cpuQuota, err := readInt64(filepath.Join(cpuDir, "cpu.cfs_quota_us"))
	if err != nil {
		return nil, fmt.Errorf("cannot determine cgroup limits: %v", err)
	}

	cpuPeriod, err := readInt64(filepath.Join(cpuDir, "cpu.cfs_period_us"))
	if err != nil {
		return nil, fmt.Errorf("cannot determine cgroup limits: %v", err)
	}

	cpuShares, err := readInt64(filepath.Join(cpuDir, "cpu.shares"))
	if err != nil {
		return nil, fmt.Errorf("cannot determine cgroup limits: %v", err)
	}

	return &s2iapi.CGroupLimits{
		CPUShares:        cpuShares,
		CPUPeriod:        cpuPeriod,
		CPUQuota:         cpuQuota,
		MemoryLimitBytes: byteLimit,
		// Set memoryswap==memorylimit, this ensures no swapping occurs.
		// see: https://docs.docker.com/engine/reference/run/#runtime-constraints-on-cpu-and-memory
		MemorySwap: byteLimit,
	}, nil
}

func readInt64(filePath string) (int64, error) {
	data, err := ioutil.ReadFile(filePath)
	if err != nil {
		return -1, err
	}
	s := strings.TrimSpace(string(data))
	val, err := strconv.ParseInt(s, 10, 64)
	// overflow errors are ok, we'll get return a math.MaxInt64 value which is more
	// than enough anyway.  For underflow we'll return MinInt64 and the error.
	if err != nil && err.(*strconv.NumError).Err == strconv.ErrRange {
		if s[0] == '-' {
			return math.MinInt64, err
		}
		return math.MaxInt64, nil
	} else if err != nil {
		return -1, err
	}
	return val, nil
}

// MergeEnv will take an existing environment and merge it with a new set of
// variables. For variables with the same name in both, only the one in the
// new environment will be kept.
func MergeEnv(oldEnv, newEnv []string) []string {
	key := func(e string) string {
		i := strings.Index(e, "=")
		if i == -1 {
			return e
		}
		return e[:i]
	}
	result := []string{}
	newVars := map[string]struct{}{}
	for _, e := range newEnv {
		newVars[key(e)] = struct{}{}
	}
	result = append(result, newEnv...)
	for _, e := range oldEnv {
		if _, exists := newVars[key(e)]; exists {
			continue
		}
		result = append(result, e)
	}
	return result
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package validation

import (
	unversionedvalidation "k8s.io/apimachinery/pkg/apis/meta/v1/validation"
	"k8s.io/apimachinery/pkg/util/intstr"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/apimachinery/pkg/util/validation"
	"k8s.io/apimachinery/pkg/util/validation/field"
	api "k8s.io/kubernetes/pkg/apis/core"
	apivalidation "k8s.io/kubernetes/pkg/apis/core/validation"
	"k8s.io/kubernetes/pkg/apis/networking"
)

// ValidateNetworkPolicyName can be used to check whether the given networkpolicy
// name is valid.
func ValidateNetworkPolicyName(name string, prefix bool) []string {
	return apivalidation.NameIsDNSSubdomain(name, prefix)
}

// ValidateNetworkPolicySpec tests if required fields in the networkpolicy spec are set.
func ValidateNetworkPolicySpec(spec *networking.NetworkPolicySpec, fldPath *field.Path) field.ErrorList {
	allErrs := field.ErrorList{}
	allErrs = append(allErrs, unversionedvalidation.ValidateLabelSelector(&spec.PodSelector, fldPath.Child("podSelector"))...)

	// Validate ingress rules.
	for i, ingress := range spec.Ingress {
		ingressPath := fldPath.Child("ingress").Index(i)
		for i, port := range ingress.Ports {
			portPath := ingressPath.Child("ports").Index(i)
			if port.Protocol != nil && *port.Protocol != api.ProtocolTCP && *port.Protocol != api.ProtocolUDP {
				allErrs = append(allErrs, field.NotSupported(portPath.Child("protocol"), *port.Protocol, []string{string(api.ProtocolTCP), string(api.ProtocolUDP)}))
			}
			if port.Port != nil {
				if port.Port.Type == intstr.Int {
					for _, msg := range validation.IsValidPortNum(int(port.Port.IntVal)) {
						allErrs = append(allErrs, field.Invalid(portPath.Child("port"), port.Port.IntVal, msg))
					}
				} else {
					for _, msg := range validation.IsValidPortName(port.Port.StrVal) {
						allErrs = append(allErrs, field.Invalid(portPath.Child("port"), port.Port.StrVal, msg))
					}
				}
			}
		}
		for i, from := range ingress.From {
			fromPath := ingressPath.Child("from").Index(i)
			numFroms := 0
			if from.PodSelector != nil {
				numFroms++
				allErrs = append(allErrs, unversionedvalidation.ValidateLabelSelector(from.PodSelector, fromPath.Child("podSelector"))...)
			}
			if from.NamespaceSelector != nil {
				numFroms++
				allErrs = append(allErrs, unversionedvalidation.ValidateLabelSelector(from.NamespaceSelector, fromPath.Child("namespaceSelector"))...)
			}
			if from.IPBlock != nil {
				numFroms++
				allErrs = append(allErrs, ValidateIPBlock(from.IPBlock, fromPath.Child("ipBlock"))...)
			}
			if numFroms == 0 {
				allErrs = append(allErrs, field.Required(fromPath, "must specify a from type"))
			} else if numFroms > 1 {
				allErrs = append(allErrs, field.Forbidden(fromPath, "may not specify more than 1 from type"))
			}
		}
	}
	// Validate egress rules
	for i, egress := range spec.Egress {
		egressPath := fldPath.Child("egress").Index(i)
		for i, port := range egress.Ports {
			portPath := egressPath.Child("ports").Index(i)
			if port.Protocol != nil && *port.Protocol != api.ProtocolTCP && *port.Protocol != api.ProtocolUDP {
				allErrs = append(allErrs, field.NotSupported(portPath.Child("protocol"), *port.Protocol, []string{string(api.ProtocolTCP), string(api.ProtocolUDP)}))
			}
			if port.Port != nil {
				if port.Port.Type == intstr.Int {
					for _, msg := range validation.IsValidPortNum(int(port.Port.IntVal)) {
						allErrs = append(allErrs, field.Invalid(portPath.Child("port"), port.Port.IntVal, msg))
					}
				} else {
					for _, msg := range validation.IsValidPortName(port.Port.StrVal) {
						allErrs = append(allErrs, field.Invalid(portPath.Child("port"), port.Port.StrVal, msg))
					}
				}
			}
		}
		for i, to := range egress.To {
			toPath := egressPath.Child("to").Index(i)
			numTo := 0
			if to.PodSelector != nil {
				numTo++
				allErrs = append(allErrs, unversionedvalidation.ValidateLabelSelector(to.PodSelector, toPath.Child("podSelector"))...)
			}
			if to.NamespaceSelector != nil {
				numTo++
				allErrs = append(allErrs, unversionedvalidation.ValidateLabelSelector(to.NamespaceSelector, toPath.Child("namespaceSelector"))...)
			}
			if to.IPBlock != nil {
				numTo++
				allErrs = append(allErrs, ValidateIPBlock(to.IPBlock, toPath.Child("ipBlock"))...)
			}
			if numTo == 0 {
				allErrs = append(allErrs, field.Required(toPath, "must specify a to type"))
			} else if numTo > 1 {
				allErrs = append(allErrs, field.Forbidden(toPath, "may not specify more than 1 to type"))
			}
		}
	}
	// Validate PolicyTypes
	allowed := sets.NewString(string(networking.PolicyTypeIngress), string(networking.PolicyTypeEgress))
	if len(spec.PolicyTypes) > len(allowed) {
		allErrs = append(allErrs, field.Invalid(fldPath.Child("policyTypes"), &spec.PolicyTypes, "may not specify more than two policyTypes"))
		return allErrs
	}
	for i, pType := range spec.PolicyTypes {
		policyPath := fldPath.Child("policyTypes").Index(i)
		for _, p := range spec.PolicyTypes {
			if !allowed.Has(string(p)) {
				allErrs = append(allErrs, field.NotSupported(policyPath, pType, []string{string(networking.PolicyTypeIngress), string(networking.PolicyTypeEgress)}))
			}
		}
	}
	return allErrs
}

// ValidateNetworkPolicy validates a networkpolicy.
func ValidateNetworkPolicy(np *networking.NetworkPolicy) field.ErrorList {
	allErrs := apivalidation.ValidateObjectMeta(&np.ObjectMeta, true, ValidateNetworkPolicyName, field.NewPath("metadata"))
	allErrs = append(allErrs, ValidateNetworkPolicySpec(&np.Spec, field.NewPath("spec"))...)
	return allErrs
}

// ValidateNetworkPolicyUpdate tests if an update to a NetworkPolicy is valid.
func ValidateNetworkPolicyUpdate(update, old *networking.NetworkPolicy) field.ErrorList {
	allErrs := field.ErrorList{}
	allErrs = append(allErrs, apivalidation.ValidateObjectMetaUpdate(&update.ObjectMeta, &old.ObjectMeta, field.NewPath("metadata"))...)
	allErrs = append(allErrs, ValidateNetworkPolicySpec(&update.Spec, field.NewPath("spec"))...)
	return allErrs
}

// ValidateIPBlock validates a cidr and the except fields of an IpBlock NetworkPolicyPeer
func ValidateIPBlock(ipb *networking.IPBlock, fldPath *field.Path) field.ErrorList {
	allErrs := field.ErrorList{}
	if len(ipb.CIDR) == 0 || ipb.CIDR == "" {
		allErrs = append(allErrs, field.Required(fldPath.Child("cidr"), ""))
		return allErrs
	}
	cidrIPNet, err := apivalidation.ValidateCIDR(ipb.CIDR)
	if err != nil {
		allErrs = append(allErrs, field.Invalid(fldPath.Child("cidr"), ipb.CIDR, "not a valid CIDR"))
		return allErrs
	}
	exceptCIDR := ipb.Except
	for i, exceptIP := range exceptCIDR {
		exceptPath := fldPath.Child("except").Index(i)
		exceptCIDR, err := apivalidation.ValidateCIDR(exceptIP)
		if err != nil {
			allErrs = append(allErrs, field.Invalid(exceptPath, exceptIP, "not a valid CIDR"))
			return allErrs
		}
		if !cidrIPNet.Contains(exceptCIDR.IP) {
			allErrs = append(allErrs, field.Invalid(exceptPath, exceptCIDR.IP, "not within CIDR range"))
		}
	}
	return allErrs
}
<commit_msg>Factor out duplicated NetworkPolicy validation code<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package validation

import (
	unversionedvalidation "k8s.io/apimachinery/pkg/apis/meta/v1/validation"
	"k8s.io/apimachinery/pkg/util/intstr"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/apimachinery/pkg/util/validation"
	"k8s.io/apimachinery/pkg/util/validation/field"
	api "k8s.io/kubernetes/pkg/apis/core"
	apivalidation "k8s.io/kubernetes/pkg/apis/core/validation"
	"k8s.io/kubernetes/pkg/apis/networking"
)

// ValidateNetworkPolicyName can be used to check whether the given networkpolicy
// name is valid.
func ValidateNetworkPolicyName(name string, prefix bool) []string {
	return apivalidation.NameIsDNSSubdomain(name, prefix)
}

// ValidateNetworkPolicyPort validates a NetworkPolicyPort
func ValidateNetworkPolicyPort(port *networking.NetworkPolicyPort, portPath *field.Path) field.ErrorList {
	allErrs := field.ErrorList{}

	if port.Protocol != nil && *port.Protocol != api.ProtocolTCP && *port.Protocol != api.ProtocolUDP {
		allErrs = append(allErrs, field.NotSupported(portPath.Child("protocol"), *port.Protocol, []string{string(api.ProtocolTCP), string(api.ProtocolUDP)}))
	}
	if port.Port != nil {
		if port.Port.Type == intstr.Int {
			for _, msg := range validation.IsValidPortNum(int(port.Port.IntVal)) {
				allErrs = append(allErrs, field.Invalid(portPath.Child("port"), port.Port.IntVal, msg))
			}
		} else {
			for _, msg := range validation.IsValidPortName(port.Port.StrVal) {
				allErrs = append(allErrs, field.Invalid(portPath.Child("port"), port.Port.StrVal, msg))
			}
		}
	}

	return allErrs
}

// ValidateNetworkPolicyPeer validates a NetworkPolicyPeer
func ValidateNetworkPolicyPeer(peer *networking.NetworkPolicyPeer, peerPath *field.Path) field.ErrorList {
	allErrs := field.ErrorList{}
	numPeers := 0

	if peer.PodSelector != nil {
		numPeers++
		allErrs = append(allErrs, unversionedvalidation.ValidateLabelSelector(peer.PodSelector, peerPath.Child("podSelector"))...)
	}
	if peer.NamespaceSelector != nil {
		numPeers++
		allErrs = append(allErrs, unversionedvalidation.ValidateLabelSelector(peer.NamespaceSelector, peerPath.Child("namespaceSelector"))...)
	}
	if peer.IPBlock != nil {
		numPeers++
		allErrs = append(allErrs, ValidateIPBlock(peer.IPBlock, peerPath.Child("ipBlock"))...)
	}

	if numPeers == 0 {
		allErrs = append(allErrs, field.Required(peerPath, "must specify a peer"))
	} else if numPeers > 1 {
		allErrs = append(allErrs, field.Forbidden(peerPath, "may not specify more than 1 peer"))
	}

	return allErrs
}

// ValidateNetworkPolicySpec tests if required fields in the networkpolicy spec are set.
func ValidateNetworkPolicySpec(spec *networking.NetworkPolicySpec, fldPath *field.Path) field.ErrorList {
	allErrs := field.ErrorList{}
	allErrs = append(allErrs, unversionedvalidation.ValidateLabelSelector(&spec.PodSelector, fldPath.Child("podSelector"))...)

	// Validate ingress rules.
	for i, ingress := range spec.Ingress {
		ingressPath := fldPath.Child("ingress").Index(i)
		for i, port := range ingress.Ports {
			portPath := ingressPath.Child("ports").Index(i)
			allErrs = append(allErrs, ValidateNetworkPolicyPort(&port, portPath)...)
		}
		for i, from := range ingress.From {
			fromPath := ingressPath.Child("from").Index(i)
			allErrs = append(allErrs, ValidateNetworkPolicyPeer(&from, fromPath)...)
		}
	}
	// Validate egress rules
	for i, egress := range spec.Egress {
		egressPath := fldPath.Child("egress").Index(i)
		for i, port := range egress.Ports {
			portPath := egressPath.Child("ports").Index(i)
			allErrs = append(allErrs, ValidateNetworkPolicyPort(&port, portPath)...)
		}
		for i, to := range egress.To {
			toPath := egressPath.Child("to").Index(i)
			allErrs = append(allErrs, ValidateNetworkPolicyPeer(&to, toPath)...)
		}
	}
	// Validate PolicyTypes
	allowed := sets.NewString(string(networking.PolicyTypeIngress), string(networking.PolicyTypeEgress))
	if len(spec.PolicyTypes) > len(allowed) {
		allErrs = append(allErrs, field.Invalid(fldPath.Child("policyTypes"), &spec.PolicyTypes, "may not specify more than two policyTypes"))
		return allErrs
	}
	for i, pType := range spec.PolicyTypes {
		policyPath := fldPath.Child("policyTypes").Index(i)
		for _, p := range spec.PolicyTypes {
			if !allowed.Has(string(p)) {
				allErrs = append(allErrs, field.NotSupported(policyPath, pType, []string{string(networking.PolicyTypeIngress), string(networking.PolicyTypeEgress)}))
			}
		}
	}
	return allErrs
}

// ValidateNetworkPolicy validates a networkpolicy.
func ValidateNetworkPolicy(np *networking.NetworkPolicy) field.ErrorList {
	allErrs := apivalidation.ValidateObjectMeta(&np.ObjectMeta, true, ValidateNetworkPolicyName, field.NewPath("metadata"))
	allErrs = append(allErrs, ValidateNetworkPolicySpec(&np.Spec, field.NewPath("spec"))...)
	return allErrs
}

// ValidateNetworkPolicyUpdate tests if an update to a NetworkPolicy is valid.
func ValidateNetworkPolicyUpdate(update, old *networking.NetworkPolicy) field.ErrorList {
	allErrs := field.ErrorList{}
	allErrs = append(allErrs, apivalidation.ValidateObjectMetaUpdate(&update.ObjectMeta, &old.ObjectMeta, field.NewPath("metadata"))...)
	allErrs = append(allErrs, ValidateNetworkPolicySpec(&update.Spec, field.NewPath("spec"))...)
	return allErrs
}

// ValidateIPBlock validates a cidr and the except fields of an IpBlock NetworkPolicyPeer
func ValidateIPBlock(ipb *networking.IPBlock, fldPath *field.Path) field.ErrorList {
	allErrs := field.ErrorList{}
	if len(ipb.CIDR) == 0 || ipb.CIDR == "" {
		allErrs = append(allErrs, field.Required(fldPath.Child("cidr"), ""))
		return allErrs
	}
	cidrIPNet, err := apivalidation.ValidateCIDR(ipb.CIDR)
	if err != nil {
		allErrs = append(allErrs, field.Invalid(fldPath.Child("cidr"), ipb.CIDR, "not a valid CIDR"))
		return allErrs
	}
	exceptCIDR := ipb.Except
	for i, exceptIP := range exceptCIDR {
		exceptPath := fldPath.Child("except").Index(i)
		exceptCIDR, err := apivalidation.ValidateCIDR(exceptIP)
		if err != nil {
			allErrs = append(allErrs, field.Invalid(exceptPath, exceptIP, "not a valid CIDR"))
			return allErrs
		}
		if !cidrIPNet.Contains(exceptCIDR.IP) {
			allErrs = append(allErrs, field.Invalid(exceptPath, exceptCIDR.IP, "not within CIDR range"))
		}
	}
	return allErrs
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"testing"

	"github.com/pkg/errors"
)

// buildRetool builds retool in a temporary directory and returns the path to the built binary
func buildRetool() (string, error) {
	dir, err := ioutil.TempDir("", "")
	if err != nil {
		return "", errors.Wrap(err, "unable to create temporary build directory")
	}
	output := filepath.Join(dir, "retool")
	cmd := exec.Command("go", "build", "-o", output, ".")
	_, err = cmd.Output()
	if err != nil {
		return "", errors.Wrap(err, "unable to build retool binary")
	}
	return output, nil
}

func TestRetool(t *testing.T) {
	// These integration tests require more than most go tests: they require a go compiler to build
	// retool, a working version of git to perform retool's operations, and network access to do the
	// git fetches.
	retool, err := buildRetool()
	if err != nil {
		t.Fatal(err)
	}
	defer os.Remove(retool)

	t.Run("cache pollution", func(t *testing.T) {
		dir, err := ioutil.TempDir("", "")
		if err != nil {
			t.Fatalf("unable to make temp dir: %s", err)
		}
		defer os.RemoveAll(dir)

		// This should fail because this version of mockery has an import line that points to uber's
		// internal repo, which can't be reached:
		cmd := exec.Command(retool, "-base-dir", dir, "add",
			"github.com/vektra/mockery/cmd/mockery", "d895b9fcc32730719faaccd7840ad7277c94c2d0",
		)
		cmd.Dir = dir
		_, err = cmd.Output()
		if err == nil {
			t.Fatal("expected error when adding mockery at broken commit d895b9, but got no error")
		}

		// Now, without cleaning the cache, try again on a healthy commit. In
		// ff9a1fda7478ede6250ee3c7e4ce32dc30096236 of retool and earlier, this would still fail because
		// the cache would be polluted with a bad source tree.
		cmd = exec.Command(retool, "-base-dir", dir, "add",
			"github.com/vektra/mockery/cmd/mockery", "origin/master",
		)
		cmd.Dir = dir
		_, err = cmd.Output()
		if err != nil {
			t.Fatalf("expected no error when adding mockery at broken commit d895b9, but got this:\n%s", string(err.(*exec.ExitError).Stderr))
		}
	})

	t.Run("version", func(t *testing.T) {
		dir, err := ioutil.TempDir("", "")
		if err != nil {
			t.Fatalf("unable to make temp dir: %s", err)
		}
		defer os.RemoveAll(dir)

		// Should work even in a directory without tools.json
		cmd := exec.Command(retool, "version")
		cmd.Dir = dir
		out, err := cmd.Output()
		if err != nil {
			t.Fatalf("expected no errors when using retool version, have this:\n%s", string(out))
		}
		if want := fmt.Sprintf("retool %s", version); string(out) != want {
			t.Errorf("have=%q, want=%q", string(out), want)
		}
	})

	t.Run("build", func(t *testing.T) {
		dir, err := ioutil.TempDir("", "")
		if err != nil {
			t.Fatalf("unable to make temp dir: %s", err)
		}
		defer os.RemoveAll(dir)

		cmd := exec.Command(retool, "-base-dir", dir, "add",
			"github.com/twitchtv/retool", "origin/master",
		)
		cmd.Dir = dir
		_, err = cmd.Output()
		if err != nil {
			t.Fatalf("expected no errors when using retool add, have this:\n%s", string(err.(*exec.ExitError).Stderr))
		}

		// Suppose we only have _tools/src available. Does `retool build` work?
		os.RemoveAll(filepath.Join(dir, "_tools", "bin"))
		os.RemoveAll(filepath.Join(dir, "_tools", "pkg"))
		os.RemoveAll(filepath.Join(dir, "_tools", "manifest.json"))

		cmd = exec.Command(retool, "-base-dir", dir, "build")
		cmd.Dir = dir
		_, err = cmd.Output()
		if err != nil {
			t.Fatalf("expected no errors when using retool build, have this:\n%s", string(err.(*exec.ExitError).Stderr))
		}

		// Now the binary should be installed
		_, err = os.Stat(filepath.Join(dir, "_tools", "bin", "retool"))
		if err != nil {
			t.Fatalf("unable to stat _tools/bin/retool after calling retool build: %s", err)
		}
	})

}
<commit_msg>Add failing test case of removed deps<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"testing"

	"github.com/pkg/errors"
)

// buildRetool builds retool in a temporary directory and returns the path to the built binary
func buildRetool() (string, error) {
	dir, err := ioutil.TempDir("", "")
	if err != nil {
		return "", errors.Wrap(err, "unable to create temporary build directory")
	}
	output := filepath.Join(dir, "retool")
	cmd := exec.Command("go", "build", "-o", output, ".")
	_, err = cmd.Output()
	if err != nil {
		return "", errors.Wrap(err, "unable to build retool binary")
	}
	return output, nil
}

func TestRetool(t *testing.T) {
	// These integration tests require more than most go tests: they require a go compiler to build
	// retool, a working version of git to perform retool's operations, and network access to do the
	// git fetches.
	retool, err := buildRetool()
	if err != nil {
		t.Fatal(err)
	}
	defer os.Remove(retool)

	t.Run("cache pollution", func(t *testing.T) {
		dir, err := ioutil.TempDir("", "")
		if err != nil {
			t.Fatalf("unable to make temp dir: %s", err)
		}
		defer os.RemoveAll(dir)

		// This should fail because this version of mockery has an import line that points to uber's
		// internal repo, which can't be reached:
		cmd := exec.Command(retool, "-base-dir", dir, "add",
			"github.com/vektra/mockery/cmd/mockery", "d895b9fcc32730719faaccd7840ad7277c94c2d0",
		)
		cmd.Dir = dir
		_, err = cmd.Output()
		if err == nil {
			t.Fatal("expected error when adding mockery at broken commit d895b9, but got no error")
		}

		// Now, without cleaning the cache, try again on a healthy commit. In
		// ff9a1fda7478ede6250ee3c7e4ce32dc30096236 of retool and earlier, this would still fail because
		// the cache would be polluted with a bad source tree.
		cmd = exec.Command(retool, "-base-dir", dir, "add",
			"github.com/vektra/mockery/cmd/mockery", "origin/master",
		)
		cmd.Dir = dir
		_, err = cmd.Output()
		if err != nil {
			t.Fatalf("expected no error when adding mockery at broken commit d895b9, but got this:\n%s", string(err.(*exec.ExitError).Stderr))
		}
	})

	t.Run("version", func(t *testing.T) {
		dir, err := ioutil.TempDir("", "")
		if err != nil {
			t.Fatalf("unable to make temp dir: %s", err)
		}
		defer os.RemoveAll(dir)

		// Should work even in a directory without tools.json
		cmd := exec.Command(retool, "version")
		cmd.Dir = dir
		out, err := cmd.Output()
		if err != nil {
			t.Fatalf("expected no errors when using retool version, have this:\n%s", string(out))
		}
		if want := fmt.Sprintf("retool %s", version); string(out) != want {
			t.Errorf("have=%q, want=%q", string(out), want)
		}
	})

	t.Run("build", func(t *testing.T) {
		dir, err := ioutil.TempDir("", "")
		if err != nil {
			t.Fatalf("unable to make temp dir: %s", err)
		}
		defer os.RemoveAll(dir)

		cmd := exec.Command(retool, "-base-dir", dir, "add",
			"github.com/twitchtv/retool", "origin/master",
		)
		cmd.Dir = dir
		_, err = cmd.Output()
		if err != nil {
			t.Fatalf("expected no errors when using retool add, have this:\n%s", string(err.(*exec.ExitError).Stderr))
		}

		// Suppose we only have _tools/src available. Does `retool build` work?
		os.RemoveAll(filepath.Join(dir, "_tools", "bin"))
		os.RemoveAll(filepath.Join(dir, "_tools", "pkg"))
		os.RemoveAll(filepath.Join(dir, "_tools", "manifest.json"))

		cmd = exec.Command(retool, "-base-dir", dir, "build")
		cmd.Dir = dir
		_, err = cmd.Output()
		if err != nil {
			t.Fatalf("expected no errors when using retool build, have this:\n%s", string(err.(*exec.ExitError).Stderr))
		}

		// Now the binary should be installed
		_, err = os.Stat(filepath.Join(dir, "_tools", "bin", "retool"))
		if err != nil {
			t.Fatalf("unable to stat _tools/bin/retool after calling retool build: %s", err)
		}
	})

	t.Run("dep_added", func(t *testing.T) {
		dir, err := ioutil.TempDir("", "")
		if err != nil {
			t.Fatalf("unable to make temp dir: %s", err)
		}
		defer os.RemoveAll(dir)

		// Use a package which used to have a dependency (in this case, one on
		// github.com/spenczar/retool_test_lib), but doesn't have that dependency for HEAD of
		// origin/master today.
		cmd := exec.Command(retool, "-base-dir", dir, "add",
			"github.com/spenczar/retool_test_app", "origin/has_dep",
		)
		cmd.Dir = dir
		_, err = cmd.Output()
		if err != nil {
			t.Fatalf("expected no errors when using retool add, have this:\n%s", string(err.(*exec.ExitError).Stderr))
		}
	})
}
<|endoftext|>


<commit_before>package rainforest

import (
	"bytes"
	"crypto/md5"
	"encoding/hex"
	"fmt"
	"io/ioutil"
	"mime"
	"mime/multipart"
	"net/http"
	"path/filepath"
	"strconv"

	"github.com/urfave/cli"
)

// UploadedFile represents a file that has been uploaded to Rainforest
type UploadedFile struct {
	ID        int    `json:"id"`
	Signature string `json:"signature"`
	Digest    string `json:"digest"`
	MimeType  string `json:"mime_type"`
	Size      int64  `json:"size"`
	Name      string `json:"name"`
}

// GetUploadedFiles returns information for all all files uploaded to the
// given test before.
func (c *Client) GetUploadedFiles(fileID int) ([]UploadedFile, error) {
	req, err := c.NewRequest("GET", "tests/"+strconv.Itoa(fileID)+"/files", nil)
	if err != nil {
		return nil, err
	}

	var fileResp []UploadedFile
	_, err = c.Do(req, &fileResp)
	return fileResp, err
}

// AWSFileInfo represents the response when uploading new file data to Rainforest.
// It contains information used to upload data the file to AWS.
type AWSFileInfo struct {
	FileID        int    `json:"file_id"`
	FileSignature string `json:"file_signature"`
	URL           string `json:"aws_url"`
	Key           string `json:"aws_key"`
	AccessID      string `json:"aws_access_id"`
	Policy        string `json:"aws_policy"`
	ACL           string `json:"aws_acl"`
	Signature     string `json:"aws_signature"`
}

// MultipartFormRequest creates a http.Request containing the required body for
// uploading a file to AWS given the values stored in the receiving AWSFileInfo struct.
func (aws *AWSFileInfo) MultipartFormRequest(fileName string, fileContents []byte) (*http.Request, error) {
	var req *http.Request
	fileExt := filepath.Ext(fileName)

	buffer := new(bytes.Buffer)
	writer := multipart.NewWriter(buffer)

	writer.WriteField("key", aws.Key)
	writer.WriteField("AWSAccessKeyId", aws.AccessID)
	writer.WriteField("acl", aws.ACL)
	writer.WriteField("policy", aws.Policy)
	writer.WriteField("signature", aws.Signature)
	writer.WriteField("Content-Type", mime.TypeByExtension(fileExt))

	part, err := writer.CreateFormFile("file", fileName)
	part.Write(fileContents)

	url := aws.URL
	req, err = http.NewRequest("POST", url, buffer)
	if err != nil {
		return req, err
	}

	req.Header.Set("Content-Type", writer.FormDataContentType())
	writer.Close()
	req.ContentLength = int64(buffer.Len())

	return req, nil
}

// OSFileInfo is an abstraction for an object that contains file information.
// os.FileInfo implements OSFileInfo.
type OSFileInfo interface {
	Size() int64
}

// OSFile is an abstraction for a file object that contains a Name and returns
// Stats. os.File implements OSFile.
type OSFile interface {
	Name() string
	Stat() (OSFileInfo, error)
}

// CreateTestFile creates a UploadedFile resource by sending file information to
// Rainforest. This information is used for uploading the actual file to AWS.
func (c *Client) CreateTestFile(testID int, file OSFile, fileContents []byte) (*AWSFileInfo, error) {
	awsFileInfo := &AWSFileInfo{}
	fileName := file.Name()
	fileInfo, err := file.Stat()

	if err != nil {
		return awsFileInfo, err
	}

	md5CheckSum := md5.Sum(fileContents)
	hexDigest := hex.EncodeToString(md5CheckSum[:16])

	body := UploadedFile{
		MimeType: mime.TypeByExtension(filepath.Ext(fileName)),
		Size:     fileInfo.Size(),
		Name:     fileName,
		Digest:   hexDigest,
	}

	url := "tests/" + strconv.Itoa(testID) + "/files"
	req, err := c.NewRequest("POST", url, body)
	if err != nil {
		return awsFileInfo, err
	}

	_, err = c.Do(req, awsFileInfo)
	return awsFileInfo, err
}

// UploadTestFile is a function that uploads the actual file contents to AWS
func (c *Client) UploadTestFile(fileName string, fileContents []byte, awsFileInfo *AWSFileInfo) error {
	req, err := awsFileInfo.MultipartFormRequest(fileName, fileContents)

	var resp *http.Response
	resp, err = c.client.Do(req)

	if err != nil {
		return err
	}

	status := resp.StatusCode
	if status >= 300 {
		var body []byte
		body, err = ioutil.ReadAll(resp.Body)

		if err != nil {
			return err
		}

		errMsg := fmt.Sprintf("There was an error uploading your file - %v: %v", fileName, string(body))
		return cli.NewExitError(errMsg, 1)
	}

	return nil
}
<commit_msg>remove unnecessary wrapper<commit_after>package rainforest

import (
	"bytes"
	"crypto/md5"
	"encoding/hex"
	"fmt"
	"io/ioutil"
	"mime"
	"mime/multipart"
	"net/http"
	"path/filepath"
	"strconv"
)

// UploadedFile represents a file that has been uploaded to Rainforest
type UploadedFile struct {
	ID        int    `json:"id"`
	Signature string `json:"signature"`
	Digest    string `json:"digest"`
	MimeType  string `json:"mime_type"`
	Size      int64  `json:"size"`
	Name      string `json:"name"`
}

// GetUploadedFiles returns information for all all files uploaded to the
// given test before.
func (c *Client) GetUploadedFiles(fileID int) ([]UploadedFile, error) {
	req, err := c.NewRequest("GET", "tests/"+strconv.Itoa(fileID)+"/files", nil)
	if err != nil {
		return nil, err
	}

	var fileResp []UploadedFile
	_, err = c.Do(req, &fileResp)
	return fileResp, err
}

// AWSFileInfo represents the response when uploading new file data to Rainforest.
// It contains information used to upload data the file to AWS.
type AWSFileInfo struct {
	FileID        int    `json:"file_id"`
	FileSignature string `json:"file_signature"`
	URL           string `json:"aws_url"`
	Key           string `json:"aws_key"`
	AccessID      string `json:"aws_access_id"`
	Policy        string `json:"aws_policy"`
	ACL           string `json:"aws_acl"`
	Signature     string `json:"aws_signature"`
}

// MultipartFormRequest creates a http.Request containing the required body for
// uploading a file to AWS given the values stored in the receiving AWSFileInfo struct.
func (aws *AWSFileInfo) MultipartFormRequest(fileName string, fileContents []byte) (*http.Request, error) {
	var req *http.Request
	fileExt := filepath.Ext(fileName)

	buffer := new(bytes.Buffer)
	writer := multipart.NewWriter(buffer)

	writer.WriteField("key", aws.Key)
	writer.WriteField("AWSAccessKeyId", aws.AccessID)
	writer.WriteField("acl", aws.ACL)
	writer.WriteField("policy", aws.Policy)
	writer.WriteField("signature", aws.Signature)
	writer.WriteField("Content-Type", mime.TypeByExtension(fileExt))

	part, err := writer.CreateFormFile("file", fileName)
	part.Write(fileContents)

	url := aws.URL
	req, err = http.NewRequest("POST", url, buffer)
	if err != nil {
		return req, err
	}

	req.Header.Set("Content-Type", writer.FormDataContentType())
	writer.Close()
	req.ContentLength = int64(buffer.Len())

	return req, nil
}

// OSFileInfo is an abstraction for an object that contains file information.
// os.FileInfo implements OSFileInfo.
type OSFileInfo interface {
	Size() int64
}

// OSFile is an abstraction for a file object that contains a Name and returns
// Stats. os.File implements OSFile.
type OSFile interface {
	Name() string
	Stat() (OSFileInfo, error)
}

// CreateTestFile creates a UploadedFile resource by sending file information to
// Rainforest. This information is used for uploading the actual file to AWS.
func (c *Client) CreateTestFile(testID int, file OSFile, fileContents []byte) (*AWSFileInfo, error) {
	awsFileInfo := &AWSFileInfo{}
	fileName := file.Name()
	fileInfo, err := file.Stat()

	if err != nil {
		return awsFileInfo, err
	}

	md5CheckSum := md5.Sum(fileContents)
	hexDigest := hex.EncodeToString(md5CheckSum[:16])

	body := UploadedFile{
		MimeType: mime.TypeByExtension(filepath.Ext(fileName)),
		Size:     fileInfo.Size(),
		Name:     fileName,
		Digest:   hexDigest,
	}

	url := "tests/" + strconv.Itoa(testID) + "/files"
	req, err := c.NewRequest("POST", url, body)
	if err != nil {
		return awsFileInfo, err
	}

	_, err = c.Do(req, awsFileInfo)
	return awsFileInfo, err
}

// UploadTestFile is a function that uploads the actual file contents to AWS
func (c *Client) UploadTestFile(fileName string, fileContents []byte, awsFileInfo *AWSFileInfo) error {
	req, err := awsFileInfo.MultipartFormRequest(fileName, fileContents)

	var resp *http.Response
	resp, err = c.client.Do(req)

	if err != nil {
		return err
	}

	status := resp.StatusCode
	if status >= 300 {
		var body []byte
		body, err = ioutil.ReadAll(resp.Body)

		if err != nil {
			return err
		}

		return fmt.Errorf("There was an error uploading your file - %v: %v", fileName, string(body))
	}

	return nil
}
<|endoftext|>


<commit_before>package transparent

import (
	"math/rand"
	"os"
	"testing"
	"time"

	lru "github.com/hashicorp/golang-lru"
)

func TestMain(m *testing.M) {
	MyInit()
	retCode := m.Run()
	os.Exit(retCode)
	MyTeardown()
}

// Define dummy source
type dummySource struct {
	list map[int]string
}

func (d dummySource) Get(k interface{}) (interface{}, bool) {
	time.Sleep(5 * time.Millisecond)

	return d.list[k.(int)], true
}
func (d dummySource) Add(k, v interface{}) bool {
	time.Sleep(5 * time.Millisecond)
	d.list[k.(int)] = v.(string)
	return true
}

var d dummySource
var c Cache
var tiered Cache

func MyInit() {
	rand.Seed(time.Now().UnixNano())
	d = dummySource{}
	d.list = make(map[int]string, 0)
	c = Cache{
		cache: d,
		next:  nil,
	}

	lru, err := lru.New(10)
	if err != nil {
		panic("LRU error")
	}
	tiered = Cache{
		cache: lru,
		next:  &c,
	}
	c.Initialize(1000)
	tiered.Initialize(1000)
}

func MyTeardown() {
	c.Finalize()
	tiered.Finalize()
}

// Simple Set and Get
func TestCache(t *testing.T) {
	c.SetWriteBack(100, "test")
	value := c.Get(100)
	if value != "test" {
		t.Error(value)
	}
}

// Tiered, Set and Get
func TestTieredCache(t *testing.T) {
	value := tiered.Get(100)
	if value != "test" {
		t.Error(value)
	}
	tiered.SetWriteThrough(100, "test")

	value = tiered.Get(100)
	if value != "test" {
		t.Error(value)
	}
}

func BenchmarkCacheGet(b *testing.B) {
	for i := 0; i < b.N; i++ {
		r := rand.Intn(5)
		c.Get(r)
	}
}

func BenchmarkCacheSetWriteBack(b *testing.B) {
	for i := 0; i < b.N; i++ {
		r := rand.Intn(5)
		c.SetWriteBack(r, "benchmarking")
	}
}

func BenchmarkCacheSetWriteThrough(b *testing.B) {
	for i := 0; i < b.N; i++ {
		r := rand.Intn(5)
		c.SetWriteThrough(r, "benchmarking")
	}
}

// Tiered
func BenchmarkTieredCacheGet(b *testing.B) {
	for i := 0; i < b.N; i++ {
		r := rand.Intn(5)
		tiered.Get(r)
	}
}

func BenchmarkTieredCacheSetWriteBack(b *testing.B) {
	for i := 0; i < 100; i++ {
		r := rand.Intn(5)
		tiered.SetWriteBack(r, "benchmarking")
	}
}

func BenchmarkTieredCacheSetWriteThrough(b *testing.B) {
	for i := 0; i < b.N; i++ {
		r := rand.Intn(5)
		tiered.SetWriteThrough(r, "benchmarking")
	}
}
<commit_msg>Fix test<commit_after>package transparent

import (
	"math/rand"
	"os"
	"testing"
	"time"

	lru "github.com/hashicorp/golang-lru"
)

func TestMain(m *testing.M) {
	MyInit()
	retCode := m.Run()
	MyTeardown()
	os.Exit(retCode)
}

// Define dummy source
type dummySource struct {
	list map[int]string
}

func (d dummySource) Get(k interface{}) (interface{}, bool) {
	time.Sleep(5 * time.Millisecond)

	return d.list[k.(int)], true
}
func (d dummySource) Add(k, v interface{}) bool {
	time.Sleep(5 * time.Millisecond)
	d.list[k.(int)] = v.(string)
	return true
}

var d dummySource
var c Cache
var tiered Cache

func MyInit() {
	rand.Seed(time.Now().UnixNano())
	d = dummySource{}
	d.list = make(map[int]string, 0)
	c = Cache{
		cache: d,
		next:  nil,
	}

	lru, err := lru.New(10)
	if err != nil {
		panic("LRU error")
	}
	tiered = Cache{
		cache: lru,
		next:  &c,
	}
	c.Initialize(300)
	tiered.Initialize(300)
}

func MyTeardown() {
	c.Finalize()
	tiered.Finalize()
}

// Simple Set and Get
func TestCache(t *testing.T) {
	c.SetWriteBack(100, "test")
	value := c.Get(100)
	if value != "test" {
		t.Error(value)
	}
}

// Tiered, Set and Get
func TestTieredCache(t *testing.T) {
	value := tiered.Get(100)
	if value != "test" {
		t.Error(value)
	}
	tiered.SetWriteThrough(100, "test")

	value = tiered.Get(100)
	if value != "test" {
		t.Error(value)
	}
}

func BenchmarkCacheGet(b *testing.B) {
	for i := 0; i < b.N; i++ {
		r := rand.Intn(5)
		c.Get(r)
	}
}

func BenchmarkCacheSetWriteBack(b *testing.B) {
	for i := 0; i < b.N; i++ {
		r := rand.Intn(5)
		c.SetWriteBack(r, "benchmarking")
	}
}

func BenchmarkCacheSetWriteThrough(b *testing.B) {
	for i := 0; i < b.N; i++ {
		r := rand.Intn(5)
		c.SetWriteThrough(r, "benchmarking")
	}
}

// Tiered
func BenchmarkTieredCacheGet(b *testing.B) {
	for i := 0; i < b.N; i++ {
		r := rand.Intn(5)
		tiered.Get(r)
	}
}

func BenchmarkTieredCacheSetWriteBack(b *testing.B) {
	for i := 0; i < 100; i++ {
		r := rand.Intn(5)
		tiered.SetWriteBack(r, "benchmarking")
	}
}

func BenchmarkTieredCacheSetWriteThrough(b *testing.B) {
	for i := 0; i < b.N; i++ {
		r := rand.Intn(5)
		tiered.SetWriteThrough(r, "benchmarking")
	}
}
<|endoftext|>


<commit_before>// Copyright (C) 2017, 2018 Damon Revoe. All rights reserved.
// Use of this source code is governed by the MIT
// license, which can be found in the LICENSE file.

package main

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"gopkg.in/yaml.v2"
)

var packageDefinitionFilename = appName + ".yaml"

type packageDefinition struct {
	PackageName string
	description string
	packageType string
	pathname    string
	required    packageDefinitionList
	allRequired packageDefinitionList
	dependent   packageDefinitionList
	params      templateParams
}

func getRequiredField(pathname string, params templateParams,
	fieldName string) (interface{}, error) {
	if value := params[fieldName]; value != nil {
		return value, nil
	}
	return nil, errors.New(pathname +
		": missing required field '" + fieldName + "'")
}

func getRequiredStringField(pathname string, params templateParams,
	fieldName string) (string, error) {
	if value, err := getRequiredField(pathname,
		params, fieldName); err != nil {
		return "", err
	} else if stringValue, ok := value.(string); ok {
		return stringValue, nil
	} else {
		return "", errors.New(pathname +
			": '" + fieldName + "' field must be a string")
	}
}

func loadPackageDefinition(pathname string) (*packageDefinition, []string,
	error) {
	data, err := ioutil.ReadFile(pathname)
	if err != nil {
		return nil, nil, err
	}

	var params templateParams

	if err = yaml.Unmarshal(data, &params); err != nil {
		errMessage := strings.TrimPrefix(err.Error(), "yaml: ")
		err = errors.New(pathname + ": " + errMessage)
		return nil, nil, err
	}

	packageName, err := getRequiredStringField(pathname, params, "name")
	if err != nil {
		return nil, nil, err
	}

	description, err := getRequiredStringField(pathname, params,
		"description")
	if err != nil {
		return nil, nil, err
	}

	packageType, err := getRequiredStringField(pathname, params, "type")
	if err != nil {
		return nil, nil, err
	}

	_, err = getRequiredStringField(pathname, params, "version")
	if err != nil {
		return nil, nil, err
	}

	requires := []string{}

	if requiredPackages := params["requires"]; requiredPackages != nil {
		pkgList, ok := requiredPackages.([]interface{})
		if !ok {
			return nil, nil, errors.New(pathname +
				": 'requires' must be a list")
		}
		for _, pkgName := range pkgList {
			pkgNameStr, ok := pkgName.(string)
			if !ok {
				return nil, nil, errors.New(pathname +
					": 'requires' must be " +
					"a list of strings")
			}
			requires = append(requires, pkgNameStr)
		}
	}

	return &packageDefinition{
		packageName,
		description,
		packageType,
		pathname,
		/*required*/ packageDefinitionList{},
		/*allRequired*/ packageDefinitionList{},
		/*dependent*/ packageDefinitionList{},
		params}, requires, nil
}

type packageDefinitionList []*packageDefinition

type packageIndex struct {
	packageByName   map[string]*packageDefinition
	orderedPackages packageDefinitionList
}

func (pi *packageIndex) getPackageByName(pkgName string) (
	*packageDefinition, error) {
	if pd := pi.packageByName[pkgName]; pd != nil {
		return pd, nil
	}
	return nil, errors.New("no such package: " + pkgName)
}

func getPackagePathFromEnvironment() (string, error) {
	if pkgpath := flags.pkgPath; pkgpath != "" {
		return pkgpath, nil
	}

	if pkgpath := os.Getenv(pkgPathEnvVar); pkgpath != "" {
		return pkgpath, nil
	}

	return "", errors.New("--pkgpath is not given and $" +
		pkgPathEnvVar + " is not defined")
}

func getPackagePathFromWorkspaceOrEnvironment(
	wp *workspaceParams) (string, error) {

	if wp.PkgPath != "" {
		return wp.PkgPath, nil
	}

	return getPackagePathFromEnvironment()
}

func readPackageDefinitions(workspaceDir string, wp *workspaceParams) (
	*packageIndex, error) {
	var packages packageDefinitionList
	dependencies := [][]string{}

	pkgpath, err := getPackagePathFromWorkspaceOrEnvironment(wp)
	if err != nil {
		return nil, err
	}

	pkgpathDirs := append(strings.Split(pkgpath, ":"),
		filepath.Join(filepath.Dir(os.Args[0]), "templates"))

	for _, pkgpathDir := range pkgpathDirs {
		dirEntries, _ := ioutil.ReadDir(pkgpathDir)

		for _, dirEntry := range dirEntries {
			dirEntryPathname := filepath.Join(pkgpathDir,
				dirEntry.Name(), packageDefinitionFilename)

			fileInfo, err := os.Stat(dirEntryPathname)
			if err != nil || !fileInfo.Mode().IsRegular() {
				continue
			}

			pd, requires, err := loadPackageDefinition(
				dirEntryPathname)
			if err != nil {
				return nil, err
			}

			packages = append(packages, pd)
			dependencies = append(dependencies, requires)
		}
	}

	return buildPackageIndex(packages, dependencies)
}

type topologicalSorter struct {
	visited         map[*packageDefinition]int
	orderedPackages packageDefinitionList
}

const (
	unvisited = iota
	beingVisited
	visited
)

// cycle returns a string representing the cycle that
// has been detected in visit()
func (ts *topologicalSorter) cycle(pd, endp *packageDefinition) string {
	for _, dep := range pd.required {
		if ts.visited[dep] == beingVisited {
			if dep == endp {
				return pd.PackageName + " -> " +
					endp.PackageName
			}
			if cycle := ts.cycle(dep, endp); cycle != "" {
				return pd.PackageName + " -> " + cycle
			}
		}
	}
	return ""
}

func (ts *topologicalSorter) visit(pd *packageDefinition) error {
	switch ts.visited[pd] {
	case unvisited:
		ts.visited[pd] = beingVisited
		for _, dep := range pd.required {
			err := ts.visit(dep)
			if err != nil {
				return err
			}
		}
		ts.visited[pd] = visited
		ts.orderedPackages = append(ts.orderedPackages, pd)
	case beingVisited:
		return errors.New("circular dependency detected: " +
			ts.cycle(pd, pd))
	}
	return nil
}

// topologicalSort sorts the given package list using an algorithm based
// on depth-first search. Packages in the returned list are ordered so that
// all dependent packages come after the packages they depend on.
func topologicalSort(packages packageDefinitionList) (packageDefinitionList,
	error) {
	ts := topologicalSorter{make(map[*packageDefinition]int),
		packageDefinitionList{}}

	for _, pd := range packages {
		if ts.visited[pd] == unvisited {
			if err := ts.visit(pd); err != nil {
				return nil, err
			}
		}
	}

	return ts.orderedPackages, nil
}

// buildPackageIndex creates two types of structures for the
// input list of packages:
// 1. A map from package names to their definitions, and
// 2. A list of packages that contains a topological ordering
//    of the package dependency DAG.
func buildPackageIndex(packages packageDefinitionList,
	dependencies [][]string) (*packageIndex, error) {
	pi := &packageIndex{make(map[string]*packageDefinition),
		packageDefinitionList{}}

	// Create the packageByName index.
	for _, pd := range packages {
		// Having two different packages with the same name
		// is not allowed.
		if dup, ok := pi.packageByName[pd.PackageName]; ok {
			return nil, errors.New("duplicate package name: " +
				pd.PackageName + " (from " + pd.pathname +
				"); previously declared in " + dup.pathname)
		}
		pi.packageByName[pd.PackageName] = pd
	}

	// Resolve dependencies and establish the edges of the
	// reverse dependency DAG.
	for i, pd := range packages {
		for _, dep := range dependencies[i] {
			depp := pi.packageByName[dep]
			if depp == nil {
				return nil, errors.New("package " +
					pd.PackageName + " requires " +
					dep + ", which is not " +
					"available in the search path")
			}
			pd.required = append(pd.required, depp)
			depp.dependent = append(depp.dependent, pd)
		}
	}

	// Apply topological sorting to the dependency DAG so that
	// no package comes before the packages it depends on.
	var err error
	pi.orderedPackages, err = topologicalSort(packages)
	if err != nil {
		return nil, err
	}

	// For each package, find all of its dependencies,
	// including indirect ones.
	for _, pd := range pi.orderedPackages {
		added := make(map[*packageDefinition]bool)

		addDep := func(dep *packageDefinition) {
			if !added[dep] {
				pd.allRequired = append(pd.allRequired, dep)
				added[dep] = true
			}
		}

		// Recursion is not needed because the packages
		// are already ordered in such a way that the current
		// package never depends on those that follow it.
		for _, required := range pd.required {
			for _, dep := range required.allRequired {
				addDep(dep)
			}
			addDep(required)
		}
	}

	return pi, nil
}

func packageNames(pkgList packageDefinitionList) string {
	names := []string{}
	for _, pd := range pkgList {
		names = append(names, pd.PackageName)
	}
	return strings.Join(names, ", ")
}

func printListOfPackages(pkgList packageDefinitionList) {
	fmt.Println("List of packages:")

	for _, pd := range pkgList {
		fmt.Println("Name:", pd.PackageName)
		fmt.Println("Description:", pd.description)
		fmt.Println("Type:", pd.packageType)
		if len(pd.required) > 0 {
			fmt.Println("Requires:", packageNames(pd.required))
		}
		fmt.Println()
	}
}
<commit_msg>Detect and report redundant dependencies<commit_after>// Copyright (C) 2017, 2018 Damon Revoe. All rights reserved.
// Use of this source code is governed by the MIT
// license, which can be found in the LICENSE file.

package main

import (
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"

	"gopkg.in/yaml.v2"
)

var packageDefinitionFilename = appName + ".yaml"

type packageDefinition struct {
	PackageName  string
	description  string
	packageType  string
	pathname     string
	required     packageDefinitionList // Explicitly required packages
	allRequired  packageDefinitionList // Required + indirectly required
	uniqRequired packageDefinitionList // 'required' sans indirect reqs
	dependent    packageDefinitionList // Packages that depend on this one
	params       templateParams
}

type packageDefinitionList []*packageDefinition

func getRequiredField(pathname string, params templateParams,
	fieldName string) (interface{}, error) {
	if value := params[fieldName]; value != nil {
		return value, nil
	}
	return nil, errors.New(pathname +
		": missing required field '" + fieldName + "'")
}

func getRequiredStringField(pathname string, params templateParams,
	fieldName string) (string, error) {
	if value, err := getRequiredField(pathname,
		params, fieldName); err != nil {
		return "", err
	} else if stringValue, ok := value.(string); ok {
		return stringValue, nil
	} else {
		return "", errors.New(pathname +
			": '" + fieldName + "' field must be a string")
	}
}

func loadPackageDefinition(pathname string) (*packageDefinition, []string,
	error) {
	data, err := ioutil.ReadFile(pathname)
	if err != nil {
		return nil, nil, err
	}

	var params templateParams

	if err = yaml.Unmarshal(data, &params); err != nil {
		errMessage := strings.TrimPrefix(err.Error(), "yaml: ")
		err = errors.New(pathname + ": " + errMessage)
		return nil, nil, err
	}

	packageName, err := getRequiredStringField(pathname, params, "name")
	if err != nil {
		return nil, nil, err
	}

	description, err := getRequiredStringField(pathname, params,
		"description")
	if err != nil {
		return nil, nil, err
	}

	packageType, err := getRequiredStringField(pathname, params, "type")
	if err != nil {
		return nil, nil, err
	}

	_, err = getRequiredStringField(pathname, params, "version")
	if err != nil {
		return nil, nil, err
	}

	requires := []string{}

	if requiredPackages := params["requires"]; requiredPackages != nil {
		pkgList, ok := requiredPackages.([]interface{})
		if !ok {
			return nil, nil, errors.New(pathname +
				": 'requires' must be a list")
		}
		for _, pkgName := range pkgList {
			pkgNameStr, ok := pkgName.(string)
			if !ok {
				return nil, nil, errors.New(pathname +
					": 'requires' must be " +
					"a list of strings")
			}
			requires = append(requires, pkgNameStr)
		}
	}

	return &packageDefinition{
		packageName,
		description,
		packageType,
		pathname,
		/*required*/ packageDefinitionList{},
		/*allRequired*/ packageDefinitionList{},
		/*uniqRequired*/ packageDefinitionList{},
		/*dependent*/ packageDefinitionList{},
		params}, requires, nil
}

type packageIndex struct {
	packageByName   map[string]*packageDefinition
	orderedPackages packageDefinitionList
}

func (pi *packageIndex) getPackageByName(pkgName string) (
	*packageDefinition, error) {
	if pd := pi.packageByName[pkgName]; pd != nil {
		return pd, nil
	}
	return nil, errors.New("no such package: " + pkgName)
}

func getPackagePathFromEnvironment() (string, error) {
	if pkgpath := flags.pkgPath; pkgpath != "" {
		return pkgpath, nil
	}

	if pkgpath := os.Getenv(pkgPathEnvVar); pkgpath != "" {
		return pkgpath, nil
	}

	return "", errors.New("--pkgpath is not given and $" +
		pkgPathEnvVar + " is not defined")
}

func getPackagePathFromWorkspaceOrEnvironment(
	wp *workspaceParams) (string, error) {

	if wp.PkgPath != "" {
		return wp.PkgPath, nil
	}

	return getPackagePathFromEnvironment()
}

func readPackageDefinitions(workspaceDir string, wp *workspaceParams) (
	*packageIndex, error) {
	var packages packageDefinitionList
	dependencies := [][]string{}

	pkgpath, err := getPackagePathFromWorkspaceOrEnvironment(wp)
	if err != nil {
		return nil, err
	}

	pkgpathDirs := append(strings.Split(pkgpath, ":"),
		filepath.Join(filepath.Dir(os.Args[0]), "templates"))

	for _, pkgpathDir := range pkgpathDirs {
		dirEntries, _ := ioutil.ReadDir(pkgpathDir)

		for _, dirEntry := range dirEntries {
			dirEntryPathname := filepath.Join(pkgpathDir,
				dirEntry.Name(), packageDefinitionFilename)

			fileInfo, err := os.Stat(dirEntryPathname)
			if err != nil || !fileInfo.Mode().IsRegular() {
				continue
			}

			pd, requires, err := loadPackageDefinition(
				dirEntryPathname)
			if err != nil {
				return nil, err
			}

			packages = append(packages, pd)
			dependencies = append(dependencies, requires)
		}
	}

	return buildPackageIndex(packages, dependencies)
}

type topologicalSorter struct {
	visited         map[*packageDefinition]int
	orderedPackages packageDefinitionList
}

const (
	unvisited = iota
	beingVisited
	visited
)

// cycle returns a string representing the cycle that
// has been detected in visit()
func (ts *topologicalSorter) cycle(pd, endp *packageDefinition) string {
	for _, dep := range pd.required {
		if ts.visited[dep] == beingVisited {
			if dep == endp {
				return pd.PackageName + " -> " +
					endp.PackageName
			}
			if cycle := ts.cycle(dep, endp); cycle != "" {
				return pd.PackageName + " -> " + cycle
			}
		}
	}
	return ""
}

func (ts *topologicalSorter) visit(pd *packageDefinition) error {
	switch ts.visited[pd] {
	case unvisited:
		ts.visited[pd] = beingVisited
		for _, dep := range pd.required {
			err := ts.visit(dep)
			if err != nil {
				return err
			}
		}
		ts.visited[pd] = visited
		ts.orderedPackages = append(ts.orderedPackages, pd)
	case beingVisited:
		return errors.New("circular dependency detected: " +
			ts.cycle(pd, pd))
	}
	return nil
}

// topologicalSort sorts the given package list using an algorithm based
// on depth-first search. Packages in the returned list are ordered so that
// all dependent packages come after the packages they depend on.
func topologicalSort(packages packageDefinitionList) (packageDefinitionList,
	error) {
	ts := topologicalSorter{make(map[*packageDefinition]int),
		packageDefinitionList{}}

	for _, pd := range packages {
		if ts.visited[pd] == unvisited {
			if err := ts.visit(pd); err != nil {
				return nil, err
			}
		}
	}

	return ts.orderedPackages, nil
}

// buildPackageIndex creates two types of structures for the
// input list of packages:
// 1. A map from package names to their definitions, and
// 2. A list of packages that contains a topological ordering
//    of the package dependency DAG.
func buildPackageIndex(packages packageDefinitionList,
	dependencies [][]string) (*packageIndex, error) {
	pi := &packageIndex{make(map[string]*packageDefinition),
		packageDefinitionList{}}

	// Create the packageByName index.
	for _, pd := range packages {
		// Having two different packages with the same name
		// is not allowed.
		if dup, ok := pi.packageByName[pd.PackageName]; ok {
			return nil, errors.New("duplicate package name: " +
				pd.PackageName + " (from " + pd.pathname +
				"); previously declared in " + dup.pathname)
		}
		pi.packageByName[pd.PackageName] = pd
	}

	// Resolve dependencies and establish the edges of the
	// reverse dependency DAG.
	for i, pd := range packages {
		for _, dep := range dependencies[i] {
			depp := pi.packageByName[dep]
			if depp == nil {
				return nil, errors.New("package " +
					pd.PackageName + " requires " +
					dep + ", which is not " +
					"available in the search path")
			}
			pd.required = append(pd.required, depp)
			depp.dependent = append(depp.dependent, pd)
		}
	}

	// Apply topological sorting to the dependency DAG so that
	// no package comes before the packages it depends on.
	var err error
	pi.orderedPackages, err = topologicalSort(packages)
	if err != nil {
		return nil, err
	}

	// For each package, find all of its dependencies,
	// including indirect ones. This computes the transitive
	// closure of the dependency DAG. Additionally, the second
	// nested loop also computes the transitive reduction
	// of the DAG
	for _, pd := range pi.orderedPackages {
		allRequired := make(map[*packageDefinition]bool)

		// Recursion is not needed because the packages
		// are already ordered in such a way that the current
		// package never depends on those that follow it.
		for _, required := range pd.required {
			for _, dep := range required.allRequired {
				if !allRequired[dep] {
					pd.allRequired = append(
						pd.allRequired, dep)
					allRequired[dep] = true
				}
			}
		}

		// This loop cannot be merged with the previous one.
		// All indirect dependencies must be collected before
		// checking direct dependencies for redundancy.
		for _, required := range pd.required {
			if !allRequired[required] {
				pd.allRequired = append(
					pd.allRequired, required)
				allRequired[required] = true

				// Update the list of dependencies
				// exclusive to the current package.
				pd.uniqRequired = append(
					pd.uniqRequired, required)
			} else if !flags.quiet {
				log.Printf("%s: redundant dependency on %s\n",
					pd.PackageName, required.PackageName)
			}
		}
	}

	return pi, nil
}

func packageNames(pkgList packageDefinitionList) string {
	names := []string{}
	for _, pd := range pkgList {
		names = append(names, pd.PackageName)
	}
	return strings.Join(names, ", ")
}

func printListOfPackages(pkgList packageDefinitionList) {
	fmt.Println("List of packages:")

	for _, pd := range pkgList {
		fmt.Println("Name:", pd.PackageName)
		fmt.Println("Description:", pd.description)
		fmt.Println("Type:", pd.packageType)
		if len(pd.required) > 0 {
			fmt.Println("Requires:", packageNames(pd.required))
		}
		fmt.Println()
	}
}
<|endoftext|>


<commit_before>package lnwallet

import (
	"fmt"

	"github.com/roasbeef/btcd/txscript"
	"github.com/roasbeef/btcd/wire"
)

// WitnessType determines how an output's witness will be generated. The
// default commitmentTimeLock type will generate a witness that will allow
// spending of a time-locked transaction enforced by CheckSequenceVerify.
type WitnessType uint16

const (
	// CommitmentTimeLock is a witness that allows us to spend the output of a
	// commitment transaction after a relative lock-time lockout.
	CommitmentTimeLock WitnessType = 0

	// CommitmentNoDelay is a witness that allows us to spend a settled no-delay
	// output immediately on a counterparty's commitment transaction.
	CommitmentNoDelay WitnessType = 1

	// CommitmentRevoke is a witness that allows us to sweep the settled output
	// of a malicious counterparty's who broadcasts a revoked commitment
	// transaction.
	CommitmentRevoke WitnessType = 2
)

// WitnessGenerator represents a function which is able to generate the final
// witness for a particular public key script. This function acts as an
// abstraction layer, hiding the details of the underlying script.
type WitnessGenerator func(tx *wire.MsgTx, hc *txscript.TxSigHashes,
	inputIndex int) ([][]byte, error)

// GenWitnessFunc will return a WitnessGenerator function that an output
// uses to generate the witness for a sweep transaction.
func (wt WitnessType) GenWitnessFunc(signer *Signer,
	descriptor *SignDescriptor) WitnessGenerator {

	return func(tx *wire.MsgTx, hc *txscript.TxSigHashes,
		inputIndex int) ([][]byte, error) {

		desc := descriptor
		desc.SigHashes = hc
		desc.InputIndex = inputIndex

		switch wt {
		case CommitmentTimeLock:
			return CommitSpendTimeout(*signer, desc, tx)
		case CommitmentNoDelay:
			return CommitSpendNoDelay(*signer, desc, tx)
		case CommitmentRevoke:
			return CommitSpendRevoke(*signer, desc, tx)
		default:
			return nil, fmt.Errorf("unknown witness type: %v", wt)
		}
	}

}
<commit_msg>lnwallet/witnessgen: remove use of pointer to Signer iface<commit_after>package lnwallet

import (
	"fmt"

	"github.com/roasbeef/btcd/txscript"
	"github.com/roasbeef/btcd/wire"
)

// WitnessType determines how an output's witness will be generated. The
// default commitmentTimeLock type will generate a witness that will allow
// spending of a time-locked transaction enforced by CheckSequenceVerify.
type WitnessType uint16

const (
	// CommitmentTimeLock is a witness that allows us to spend the output of a
	// commitment transaction after a relative lock-time lockout.
	CommitmentTimeLock WitnessType = 0

	// CommitmentNoDelay is a witness that allows us to spend a settled no-delay
	// output immediately on a counterparty's commitment transaction.
	CommitmentNoDelay WitnessType = 1

	// CommitmentRevoke is a witness that allows us to sweep the settled output
	// of a malicious counterparty's who broadcasts a revoked commitment
	// transaction.
	CommitmentRevoke WitnessType = 2
)

// WitnessGenerator represents a function which is able to generate the final
// witness for a particular public key script. This function acts as an
// abstraction layer, hiding the details of the underlying script.
type WitnessGenerator func(tx *wire.MsgTx, hc *txscript.TxSigHashes,
	inputIndex int) ([][]byte, error)

// GenWitnessFunc will return a WitnessGenerator function that an output
// uses to generate the witness for a sweep transaction.
func (wt WitnessType) GenWitnessFunc(signer Signer,
	descriptor *SignDescriptor) WitnessGenerator {

	return func(tx *wire.MsgTx, hc *txscript.TxSigHashes,
		inputIndex int) ([][]byte, error) {

		desc := descriptor
		desc.SigHashes = hc
		desc.InputIndex = inputIndex

		switch wt {
		case CommitmentTimeLock:
			return CommitSpendTimeout(signer, desc, tx)
		case CommitmentNoDelay:
			return CommitSpendNoDelay(signer, desc, tx)
		case CommitmentRevoke:
			return CommitSpendRevoke(signer, desc, tx)
		default:
			return nil, fmt.Errorf("unknown witness type: %v", wt)
		}
	}

}
<|endoftext|>