<commit_before>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package naming

import (
	"bytes"
	"crypto/rand"
	"encoding/binary"
	"encoding/hex"
	"errors"
	"io"
)

var (
	ErrInvalidString = errors.New("string is of the wrong format and/or size")
)

// RoutingIDs have one essential property, namely that they are, to a very
// high probability globally unique. Global uniqueness is required in order
// to support comparing Endpoints for equality; this is required for sharing
// connections, for proxying (though global uniqueness is not strictly
// required) and determining if different names resolve to the same endpoint.
type RoutingID struct {
	value [routingIDLength]byte
}

const (
	routingIDLength          = 16
	firstUnreservedRoutingID = 1024
)

var (
	// NullRoutingID is a special value representing the nil route.
	NullRoutingID = FixedRoutingID(0)

	errNotARoutingID = errors.New("Not a RoutingID")
)

// FixedRoutingID returns a routing ID from a constant.
func FixedRoutingID(i uint64) RoutingID {
	var rid RoutingID
	binary.BigEndian.PutUint64(rid.value[8:16], i)
	return rid
}

// IsReserved() returns true iff the RoutingID is in the reserved range.
func (rid RoutingID) IsReserved() bool {
	return isZero(rid.value[0:14]) && isLessThan(rid.value[15:16], firstUnreservedRoutingID)
}

func isZero(buf []byte) bool {
	for _, b := range buf {
		if b != 0 {
			return false
		}
	}
	return true
}

func isLessThan(buf []byte, j uint16) bool {
	return binary.BigEndian.Uint16(buf) < j
}

// String returns a print representation of the RoutingID.
func (rid RoutingID) String() string {
	return hex.EncodeToString(rid.value[:])
}

// FromString reads an RoutingID from a hex encoded string. If the argument
// string is of zero length the RoutingID will be set to NullRoutingID
func (rid *RoutingID) FromString(s string) error {
	if len(s) == 0 {
		*rid = NullRoutingID
		return nil
	}
	b, err := hex.DecodeString(s)
	if err != nil {
		return err
	}
	if len(b) != routingIDLength {
		return ErrInvalidString
	}
	copy(rid.value[:], b)
	return nil
}

// Read a RoutingID from an io.Reader.
func ReadRoutingID(reader io.Reader) (RoutingID, error) {
	var rid RoutingID
	_, err := io.ReadFull(reader, rid.value[:])
	return rid, err
}

// Write a RoutingID to an io.Writer.
func (rid RoutingID) Write(writer io.Writer) error {
	_, err := writer.Write(rid.value[:])
	return err
}

func NewRoutingID() (RoutingID, error) {
	var rid RoutingID
	for {
		_, err := io.ReadFull(rand.Reader, rid.value[:])
		if err != nil {
			return NullRoutingID, err
		}
		if !rid.IsReserved() {
			return rid, nil
		}
	}
	return rid, nil
}

func Compare(a, b RoutingID) bool {
	return bytes.Compare(a.value[:], b.value[:]) == 0
}

// Implement EndpointOpt so that RoutingID can be passed as an optional
// argument to FormatEndpoint
func (RoutingID) EndpointOpt() {}
<commit_msg>v.io/v23/naming: Convert to verror<commit_after>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package naming

import (
	"bytes"
	"crypto/rand"
	"encoding/binary"
	"encoding/hex"
	"io"

	"v.io/v23/verror"
)

var (
	errInvalidString = verror.Register(pkgPath+".errInvalidString", verror.NoRetry, "{1:}{2:} string is of the wrong format and/or size{:_}")
	errNotARoutingID = verror.Register(pkgPath+".errNotARoutingID", verror.NoRetry, "{1:}{2:} Not a RoutingID{:_}")
)

// RoutingIDs have one essential property, namely that they are, to a very
// high probability globally unique. Global uniqueness is required in order
// to support comparing Endpoints for equality; this is required for sharing
// connections, for proxying (though global uniqueness is not strictly
// required) and determining if different names resolve to the same endpoint.
type RoutingID struct {
	value [routingIDLength]byte
}

const (
	routingIDLength          = 16
	firstUnreservedRoutingID = 1024
)

var (
	// NullRoutingID is a special value representing the nil route.
	NullRoutingID = FixedRoutingID(0)
)

// FixedRoutingID returns a routing ID from a constant.
func FixedRoutingID(i uint64) RoutingID {
	var rid RoutingID
	binary.BigEndian.PutUint64(rid.value[8:16], i)
	return rid
}

// IsReserved() returns true iff the RoutingID is in the reserved range.
func (rid RoutingID) IsReserved() bool {
	return isZero(rid.value[0:14]) && isLessThan(rid.value[15:16], firstUnreservedRoutingID)
}

func isZero(buf []byte) bool {
	for _, b := range buf {
		if b != 0 {
			return false
		}
	}
	return true
}

func isLessThan(buf []byte, j uint16) bool {
	return binary.BigEndian.Uint16(buf) < j
}

// String returns a print representation of the RoutingID.
func (rid RoutingID) String() string {
	return hex.EncodeToString(rid.value[:])
}

// FromString reads an RoutingID from a hex encoded string. If the argument
// string is of zero length the RoutingID will be set to NullRoutingID
func (rid *RoutingID) FromString(s string) error {
	if len(s) == 0 {
		*rid = NullRoutingID
		return nil
	}
	b, err := hex.DecodeString(s)
	if err != nil {
		return err
	}
	if len(b) != routingIDLength {
		return verror.New(errInvalidString, nil)
	}
	copy(rid.value[:], b)
	return nil
}

// Read a RoutingID from an io.Reader.
func ReadRoutingID(reader io.Reader) (RoutingID, error) {
	var rid RoutingID
	_, err := io.ReadFull(reader, rid.value[:])
	return rid, err
}

// Write a RoutingID to an io.Writer.
func (rid RoutingID) Write(writer io.Writer) error {
	_, err := writer.Write(rid.value[:])
	return err
}

func NewRoutingID() (RoutingID, error) {
	var rid RoutingID
	for {
		_, err := io.ReadFull(rand.Reader, rid.value[:])
		if err != nil {
			return NullRoutingID, err
		}
		if !rid.IsReserved() {
			return rid, nil
		}
	}
	return rid, nil
}

func Compare(a, b RoutingID) bool {
	return bytes.Compare(a.value[:], b.value[:]) == 0
}

// Implement EndpointOpt so that RoutingID can be passed as an optional
// argument to FormatEndpoint
func (RoutingID) EndpointOpt() {}
<|endoftext|>


<commit_before>package datadog

import (
	"fmt"
	"net"
	"os"
	"testing"
	"time"

	"github.com/rcrowley/go-metrics"
	"github.com/stretchr/testify/assert"
)

var _ = fmt.Println

const addr = "127.0.0.1:9999"

// testWaitTimeout determines how long to wait for a result. Configured by
// setting the TEST_TIMEOUT environment variable
var testWaitTimeout = 1 * time.Millisecond

func newServer(t *testing.T, c int) chan []byte {
	ch := make(chan []byte, 64)

	cn, err := net.ListenPacket("udp", addr)
	if cn == nil || err != nil {
		t.Fatalf("unable to create connection; %s", err)
	}

	go func() {
		defer cn.Close()

		for ; c > 0; c-- {
			cn.SetReadDeadline(time.Now().Add(testWaitTimeout << 1))
			buf := make([]byte, 128)
			n, _, err := cn.ReadFrom(buf)
			if err != nil {
				t.Fatalf("unable to read data; %s", err)
				return
			}

			ch <- buf[:n]
		}
	}()

	return ch
}

func TestMain(m *testing.M) {
	FlushLength = 1
	t := os.Getenv("TEST_TIMEOUT")
	if d, err := time.ParseDuration(t); err == nil {
		testWaitTimeout = d
	}

	os.Exit(m.Run())
}

func TestNew_WithDefaultOptions(t *testing.T) {
	r, err := New()
	assert.NoError(t, err)
	assert.NotNil(t, r)
	assert.Equal(t, "127.0.0.1:8125", r.addr)
	assert.Equal(t, metrics.DefaultRegistry, r.registry)
}

func TestNew_WithAddress(t *testing.T) {
	r, _ := New(WithAddress("127.0.0.2:8125"))
	assert.NotNil(t, r)
	assert.Equal(t, "127.0.0.2:8125", r.addr)
}

func TestReporter_FlushCounter(t *testing.T) {
	ch := newServer(t, 1)

	r := metrics.NewRegistry()
	c := metrics.NewRegisteredCounter("foo", r)
	c.Inc(1)

	dd, _ := New(WithAddress(addr), WithRegistry(r))
	dd.Flush()

	select {
	case d := <-ch:
		assert.Equal(t, "foo:1|c", string(d))

	case <-time.After(testWaitTimeout):
		assert.Fail(t, "timeout")
	}
}

func TestReporter_FlushGauge(t *testing.T) {
	ch := newServer(t, 1)

	r := metrics.NewRegistry()
	c := metrics.NewRegisteredGauge("foo", r)
	c.Update(100)

	dd, _ := New(WithAddress(addr), WithRegistry(r))
	dd.Flush()
	select {
	case d := <-ch:
		assert.Equal(t, "foo:100.000000|g", string(d))

	case <-time.After(testWaitTimeout):
		assert.Fail(t, "timeout")
	}
}

func TestReporter_FlushGaugeFloat64(t *testing.T) {
	ch := newServer(t, 1)

	r := metrics.NewRegistry()
	c := metrics.NewRegisteredGaugeFloat64("foo", r)
	c.Update(55.55)

	dd, _ := New(WithAddress(addr), WithRegistry(r))
	dd.Flush()
	select {
	case d := <-ch:
		assert.Equal(t, "foo:55.550000|g", string(d))

	case <-time.After(testWaitTimeout):
		assert.Fail(t, "timeout")
	}
}

func TestReporter_FlushHistogram(t *testing.T) {
	n := 10
	ch := newServer(t, n)

	r := metrics.NewRegistry()
	c := metrics.NewRegisteredHistogram("foo", r, metrics.NewExpDecaySample(4, 1.0))
	c.Update(11)
	c.Update(1)

	dd, _ := New(WithAddress(addr), WithRegistry(r))
	dd.Flush()

	var res []string
	for i := 0; i<n; i++ {
		select {
		case d := <-ch:
		res = append(res, string(d))

		case <-time.After(testWaitTimeout):
			assert.FailNow(t, "timeout")
		}
	}

	assert.Equal(t, "foo.count:2.000000|g", res[0])
	assert.Equal(t, "foo.max:11.000000|g", res[1])
	assert.Equal(t, "foo.min:1.000000|g", res[2])
}
<commit_msg>completed Histogram test<commit_after>package datadog

import (
	"fmt"
	"net"
	"os"
	"testing"
	"time"

	"github.com/rcrowley/go-metrics"
	"github.com/stretchr/testify/assert"
)

var _ = fmt.Println

const addr = "127.0.0.1:9999"

// testWaitTimeout determines how long to wait for a result. Configured by
// setting the TEST_TIMEOUT environment variable
var testWaitTimeout = 1 * time.Millisecond

func newServer(t *testing.T, c int) chan []byte {
	ch := make(chan []byte, 64)

	cn, err := net.ListenPacket("udp", addr)
	if cn == nil || err != nil {
		t.Fatalf("unable to create connection; %s", err)
	}

	go func() {
		defer cn.Close()

		for ; c > 0; c-- {
			cn.SetReadDeadline(time.Now().Add(testWaitTimeout << 1))
			buf := make([]byte, 128)
			n, _, err := cn.ReadFrom(buf)
			if err != nil {
				t.Fatalf("unable to read data; %s", err)
				return
			}

			ch <- buf[:n]
		}
	}()

	return ch
}

func TestMain(m *testing.M) {
	FlushLength = 1
	t := os.Getenv("TEST_TIMEOUT")
	if d, err := time.ParseDuration(t); err == nil {
		testWaitTimeout = d
	}

	os.Exit(m.Run())
}

func TestNew_WithDefaultOptions(t *testing.T) {
	r, err := New()
	assert.NoError(t, err)
	assert.NotNil(t, r)
	assert.Equal(t, "127.0.0.1:8125", r.addr)
	assert.Equal(t, metrics.DefaultRegistry, r.registry)
}

func TestNew_WithAddress(t *testing.T) {
	r, _ := New(WithAddress("127.0.0.2:8125"))
	assert.NotNil(t, r)
	assert.Equal(t, "127.0.0.2:8125", r.addr)
}

func TestReporter_FlushCounter(t *testing.T) {
	ch := newServer(t, 1)

	r := metrics.NewRegistry()
	c := metrics.NewRegisteredCounter("foo", r)
	c.Inc(1)

	dd, _ := New(WithAddress(addr), WithRegistry(r))
	dd.Flush()

	select {
	case d := <-ch:
		assert.Equal(t, "foo:1|c", string(d))

	case <-time.After(testWaitTimeout):
		assert.Fail(t, "timeout")
	}
}

func TestReporter_FlushGauge(t *testing.T) {
	ch := newServer(t, 1)

	r := metrics.NewRegistry()
	c := metrics.NewRegisteredGauge("foo", r)
	c.Update(100)

	dd, _ := New(WithAddress(addr), WithRegistry(r))
	dd.Flush()
	select {
	case d := <-ch:
		assert.Equal(t, "foo:100.000000|g", string(d))

	case <-time.After(testWaitTimeout):
		assert.Fail(t, "timeout")
	}
}

func TestReporter_FlushGaugeFloat64(t *testing.T) {
	ch := newServer(t, 1)

	r := metrics.NewRegistry()
	c := metrics.NewRegisteredGaugeFloat64("foo", r)
	c.Update(55.55)

	dd, _ := New(WithAddress(addr), WithRegistry(r))
	dd.Flush()
	select {
	case d := <-ch:
		assert.Equal(t, "foo:55.550000|g", string(d))

	case <-time.After(testWaitTimeout):
		assert.Fail(t, "timeout")
	}
}

func TestReporter_FlushHistogram(t *testing.T) {
	n := 10
	ch := newServer(t, n)

	r := metrics.NewRegistry()
	c := metrics.NewRegisteredHistogram("foo", r, metrics.NewExpDecaySample(4, 1.0))
	c.Update(11)
	c.Update(1)

	dd, _ := New(WithAddress(addr), WithRegistry(r))
	dd.Flush()

	var res []string
	for i := 0; i<n; i++ {
		select {
		case d := <-ch:
		res = append(res, string(d))

		case <-time.After(testWaitTimeout):
			assert.FailNow(t, "timeout")
		}
	}

	e := []string{
		"foo.count:2.000000|g",
		"foo.max:11.000000|g",
		"foo.min:1.000000|g",
		"foo.mean:6.000000|g",
		"foo.stddev:5.000000|g",
		"foo.var:25.000000|g",
		"foo.pct-75.00:11.000000|g",
		"foo.pct-95.00:11.000000|g",
		"foo.pct-99.00:11.000000|g",
		"foo.pct-99.90:11.000000|g",
	}
	assert.Equal(t, e, res)
}
<|endoftext|>


<commit_before>/*
Copyright 2017 Mirantis

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"flag"
	"fmt"
	"os"
	"regexp"
	"strconv"
	"strings"
	"time"

	. "github.com/onsi/gomega"

	"k8s.io/api/core/v1"
	k8serrors "k8s.io/apimachinery/pkg/api/errors"

	"github.com/Mirantis/virtlet/tests/e2e/framework"
	. "github.com/Mirantis/virtlet/tests/e2e/ginkgo-ext"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

const (
	cephContainerName = "ceph_cluster"
	// avoid having the loop device on top of overlay2/aufs when using k-d-c
	loopDeviceTestDir = "/dind/virtlet-e2e-tests"
)

var (
	vmImageLocation       = flag.String("image", defaultVMImageLocation, "VM image URL (*without http(s)://*")
	sshUser               = flag.String("sshuser", DefaultSSHUser, "default SSH user for VMs")
	includeCloudInitTests = flag.Bool("include-cloud-init-tests", false, "include Cloud-Init tests")
	includeUnsafeTests    = flag.Bool("include-unsafe-tests", false, "include tests that can be unsafe if they're run outside the build container")
	memoryLimit           = flag.Int("memoryLimit", 160, "default VM memory limit (in MiB)")
	junitOutput           = flag.String("junitOutput", "", "JUnit XML output file")
	controller            *framework.Controller
)

// UsingCirros() returns true if cirros image is being used for tests
// (which has some limitations)
func UsingCirros() bool {
	return strings.Contains(*vmImageLocation, "cirros")
}

// scheduleWaitSSH schedules SSH interface initialization before the test context starts
func scheduleWaitSSH(vm **framework.VMInterface, ssh *framework.Executor) {
	BeforeAll(func() {
		*ssh = waitSSH(*vm)
	})

	AfterAll(func() {
		(*ssh).Close()
	})
}

func waitSSH(vm *framework.VMInterface) framework.Executor {
	var ssh framework.Executor
	Eventually(
		func() error {
			var err error
			ssh, err = vm.SSH(*sshUser, SshPrivateKey)
			if err != nil {
				return err
			}
			_, err = framework.RunSimple(ssh)
			return err
		}, 60*5, 3).Should(Succeed())
	return ssh
}

func waitVirtletPod(vm *framework.VMInterface) *framework.PodInterface {
	var virtletPod *framework.PodInterface
	Eventually(
		func() error {
			var err error
			virtletPod, err = vm.VirtletPod()
			if err != nil {
				return err
			}
			for _, c := range virtletPod.Pod.Status.Conditions {
				if c.Type == v1.PodReady && c.Status == v1.ConditionTrue {
					return nil
				}
			}
			return fmt.Errorf("Pod not ready yet: %+v", virtletPod.Pod.Status)
		}, 60*5, 3).Should(Succeed())
	return virtletPod
}

func checkCPUCount(vm *framework.VMInterface, ssh framework.Executor, cpus int) {
	proc := do(framework.RunSimple(ssh, "cat", "/proc/cpuinfo")).(string)
	Expect(regexp.MustCompile(`(?m)^processor`).FindAllString(proc, -1)).To(HaveLen(cpus))
	cpuStats := do(vm.VirshCommand("domstats", "<domain>", "--vcpu")).(string)
	match := regexp.MustCompile(`vcpu\.maximum=(\d+)`).FindStringSubmatch(cpuStats)
	Expect(match).To(HaveLen(2))
	Expect(strconv.Atoi(match[1])).To(Equal(cpus))
}

func deleteVM(vm *framework.VMInterface) {
	virtletPod, err := vm.VirtletPod()
	Expect(err).NotTo(HaveOccurred())

	domainName, err := vm.DomainName()
	Expect(err).NotTo(HaveOccurred())
	domainName = domainName[8:21] // extract 5d3f8619-fda4 from virtlet-5d3f8619-fda4-cirros-vm

	Expect(vm.Delete(time.Minute * 2)).To(Succeed())

	commands := map[string][]string{
		"domain": {"list", "--name"},
		"volume": {"vol-list", "--pool", "volumes"},
		"secret": {"secret-list"},
	}

	for key, cmd := range commands {
		Eventually(func() error {
			out, err := framework.RunVirsh(virtletPod, cmd...)
			if err != nil {
				return err
			}
			if strings.Contains(out, domainName) {
				return fmt.Errorf("%s ~%s~ was not deleted", key, domainName)
			}
			return nil
		}, "3m").Should(Succeed())
	}
}

// do asserts that function with multiple return values doesn't fail
// Considering we have func `foo(something) (something, error)`
//
// `x := do(foo(something))` is equivalent to
// val, err := fn(something)
// Expect(err).To(Succeed())
// x = val
//
// The rule is that the function must return at least 2 values,
// and the last value is interpreted as error.
func do(value interface{}, extra ...interface{}) interface{} {
	if len(extra) == 0 {
		panic("bad usage of do() -- no extra values")
	}
	lastValue := extra[len(extra)-1]
	if lastValue != nil {
		err := lastValue.(error)
		Expect(err).NotTo(HaveOccurred())
	}
	return value
}

type VMOptions framework.VMOptions

func (o VMOptions) ApplyDefaults() framework.VMOptions {
	res := framework.VMOptions(o)
	if res.Image == "" {
		res.Image = *vmImageLocation
	}
	if res.SSHKey == "" && res.SSHKeySource == "" {
		res.SSHKey = SshPublicKey
	}
	if res.VCPUCount == 0 {
		res.VCPUCount = 1
	}
	if res.DiskDriver == "" {
		res.DiskDriver = "virtio"
	}
	if res.Limits == nil {
		res.Limits = map[string]string{}
	}
	if res.Limits["memory"] == "" {
		res.Limits["memory"] = fmt.Sprintf("%dMi", *memoryLimit)
	}

	return res
}

func requireCloudInit() {
	if !*includeCloudInitTests {
		Skip("Cloud-Init tests are not enabled")
	}
}

func includeUnsafe() {
	if !*includeUnsafeTests {
		Skip("Tests that are unsafe outside the build container are disabled")
	}
}

func withLoopbackBlockDevice(virtletNodeName, devPath *string, mkfs bool) {
	var nodeExecutor framework.Executor
	var filename string
	BeforeEach(func() {
		var err error
		*virtletNodeName, err = controller.VirtletNodeName()
		Expect(err).NotTo(HaveOccurred())
		nodeExecutor, err = controller.DinDNodeExecutor(*virtletNodeName)
		Expect(err).NotTo(HaveOccurred())

		_, err = framework.RunSimple(nodeExecutor, "mkdir", "-p", loopDeviceTestDir)
		Expect(err).NotTo(HaveOccurred())

		filename, err = framework.RunSimple(nodeExecutor, "tempfile", "-d", loopDeviceTestDir, "--prefix", "ve2e-")
		Expect(err).NotTo(HaveOccurred())

		_, err = framework.RunSimple(nodeExecutor, "dd", "if=/dev/zero", "of="+filename, "bs=1M", "count=1000")
		Expect(err).NotTo(HaveOccurred())
		if mkfs {
			// We use mkfs.ext3 here because mkfs.ext4 on
			// the node may be too new for CirrOS, causing
			// errors like this in VM's dmesg:
			// [    1.316395] EXT3-fs (vdb): error: couldn't mount because of unsupported optional features (2c0)
			// [    1.320222] EXT4-fs (vdb): couldn't mount RDWR because of unsupported optional features (400)
			// [    1.339594] EXT3-fs (vdc1): error: couldn't mount because of unsupported optional features (240)
			// [    1.342850] EXT4-fs (vdc1): mounted filesystem with ordered data mode. Opts: (null)
			_, err = framework.RunSimple(nodeExecutor, "mkfs.ext3", filename)
			Expect(err).NotTo(HaveOccurred())
		}
		_, err = framework.RunSimple(nodeExecutor, "sync")
		Expect(err).NotTo(HaveOccurred())
		*devPath, err = framework.RunSimple(nodeExecutor, "losetup", "-f", filename, "--show")
		Expect(err).NotTo(HaveOccurred())
	})

	AfterEach(func() {
		// The loopback device is detached by itself upon
		// success (TODO: check why it happens), so we
		// ignore errors here
		framework.RunSimple(nodeExecutor, "losetup", "-d", *devPath)
		Expect(os.RemoveAll(loopDeviceTestDir)).To(Succeed())
	})
}

func withCeph(monitorIP, secret *string, kubeSecret string) {
	BeforeAll(func() {
		nodeExecutor, err := (*controller).DinDNodeExecutor("kube-master")
		Expect(err).NotTo(HaveOccurred())

		route, err := framework.RunSimple(nodeExecutor, "route", "-n")
		Expect(err).NotTo(HaveOccurred())

		match := regexp.MustCompile(`(?:default|0\.0\.0\.0)\s+([\d.]+)`).FindStringSubmatch(route)
		Expect(match).To(HaveLen(2))

		*monitorIP = match[1]
		cephPublicNetwork := *monitorIP + "/16"

		container, err := controller.DockerContainer(cephContainerName)
		Expect(err).NotTo(HaveOccurred())

		container.Delete()
		Expect(container.PullImage("docker.io/ceph/daemon:v3.1.0-stable-3.1-mimic-centos-7")).To(Succeed())
		Expect(container.Run("docker.io/ceph/daemon:v3.1.0-stable-3.1-mimic-centos-7",
			map[string]string{
				"MON_IP":               *monitorIP,
				"CEPH_PUBLIC_NETWORK":  cephPublicNetwork,
				"CEPH_DEMO_UID":        "foo",
				"CEPH_DEMO_ACCESS_KEY": "foo",
				"CEPH_DEMO_SECRET_KEY": "foo",
				"CEPH_DEMO_BUCKET":     "foo",
				"DEMO_DAEMONS":         "osd mds",
			},
			"host", nil, false, "demo")).To(Succeed())

		cephContainerExecutor := container.Executor(false, "")
		By("Waiting for ceph cluster")
		Eventually(func() error {
			_, err := framework.RunSimple(cephContainerExecutor, "ceph", "-s")
			return err
		}).Should(Succeed())
		By("Ceph cluster started")

		commands := []string{
			// Add rbd pool and volume
			`ceph osd pool create libvirt-pool 8 8`,
			`rbd create rbd-test-image1 --size 1G --pool libvirt-pool --image-feature layering`,
			`rbd create rbd-test-image2 --size 1G --pool libvirt-pool --image-feature layering`,
			`rbd create rbd-test-image-pv --size 1G --pool libvirt-pool --image-feature layering`,

			// Add user for virtlet
			`ceph auth get-key client.admin`,
		}
		var out string
		for _, cmd := range commands {
			out = do(framework.RunSimple(cephContainerExecutor, "/bin/bash", "-c", cmd)).(string)
		}
		if secret != nil {
			*secret = out
		}
		if kubeSecret != "" {
			// buf := bytes.NewBufferString(out)
			// decoder := base64.NewDecoder(base64.StdEncoding, buf)
			// decoded, err := ioutil.ReadAll(decoder)
			// Expect(err).NotTo(HaveOccurred())
			_, err = controller.Secrets().Create(&v1.Secret{
				ObjectMeta: metav1.ObjectMeta{
					Name: kubeSecret,
				},
				Type: "kubernetes.io/rbd",
				Data: map[string][]byte{
					"key": []byte(out),
				},
			})
			Expect(err).NotTo(HaveOccurred())
		}
	})

	AfterAll(func() {
		container, err := controller.DockerContainer(cephContainerName)
		Expect(err).NotTo(HaveOccurred())
		container.Delete()
		if kubeSecret != "" {
			Expect(controller.Secrets().Delete(kubeSecret, nil)).To(Succeed())
			Eventually(func() error {
				if _, err := controller.Secrets().Get(kubeSecret, metav1.GetOptions{}); err != nil {
					if k8serrors.IsNotFound(err) {
						return nil
					}
					return err
				}
				return fmt.Errorf("secret %s was not deleted", kubeSecret)
			}).Should(Succeed())
		}
	})
}

func makeVMWithMountAndSymlinkScript(nodeName string, PVCs []framework.PVCSpec, podCustomization func(*framework.PodInterface)) *framework.VMInterface {
	vm := controller.VM("mount-vm")
	Expect(vm.CreateAndWait(VMOptions{
		NodeName: nodeName,
		// TODO: should also have an option to test using
		// ubuntu image with volumes mounted using cloud-init
		// userdata 'mounts' section
		UserDataScript: "@virtlet-mount-script@",
		PVCs:           PVCs,
	}.ApplyDefaults(), time.Minute*5, podCustomization)).To(Succeed())
	_, err := vm.Pod()
	Expect(err).NotTo(HaveOccurred())
	return vm
}

func expectToBeUsableForFilesystem(ssh framework.Executor, devPath string) {
	Eventually(func() error {
		_, err := framework.RunSimple(ssh, fmt.Sprintf("sudo /usr/sbin/mkfs.ext2 %s", devPath))
		return err
	}, 60*5, 3).Should(Succeed())
	do(framework.RunSimple(ssh, fmt.Sprintf("sudo mount %s /mnt", devPath)))
	out := do(framework.RunSimple(ssh, "ls -l /mnt")).(string)
	Expect(out).To(ContainSubstring("lost+found"))
}
<commit_msg>Fix comment for do() in e2e<commit_after>/*
Copyright 2017 Mirantis

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"flag"
	"fmt"
	"os"
	"regexp"
	"strconv"
	"strings"
	"time"

	. "github.com/onsi/gomega"

	"k8s.io/api/core/v1"
	k8serrors "k8s.io/apimachinery/pkg/api/errors"

	"github.com/Mirantis/virtlet/tests/e2e/framework"
	. "github.com/Mirantis/virtlet/tests/e2e/ginkgo-ext"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

const (
	cephContainerName = "ceph_cluster"
	// avoid having the loop device on top of overlay2/aufs when using k-d-c
	loopDeviceTestDir = "/dind/virtlet-e2e-tests"
)

var (
	vmImageLocation       = flag.String("image", defaultVMImageLocation, "VM image URL (*without http(s)://*")
	sshUser               = flag.String("sshuser", DefaultSSHUser, "default SSH user for VMs")
	includeCloudInitTests = flag.Bool("include-cloud-init-tests", false, "include Cloud-Init tests")
	includeUnsafeTests    = flag.Bool("include-unsafe-tests", false, "include tests that can be unsafe if they're run outside the build container")
	memoryLimit           = flag.Int("memoryLimit", 160, "default VM memory limit (in MiB)")
	junitOutput           = flag.String("junitOutput", "", "JUnit XML output file")
	controller            *framework.Controller
)

// UsingCirros() returns true if cirros image is being used for tests
// (which has some limitations)
func UsingCirros() bool {
	return strings.Contains(*vmImageLocation, "cirros")
}

// scheduleWaitSSH schedules SSH interface initialization before the test context starts
func scheduleWaitSSH(vm **framework.VMInterface, ssh *framework.Executor) {
	BeforeAll(func() {
		*ssh = waitSSH(*vm)
	})

	AfterAll(func() {
		(*ssh).Close()
	})
}

func waitSSH(vm *framework.VMInterface) framework.Executor {
	var ssh framework.Executor
	Eventually(
		func() error {
			var err error
			ssh, err = vm.SSH(*sshUser, SshPrivateKey)
			if err != nil {
				return err
			}
			_, err = framework.RunSimple(ssh)
			return err
		}, 60*5, 3).Should(Succeed())
	return ssh
}

func waitVirtletPod(vm *framework.VMInterface) *framework.PodInterface {
	var virtletPod *framework.PodInterface
	Eventually(
		func() error {
			var err error
			virtletPod, err = vm.VirtletPod()
			if err != nil {
				return err
			}
			for _, c := range virtletPod.Pod.Status.Conditions {
				if c.Type == v1.PodReady && c.Status == v1.ConditionTrue {
					return nil
				}
			}
			return fmt.Errorf("Pod not ready yet: %+v", virtletPod.Pod.Status)
		}, 60*5, 3).Should(Succeed())
	return virtletPod
}

func checkCPUCount(vm *framework.VMInterface, ssh framework.Executor, cpus int) {
	proc := do(framework.RunSimple(ssh, "cat", "/proc/cpuinfo")).(string)
	Expect(regexp.MustCompile(`(?m)^processor`).FindAllString(proc, -1)).To(HaveLen(cpus))
	cpuStats := do(vm.VirshCommand("domstats", "<domain>", "--vcpu")).(string)
	match := regexp.MustCompile(`vcpu\.maximum=(\d+)`).FindStringSubmatch(cpuStats)
	Expect(match).To(HaveLen(2))
	Expect(strconv.Atoi(match[1])).To(Equal(cpus))
}

func deleteVM(vm *framework.VMInterface) {
	virtletPod, err := vm.VirtletPod()
	Expect(err).NotTo(HaveOccurred())

	domainName, err := vm.DomainName()
	Expect(err).NotTo(HaveOccurred())
	domainName = domainName[8:21] // extract 5d3f8619-fda4 from virtlet-5d3f8619-fda4-cirros-vm

	Expect(vm.Delete(time.Minute * 2)).To(Succeed())

	commands := map[string][]string{
		"domain": {"list", "--name"},
		"volume": {"vol-list", "--pool", "volumes"},
		"secret": {"secret-list"},
	}

	for key, cmd := range commands {
		Eventually(func() error {
			out, err := framework.RunVirsh(virtletPod, cmd...)
			if err != nil {
				return err
			}
			if strings.Contains(out, domainName) {
				return fmt.Errorf("%s ~%s~ was not deleted", key, domainName)
			}
			return nil
		}, "3m").Should(Succeed())
	}
}

// do asserts that function with multiple return values doesn't fail.
// Considering we have func `foo(something) (something, error)`:
//
// `x := do(foo(something))` is equivalent to
// val, err := fn(something)
// Expect(err).To(Succeed())
// x = val
//
// The rule is that the function must return at least 2 values,
// of which the first one is returned as the first value of do(),
// and the last value is interpreted as error (the second value of do()).
func do(value interface{}, extra ...interface{}) interface{} {
	if len(extra) == 0 {
		panic("bad usage of do() -- no extra values")
	}
	lastValue := extra[len(extra)-1]
	if lastValue != nil {
		err := lastValue.(error)
		Expect(err).NotTo(HaveOccurred())
	}
	return value
}

type VMOptions framework.VMOptions

func (o VMOptions) ApplyDefaults() framework.VMOptions {
	res := framework.VMOptions(o)
	if res.Image == "" {
		res.Image = *vmImageLocation
	}
	if res.SSHKey == "" && res.SSHKeySource == "" {
		res.SSHKey = SshPublicKey
	}
	if res.VCPUCount == 0 {
		res.VCPUCount = 1
	}
	if res.DiskDriver == "" {
		res.DiskDriver = "virtio"
	}
	if res.Limits == nil {
		res.Limits = map[string]string{}
	}
	if res.Limits["memory"] == "" {
		res.Limits["memory"] = fmt.Sprintf("%dMi", *memoryLimit)
	}

	return res
}

func requireCloudInit() {
	if !*includeCloudInitTests {
		Skip("Cloud-Init tests are not enabled")
	}
}

func includeUnsafe() {
	if !*includeUnsafeTests {
		Skip("Tests that are unsafe outside the build container are disabled")
	}
}

func withLoopbackBlockDevice(virtletNodeName, devPath *string, mkfs bool) {
	var nodeExecutor framework.Executor
	var filename string
	BeforeEach(func() {
		var err error
		*virtletNodeName, err = controller.VirtletNodeName()
		Expect(err).NotTo(HaveOccurred())
		nodeExecutor, err = controller.DinDNodeExecutor(*virtletNodeName)
		Expect(err).NotTo(HaveOccurred())

		_, err = framework.RunSimple(nodeExecutor, "mkdir", "-p", loopDeviceTestDir)
		Expect(err).NotTo(HaveOccurred())

		filename, err = framework.RunSimple(nodeExecutor, "tempfile", "-d", loopDeviceTestDir, "--prefix", "ve2e-")
		Expect(err).NotTo(HaveOccurred())

		_, err = framework.RunSimple(nodeExecutor, "dd", "if=/dev/zero", "of="+filename, "bs=1M", "count=1000")
		Expect(err).NotTo(HaveOccurred())
		if mkfs {
			// We use mkfs.ext3 here because mkfs.ext4 on
			// the node may be too new for CirrOS, causing
			// errors like this in VM's dmesg:
			// [    1.316395] EXT3-fs (vdb): error: couldn't mount because of unsupported optional features (2c0)
			// [    1.320222] EXT4-fs (vdb): couldn't mount RDWR because of unsupported optional features (400)
			// [    1.339594] EXT3-fs (vdc1): error: couldn't mount because of unsupported optional features (240)
			// [    1.342850] EXT4-fs (vdc1): mounted filesystem with ordered data mode. Opts: (null)
			_, err = framework.RunSimple(nodeExecutor, "mkfs.ext3", filename)
			Expect(err).NotTo(HaveOccurred())
		}
		_, err = framework.RunSimple(nodeExecutor, "sync")
		Expect(err).NotTo(HaveOccurred())
		*devPath, err = framework.RunSimple(nodeExecutor, "losetup", "-f", filename, "--show")
		Expect(err).NotTo(HaveOccurred())
	})

	AfterEach(func() {
		// The loopback device is detached by itself upon
		// success (TODO: check why it happens), so we
		// ignore errors here
		framework.RunSimple(nodeExecutor, "losetup", "-d", *devPath)
		Expect(os.RemoveAll(loopDeviceTestDir)).To(Succeed())
	})
}

func withCeph(monitorIP, secret *string, kubeSecret string) {
	BeforeAll(func() {
		nodeExecutor, err := (*controller).DinDNodeExecutor("kube-master")
		Expect(err).NotTo(HaveOccurred())

		route, err := framework.RunSimple(nodeExecutor, "route", "-n")
		Expect(err).NotTo(HaveOccurred())

		match := regexp.MustCompile(`(?:default|0\.0\.0\.0)\s+([\d.]+)`).FindStringSubmatch(route)
		Expect(match).To(HaveLen(2))

		*monitorIP = match[1]
		cephPublicNetwork := *monitorIP + "/16"

		container, err := controller.DockerContainer(cephContainerName)
		Expect(err).NotTo(HaveOccurred())

		container.Delete()
		Expect(container.PullImage("docker.io/ceph/daemon:v3.1.0-stable-3.1-mimic-centos-7")).To(Succeed())
		Expect(container.Run("docker.io/ceph/daemon:v3.1.0-stable-3.1-mimic-centos-7",
			map[string]string{
				"MON_IP":               *monitorIP,
				"CEPH_PUBLIC_NETWORK":  cephPublicNetwork,
				"CEPH_DEMO_UID":        "foo",
				"CEPH_DEMO_ACCESS_KEY": "foo",
				"CEPH_DEMO_SECRET_KEY": "foo",
				"CEPH_DEMO_BUCKET":     "foo",
				"DEMO_DAEMONS":         "osd mds",
			},
			"host", nil, false, "demo")).To(Succeed())

		cephContainerExecutor := container.Executor(false, "")
		By("Waiting for ceph cluster")
		Eventually(func() error {
			_, err := framework.RunSimple(cephContainerExecutor, "ceph", "-s")
			return err
		}).Should(Succeed())
		By("Ceph cluster started")

		commands := []string{
			// Add rbd pool and volume
			`ceph osd pool create libvirt-pool 8 8`,
			`rbd create rbd-test-image1 --size 1G --pool libvirt-pool --image-feature layering`,
			`rbd create rbd-test-image2 --size 1G --pool libvirt-pool --image-feature layering`,
			`rbd create rbd-test-image-pv --size 1G --pool libvirt-pool --image-feature layering`,

			// Add user for virtlet
			`ceph auth get-key client.admin`,
		}
		var out string
		for _, cmd := range commands {
			out = do(framework.RunSimple(cephContainerExecutor, "/bin/bash", "-c", cmd)).(string)
		}
		if secret != nil {
			*secret = out
		}
		if kubeSecret != "" {
			// buf := bytes.NewBufferString(out)
			// decoder := base64.NewDecoder(base64.StdEncoding, buf)
			// decoded, err := ioutil.ReadAll(decoder)
			// Expect(err).NotTo(HaveOccurred())
			_, err = controller.Secrets().Create(&v1.Secret{
				ObjectMeta: metav1.ObjectMeta{
					Name: kubeSecret,
				},
				Type: "kubernetes.io/rbd",
				Data: map[string][]byte{
					"key": []byte(out),
				},
			})
			Expect(err).NotTo(HaveOccurred())
		}
	})

	AfterAll(func() {
		container, err := controller.DockerContainer(cephContainerName)
		Expect(err).NotTo(HaveOccurred())
		container.Delete()
		if kubeSecret != "" {
			Expect(controller.Secrets().Delete(kubeSecret, nil)).To(Succeed())
			Eventually(func() error {
				if _, err := controller.Secrets().Get(kubeSecret, metav1.GetOptions{}); err != nil {
					if k8serrors.IsNotFound(err) {
						return nil
					}
					return err
				}
				return fmt.Errorf("secret %s was not deleted", kubeSecret)
			}).Should(Succeed())
		}
	})
}

func makeVMWithMountAndSymlinkScript(nodeName string, PVCs []framework.PVCSpec, podCustomization func(*framework.PodInterface)) *framework.VMInterface {
	vm := controller.VM("mount-vm")
	Expect(vm.CreateAndWait(VMOptions{
		NodeName: nodeName,
		// TODO: should also have an option to test using
		// ubuntu image with volumes mounted using cloud-init
		// userdata 'mounts' section
		UserDataScript: "@virtlet-mount-script@",
		PVCs:           PVCs,
	}.ApplyDefaults(), time.Minute*5, podCustomization)).To(Succeed())
	_, err := vm.Pod()
	Expect(err).NotTo(HaveOccurred())
	return vm
}

func expectToBeUsableForFilesystem(ssh framework.Executor, devPath string) {
	Eventually(func() error {
		_, err := framework.RunSimple(ssh, fmt.Sprintf("sudo /usr/sbin/mkfs.ext2 %s", devPath))
		return err
	}, 60*5, 3).Should(Succeed())
	do(framework.RunSimple(ssh, fmt.Sprintf("sudo mount %s /mnt", devPath)))
	out := do(framework.RunSimple(ssh, "ls -l /mnt")).(string)
	Expect(out).To(ContainSubstring("lost+found"))
}
<|endoftext|>


<commit_before>package processorcommand

import (
	"fmt"

	"github.com/Imgur/mandible/imageprocessor/thumbType"
)

const GM_COMMAND = "convert"

func ConvertToJpeg(filename string) (string, error) {
	outfile := fmt.Sprintf("%s_jpg", filename)

	args := []string{
		filename,
		"-flatten",
		"JPEG:" + outfile,
	}

	err := runProcessorCommand(GM_COMMAND, args)
	if err != nil {
		return "", err
	}

	return outfile, nil
}

func FixOrientation(filename string) (string, error) {
	outfile := fmt.Sprintf("%s_ort", filename)

	args := []string{
		filename,
		"-auto-orient",
		outfile,
	}

	err := runProcessorCommand(GM_COMMAND, args)
	if err != nil {
		return "", err
	}

	return outfile, nil
}

func Quality(filename string, quality int) (string, error) {
	outfile := fmt.Sprintf("%s_q", filename)

	args := []string{
		filename,
		"-quality",
		fmt.Sprintf("%d", quality),
		"-density",
		"72x72",
		outfile,
	}

	err := runProcessorCommand(GM_COMMAND, args)
	if err != nil {
		return "", err
	}

	return outfile, nil
}

func ResizePercent(filename string, percent int) (string, error) {
	outfile := fmt.Sprintf("%s_rp", filename)

	args := []string{
		filename,
		"-resize",
		fmt.Sprintf("%d%%", percent),
		outfile,
	}

	err := runProcessorCommand(GM_COMMAND, args)
	if err != nil {
		return "", err
	}

	return outfile, nil
}

func SquareThumb(filename, name string, size int, format thumbType.ThumbType) (string, error) {
	outfile := fmt.Sprintf("%s_%s", filename, name)

	args := []string{
		fmt.Sprintf("%s[0]", filename),
		"-quality",
		"94",
		"-resize",
		fmt.Sprintf("%dx%d^", size, size),
		"-gravity",
		"center",
		"-crop",
		fmt.Sprintf("%dx%d+0+0", size, size),
		"-density",
		"72x72",
		"-unsharp",
		"0.5",
		fmt.Sprintf("%s:%s", format.ToString(), outfile),
	}

	err := runProcessorCommand(GM_COMMAND, args)
	if err != nil {
		return "", err
	}

	return outfile, nil
}

func Thumb(filename, name string, width, height int, format thumbType.ThumbType) (string, error) {
	outfile := fmt.Sprintf("%s_%s", filename, name)

	args := []string{
		fmt.Sprintf("%s[0]", filename),
		"-quality",
		"83",
		"-resize",
		fmt.Sprintf("%dx%d>", width, height),
		"-density",
		"72x72",
		fmt.Sprintf("%s:%s", format.ToString(), outfile),
	}

	err := runProcessorCommand(GM_COMMAND, args)
	if err != nil {
		return "", err
	}

	return outfile, nil
}

func CircleThumb(filename, name string, width int, format thumbType.ThumbType) (string, error) {
	outfile := fmt.Sprintf("%s_%s", filename, name)

	filename, err := SquareThumb(filename, name, width*2, format)
	if err != nil {
		return "", err
	}

	args := []string{
		"-size",
		fmt.Sprintf("%dx%d", width, width),
		"xc:none",
		"-fill",
		filename,
		"-quality",
		"83",
		"-density",
		"72x72",
		"-draw",
		fmt.Sprintf("circle %d,%d %d,1", width/2, width/2, width/2),
		fmt.Sprintf("PNG:%s", outfile),
	}

	err = runProcessorCommand(GM_COMMAND, args)
	if err != nil {
		return "", err
	}

	return outfile, nil
}
<commit_msg>fix circle thumb<commit_after>package processorcommand

import (
	"fmt"

	"github.com/Imgur/mandible/imageprocessor/thumbType"
)

const GM_COMMAND = "convert"

func ConvertToJpeg(filename string) (string, error) {
	outfile := fmt.Sprintf("%s_jpg", filename)

	args := []string{
		filename,
		"-flatten",
		"JPEG:" + outfile,
	}

	err := runProcessorCommand(GM_COMMAND, args)
	if err != nil {
		return "", err
	}

	return outfile, nil
}

func FixOrientation(filename string) (string, error) {
	outfile := fmt.Sprintf("%s_ort", filename)

	args := []string{
		filename,
		"-auto-orient",
		outfile,
	}

	err := runProcessorCommand(GM_COMMAND, args)
	if err != nil {
		return "", err
	}

	return outfile, nil
}

func Quality(filename string, quality int) (string, error) {
	outfile := fmt.Sprintf("%s_q", filename)

	args := []string{
		filename,
		"-quality",
		fmt.Sprintf("%d", quality),
		"-density",
		"72x72",
		outfile,
	}

	err := runProcessorCommand(GM_COMMAND, args)
	if err != nil {
		return "", err
	}

	return outfile, nil
}

func ResizePercent(filename string, percent int) (string, error) {
	outfile := fmt.Sprintf("%s_rp", filename)

	args := []string{
		filename,
		"-resize",
		fmt.Sprintf("%d%%", percent),
		outfile,
	}

	err := runProcessorCommand(GM_COMMAND, args)
	if err != nil {
		return "", err
	}

	return outfile, nil
}

func SquareThumb(filename, name string, size int, format thumbType.ThumbType) (string, error) {
	outfile := fmt.Sprintf("%s_%s", filename, name)

	args := []string{
		fmt.Sprintf("%s[0]", filename),
		"-quality",
		"94",
		"-resize",
		fmt.Sprintf("%dx%d^", size, size),
		"-gravity",
		"center",
		"-crop",
		fmt.Sprintf("%dx%d+0+0", size, size),
		"-density",
		"72x72",
		"-unsharp",
		"0.5",
		fmt.Sprintf("%s:%s", format.ToString(), outfile),
	}

	err := runProcessorCommand(GM_COMMAND, args)
	if err != nil {
		return "", err
	}

	return outfile, nil
}

func Thumb(filename, name string, width, height int, format thumbType.ThumbType) (string, error) {
	outfile := fmt.Sprintf("%s_%s", filename, name)

	args := []string{
		fmt.Sprintf("%s[0]", filename),
		"-quality",
		"83",
		"-resize",
		fmt.Sprintf("%dx%d>", width, height),
		"-density",
		"72x72",
		fmt.Sprintf("%s:%s", format.ToString(), outfile),
	}

	err := runProcessorCommand(GM_COMMAND, args)
	if err != nil {
		return "", err
	}

	return outfile, nil
}

func CircleThumb(filename, name string, width int, format thumbType.ThumbType) (string, error) {
	outfile := fmt.Sprintf("%s_%s", filename, name)

	filename, err := SquareThumb(filename, name, width, format)
	if err != nil {
		return "", err
	}

	args := []string{
		"-size",
		fmt.Sprintf("%dx%d", width, width),
		"xc:none",
		"-fill",
		filename,
		"-quality",
		"83",
		"-density",
		"72x72",
		"-draw",
		fmt.Sprintf("circle %d,%d %d,1", width/2, width/2, width/2),
		fmt.Sprintf("PNG:%s", outfile),
	}

	err = runProcessorCommand(GM_COMMAND, args)
	if err != nil {
		return "", err
	}

	return outfile, nil
}
<|endoftext|>


<commit_before>package AuthorizeCIM

import (
	"testing"
)

var previousAuth string
var previousCharged string

func TestChargeCard(t *testing.T) {
	newTransaction := NewTransaction{
		Amount: "15.90",
		CreditCard: CreditCard{
			CardNumber:     "4007000000027",
			ExpirationDate: "10/23",
		},
	}
	response := newTransaction.Charge()
	if response.Approved() {
		previousCharged = response.TransactionID()
		t.Log("#", response.TransactionID(), "Transaction was CHARGED $", newTransaction.Amount, "\n")
		t.Log("AVS Result Code: ", response.AVS().avsResultCode+"\n")
		t.Log("AVS ACVV Result Code: ", response.AVS().cavvResultCode+"\n")
		t.Log("AVS CVV Result Code: ", response.AVS().cvvResultCode+"\n")
	} else {
		t.Log(response.ErrorMessage(), "\n")
	}
}

func TestAuthOnlyCard(t *testing.T) {
	newTransaction := NewTransaction{
		Amount: "100.00",
		CreditCard: CreditCard{
			CardNumber:     "4012888818888",
			ExpirationDate: "10/27",
		},
	}
	response := newTransaction.AuthOnly()
	if response.Approved() {
		previousAuth = response.TransactionID()
		t.Log("#", response.TransactionID(), "Transaction was AUTHORIZED $", newTransaction.Amount, "\n")
	} else {
		t.Log(response.ErrorMessage(), "\n")
	}
}

func TestCaptureAuth(t *testing.T) {
	oldTransaction := PreviousTransaction{
		Amount: "49.99",
		RefId:  previousAuth,
	}
	response := oldTransaction.Capture()
	if response.Approved() {
		t.Log("#", response.TransactionID(), "Transaction was CAPTURED $", oldTransaction.Amount, "\n")
	} else {
		t.Log(response.ErrorMessage(), "\n")
	}
}

func TestChargeCardChannel(t *testing.T) {
	newTransaction := NewTransaction{
		Amount: "38.00",
		CreditCard: CreditCard{
			CardNumber:     "4012888818888",
			ExpirationDate: "10/24",
		},
		AuthCode: "RANDOMAUTHCODE",
	}
	response := newTransaction.Charge()

	if response.Approved() {
		previousAuth = response.TransactionID()
		t.Log("#", response.TransactionID(), "Transaction was Charged Through Channel (AuthCode) $", newTransaction.Amount, "\n")
	} else {
		t.Log(response.ErrorMessage(), "\n")
	}
}

func TestRefundCard(t *testing.T) {
	newTransaction := NewTransaction{
		Amount: "15.00",
		CreditCard: CreditCard{
			CardNumber:     "4012888818888",
			ExpirationDate: "10/24",
		},
		RefTransId: "0392482938402",
	}
	response := newTransaction.Refund()
	if response.Approved() {
		t.Log("#", response.TransactionID(), "Transaction was REFUNDED $", newTransaction.Amount, "\n")
	} else {
		t.Log(response.ErrorMessage(), "\n")
	}
}

func TestVoidCard(t *testing.T) {
	newTransaction := PreviousTransaction{
		RefId: previousCharged,
	}
	response := newTransaction.Void()
	if response.Approved() {
		t.Log("#", response.TransactionID(), "Transaction was VOIDED $", newTransaction.Amount, "\n")
	} else {
		t.Log(response.ErrorMessage(), "\n")
	}
}

func TestChargeCustomerProfile(t *testing.T) {

	oldProfileId := "1810921101"
	oldPaymentId := "1805617738"

	customer := Customer{
		ID:        oldProfileId,
		PaymentID: oldPaymentId,
	}

	newTransaction := NewTransaction{
		Amount: "35.00",
	}

	response := newTransaction.ChargeProfile(customer)

	response.AVS().cvvResultCode

	if response.Approved() {
		t.Log("#", response.TransactionID(), "Customer was Charged $", newTransaction.Amount, "\n")
	} else {
		t.Log(response.ErrorMessage(), "\n")
	}
}
<commit_msg>removed broken spot<commit_after>package AuthorizeCIM

import (
	"testing"
)

var previousAuth string
var previousCharged string

func TestChargeCard(t *testing.T) {
	newTransaction := NewTransaction{
		Amount: "15.90",
		CreditCard: CreditCard{
			CardNumber:     "4007000000027",
			ExpirationDate: "10/23",
		},
	}
	response := newTransaction.Charge()
	if response.Approved() {
		previousCharged = response.TransactionID()
		t.Log("#", response.TransactionID(), "Transaction was CHARGED $", newTransaction.Amount, "\n")
		t.Log("AVS Result Code: ", response.AVS().avsResultCode+"\n")
		t.Log("AVS ACVV Result Code: ", response.AVS().cavvResultCode+"\n")
		t.Log("AVS CVV Result Code: ", response.AVS().cvvResultCode+"\n")
	} else {
		t.Log(response.ErrorMessage(), "\n")
	}
}

func TestAuthOnlyCard(t *testing.T) {
	newTransaction := NewTransaction{
		Amount: "100.00",
		CreditCard: CreditCard{
			CardNumber:     "4012888818888",
			ExpirationDate: "10/27",
		},
	}
	response := newTransaction.AuthOnly()
	if response.Approved() {
		previousAuth = response.TransactionID()
		t.Log("#", response.TransactionID(), "Transaction was AUTHORIZED $", newTransaction.Amount, "\n")
	} else {
		t.Log(response.ErrorMessage(), "\n")
	}
}

func TestCaptureAuth(t *testing.T) {
	oldTransaction := PreviousTransaction{
		Amount: "49.99",
		RefId:  previousAuth,
	}
	response := oldTransaction.Capture()
	if response.Approved() {
		t.Log("#", response.TransactionID(), "Transaction was CAPTURED $", oldTransaction.Amount, "\n")
	} else {
		t.Log(response.ErrorMessage(), "\n")
	}
}

func TestChargeCardChannel(t *testing.T) {
	newTransaction := NewTransaction{
		Amount: "38.00",
		CreditCard: CreditCard{
			CardNumber:     "4012888818888",
			ExpirationDate: "10/24",
		},
		AuthCode: "RANDOMAUTHCODE",
	}
	response := newTransaction.Charge()

	if response.Approved() {
		previousAuth = response.TransactionID()
		t.Log("#", response.TransactionID(), "Transaction was Charged Through Channel (AuthCode) $", newTransaction.Amount, "\n")
	} else {
		t.Log(response.ErrorMessage(), "\n")
	}
}

func TestRefundCard(t *testing.T) {
	newTransaction := NewTransaction{
		Amount: "15.00",
		CreditCard: CreditCard{
			CardNumber:     "4012888818888",
			ExpirationDate: "10/24",
		},
		RefTransId: "0392482938402",
	}
	response := newTransaction.Refund()
	if response.Approved() {
		t.Log("#", response.TransactionID(), "Transaction was REFUNDED $", newTransaction.Amount, "\n")
	} else {
		t.Log(response.ErrorMessage(), "\n")
	}
}

func TestVoidCard(t *testing.T) {
	newTransaction := PreviousTransaction{
		RefId: previousCharged,
	}
	response := newTransaction.Void()
	if response.Approved() {
		t.Log("#", response.TransactionID(), "Transaction was VOIDED $", newTransaction.Amount, "\n")
	} else {
		t.Log(response.ErrorMessage(), "\n")
	}
}

func TestChargeCustomerProfile(t *testing.T) {

	oldProfileId := "1810921101"
	oldPaymentId := "1805617738"

	customer := Customer{
		ID:        oldProfileId,
		PaymentID: oldPaymentId,
	}

	newTransaction := NewTransaction{
		Amount: "35.00",
	}

	response := newTransaction.ChargeProfile(customer)

	if response.Approved() {
		t.Log("#", response.TransactionID(), "Customer was Charged $", newTransaction.Amount, "\n")
	} else {
		t.Log(response.ErrorMessage(), "\n")
	}
}
<|endoftext|>


<commit_before>//
// nntp.go
//
package srnd

import (
  "bufio"
  "io"
  "log"
  "net"
  "net/textproto"
  "strings"
  "time"
)

type ConnectionInfo struct {
  mode string
  newsgroup string
  allowsPosting bool
  supportsStream bool
  state string
}

type NNTPConnection struct {
  conn net.Conn
  txtconn *textproto.Conn
  inbound bool
  debug bool
  info *ConnectionInfo
  policy *FeedPolicy
  // channel for senging sync messages
  sync chan string
  // if true we are reading data
  reading bool
  // message io
  msg_reader MessageReader
  msg_writer MessageWriter
}

// ask if they need this article
func (self *NNTPConnection) askSync(msgid string) {
  if ValidMessageID(msgid) {
    self.txtconn.PrintfLine("CHECK %s", msgid)
  }
}

func (self *NNTPConnection) HandleOutbound(d *NNTPDaemon) {
  var err error
  code, line, err := self.txtconn.ReadCodeLine(-1)
  self.info.allowsPosting = code == 200
  if ! self.info.allowsPosting {
    log.Printf("outbound feed posting not allowed: %d %s", code, line)
    self.Close()
    return
  }
  // they allow posting
  // send capabilities command
  err = self.txtconn.PrintfLine("CAPABILITIES")
  capreader := bufio.NewReader(self.txtconn.DotReader())

  // get capabilites
  for {
    line, err := capreader.ReadString('\n')
    if err != nil {
      break
    }
    line = strings.ToLower(line)
    if line == "streaming\n" {
      self.info.supportsStream = true
    } else if line == "postihavestreaming\n" {
      self.info.supportsStream = true
    }
  }

  // if they support streaming and allow posting continue
  // otherwise quit
  if ! self.info.supportsStream || ! self.info.allowsPosting {
    if self.debug {
      log.Println(self.info.supportsStream, self.info.allowsPosting)
    }

    self.Quit()
    return
  }
  err = self.txtconn.PrintfLine("MODE STREAM")
  if err != nil {
    log.Println("failed to initiated streaming mode on feed", err)
    return
  }
  code, line, err = self.txtconn.ReadCodeLine(-1)
  if err != nil {
    log.Println("failed to read response for streaming handshake on feed", err)
    return
  }
  if code == 203 {
    self.info.mode = "stream"
    log.Println("streaming mode activated")
  } else {
    log.Println("streaming mode not activated, quitting")
    self.Quit()
    return
  }
  log.Println("outfeed enter mainloop")

  go func() {
    for {
      msg_id := <- self.sync
      for self.reading {
        time.Sleep(10 * time.Millisecond)
      }
      self.askSync(msg_id)
    }
  }()

  for {
    code, line, err = self.txtconn.ReadCodeLine(-1)
    if err != nil {
      log.Println("error reading response code", err)
      return
    }
    code = int(code)
    commands := strings.Split(line, " ")
    if code == 238 && len(commands) > 1 && ValidMessageID(commands[0]) {
      msg := d.store.GetMessage(commands[0])
      if msg == nil {
        log.Println("wut? don't have message", commands[0])
        self.Quit()
        return
      }
      err = self.SendMessage(msg, d)
      if err != nil {
        log.Println("failed to send message", err)
        self.Quit()
        return
      }
    } else if code == 438 {
      // declined
      continue
    } else if code == 239 {
      // accepted
      continue
    } else if code == 439 {
      // invalid
      log.Printf("article %s was not sent to outfeed, they said it was invalid", commands[0])
    } else {
      log.Printf("invalid response from outbound feed: '%d %s'", code, line)
    }
  }
}

// just do it (tm)
func (self *NNTPConnection) SendMessage(message NNTPMessage, d *NNTPDaemon) error {
  var err error
  self.reading = true
  err = self.txtconn.PrintfLine("TAKETHIS %s", message.MessageID())
  if err != nil {
    log.Println("error in outfeed", err)
    return  err
  }
  wr := self.txtconn.DotWriter()
  err = self.msg_writer.WriteMessage(message, wr)
  wr.Close()
  self.reading = false
  if err != nil {
    log.Printf("failed to send %s via feed: %s", message.MessageID(), err)
    return err
  }
  return nil
}

// handle inbound connection
func (self *NNTPConnection) HandleInbound(d *NNTPDaemon) {

  // intitiate handshake
  var err error
  self.info.mode = "STREAM"
  log.Println("Incoming nntp connection from", self.conn.RemoteAddr())
  // send welcome
  greet := "2nd generation overchan NNTP Daemon"
  self.txtconn.PrintfLine("200 %s", greet)
  for {
    if err != nil {
      log.Println("failure in infeed", err)
      self.Quit()
      return
    }
    // read line and break if needed
    line, err := self.ReadLine()
    if len(line) == 0 || err != nil {
      break
    }
    var code int
    var msg string
    commands := strings.Split(line, " ")
    cmd := commands[0]
    // capabilities command
    if cmd == "CAPABILITIES" {
      self.sendCapabilities()
    } else if cmd == "MODE" { // mode switch
      if len(commands) == 2 {
        // get mode
        mode := strings.ToUpper(commands[1])
        // reader mode
        if mode == "READER" {
          self.info.mode = "reader"
          code = 201
          msg = "posting disallowed"
        } else if mode == "STREAM" {
          // mode stream
          self.info.mode = "stream"
          code = 203
          msg = "stream it"
        } else {
          // other modes not implemented
          code = 501
          msg = "mode not implemented"
        }
      } else {
        code = 500
        msg = "syntax error"
      }

      self.txtconn.PrintfLine("%d %s", code, msg)
    } else if self.info.mode == "stream" { // we are in stream mode
      if cmd == "TAKETHIS" {
        var newsgroup string
        if len(commands) == 2 {
          article := commands[1]
          if ValidMessageID(article) {
            code := 239
            file := d.store.CreateTempFile(article)
            for {
              var line string
              line, err = self.ReadLine()
              if err != nil {
                log.Println("error reading", article, err)
                break
              }
              if line == "." {
                break
              } else {
                // newsgroup header
                if strings.HasPrefix(strings.ToLower(line), "Newsgroup: ") {
                  if len(newsgroup) == 0 {
                    newsgroup := line[11:]
                    if ! newsgroupValidFormat(newsgroup) {
                      // bad newsgroup
                      code = 439
                    }
                  }
                }
                file.Write([]byte(line))
                file.Write([]byte("\n"))
              }
            }
            file.Close()
            // tell them our result
            self.txtconn.PrintfLine("%d %s", code, article)
            // the send was good
            if code == 239 {
              log.Println(self.conn.RemoteAddr(), "got article", article)
              // inform daemon
              d.infeed_load <- article
            } else {
              // delete unaccepted article
              _ = d.store.GetTempFilename(article)
            }
          } else {
            self.txtconn.PrintfLine("439 %s", article)
          }
        }
      }
      // check command
      if cmd == "CHECK" {
        if len(commands) == 2 {
          // check syntax
          // send error if needed
          article := commands[1]
          if ! ValidMessageID(article) {
            self.txtconn.PrintfLine("501 bad message id")
            continue
          }
          // do we already have this article?
          if d.store.HasArticle(article) {
            // ya, we got it already
            // tell them to not send it
            self.txtconn.PrintfLine("438 %s we have this article", article)
          } else {
            // nope, we do not have it
            // tell them to send it
            self.txtconn.PrintfLine("238 %s we want this article please give it", article)
          }
        }
      }
    }
  }
  self.Close()
}

func (self *NNTPConnection) sendCapabilities() {
  wr := self.txtconn.DotWriter()
  io.WriteString(wr, "101 we can haz do things\n")
  io.WriteString(wr, "VERSION 2\n")
  io.WriteString(wr, "IMPLEMENTATION srndv2 better than SRNd\n")
  io.WriteString(wr, "STREAMING\n")
  io.WriteString(wr, "READER\n")
  wr.Close()
}

func (self *NNTPConnection) Quit() {
  if ! self.inbound {
    self.txtconn.PrintfLine("QUIT")
  }
  self.Close()
}

func (self *NNTPConnection) ReadLine() (string, error) {
  line, err := self.txtconn.ReadLine()
  if err != nil {
    log.Println("error reading line in feed", err)
    return "", err
  }
  if self.debug {
    log.Println(self.conn.RemoteAddr(), "recv line", line)
  }
  return line, nil
}

// close the connection
func (self *NNTPConnection) Close() {
  err := self.conn.Close()
  if err != nil {
    log.Println(self.conn.RemoteAddr(), err)
  }
  log.Println(self.conn.RemoteAddr(), "Closed Connection")
}
<commit_msg>fix sync<commit_after>//
// nntp.go
//
package srnd

import (
  "bufio"
  "io"
  "log"
  "net"
  "net/textproto"
  "strings"
  "time"
)

type ConnectionInfo struct {
  mode string
  newsgroup string
  allowsPosting bool
  supportsStream bool
  state string
}

type NNTPConnection struct {
  conn net.Conn
  txtconn *textproto.Conn
  inbound bool
  debug bool
  info *ConnectionInfo
  policy *FeedPolicy
  // channel for senging sync messages
  sync chan string
  // if true we are reading data
  reading bool
  // message io
  msg_reader MessageReader
  msg_writer MessageWriter
}

// ask if they need this article
func (self *NNTPConnection) askSync(msgid string) {
  if ValidMessageID(msgid) {
    self.txtconn.PrintfLine("CHECK %s", msgid)
  }
}

func (self *NNTPConnection) HandleOutbound(d *NNTPDaemon) {
  var err error
  code, line, err := self.txtconn.ReadCodeLine(-1)
  self.info.allowsPosting = code == 200
  if ! self.info.allowsPosting {
    log.Printf("outbound feed posting not allowed: %d %s", code, line)
    self.Close()
    return
  }
  // they allow posting
  // send capabilities command
  err = self.txtconn.PrintfLine("CAPABILITIES")
  capreader := bufio.NewReader(self.txtconn.DotReader())

  // get capabilites
  for {
    line, err := capreader.ReadString('\n')
    if err != nil {
      break
    }
    line = strings.ToLower(line)
    if line == "streaming\n" {
      self.info.supportsStream = true
    } else if line == "postihavestreaming\n" {
      self.info.supportsStream = true
    }
  }

  // if they support streaming and allow posting continue
  // otherwise quit
  if ! self.info.supportsStream || ! self.info.allowsPosting {
    if self.debug {
      log.Println(self.info.supportsStream, self.info.allowsPosting)
    }

    self.Quit()
    return
  }
  err = self.txtconn.PrintfLine("MODE STREAM")
  if err != nil {
    log.Println("failed to initiated streaming mode on feed", err)
    return
  }
  code, line, err = self.txtconn.ReadCodeLine(-1)
  if err != nil {
    log.Println("failed to read response for streaming handshake on feed", err)
    return
  }
  if code == 203 {
    self.info.mode = "stream"
    log.Println("streaming mode activated")
  } else {
    log.Println("streaming mode not activated, quitting")
    self.Quit()
    return
  }
  log.Println("outfeed enter mainloop")

  go func() {
    for {
      msg_id := <- self.sync
      for self.reading {
        time.Sleep(10 * time.Millisecond)
      }
      self.askSync(msg_id)
    }
  }()

  for {
    code, line, err = self.txtconn.ReadCodeLine(-1)
    if err != nil {
      log.Println("error reading response code", err)
      return
    }
    code = int(code)
    commands := strings.Split(line, " ")
    if code == 238 && len(commands) > 1 && ValidMessageID(commands[0]) {
      msg := d.store.GetMessage(commands[0])
      if msg == nil {
        log.Println("wut? don't have message", commands[0])
        self.Quit()
        return
      }
      err = self.SendMessage(msg, d)
      if err != nil {
        log.Println("failed to send message", err)
        self.Quit()
        return
      }
    } else if code == 438 {
      // declined
      continue
    } else if code == 239 {
      // accepted
      continue
    } else if code == 439 {
      // invalid
      log.Printf("article %s was not sent to outfeed, they said it was invalid", commands[0])
    } else {
      log.Printf("invalid response from outbound feed: '%d %s'", code, line)
    }
  }
}

// just do it (tm)
func (self *NNTPConnection) SendMessage(message NNTPMessage, d *NNTPDaemon) error {
  var err error
  self.reading = true
  err = self.txtconn.PrintfLine("TAKETHIS %s", message.MessageID())
  if err != nil {
    log.Println("error in outfeed", err)
    return  err
  }
  wr := self.txtconn.DotWriter()
  err = self.msg_writer.WriteMessage(message, wr)
  wr.Close()
  self.reading = false
  if err != nil {
    log.Printf("failed to send %s via feed: %s", message.MessageID(), err)
    return err
  }
  return nil
}

// handle inbound connection
func (self *NNTPConnection) HandleInbound(d *NNTPDaemon) {

  // intitiate handshake
  var err error
  self.info.mode = "STREAM"
  log.Println("Incoming nntp connection from", self.conn.RemoteAddr())
  // send welcome
  greet := "2nd generation overchan NNTP Daemon"
  self.txtconn.PrintfLine("200 %s", greet)
  for {
    if err != nil {
      log.Println("failure in infeed", err)
      self.Quit()
      return
    }
    // read line and break if needed
    line, err := self.ReadLine()
    if len(line) == 0 || err != nil {
      break
    }
    var code int
    var msg string
    commands := strings.Split(line, " ")
    cmd := commands[0]
    // capabilities command
    if cmd == "CAPABILITIES" {
      self.sendCapabilities()
    } else if cmd == "MODE" { // mode switch
      if len(commands) == 2 {
        // get mode
        mode := strings.ToUpper(commands[1])
        // reader mode
        if mode == "READER" {
          self.info.mode = "reader"
          code = 201
          msg = "posting disallowed"
        } else if mode == "STREAM" {
          // mode stream
          self.info.mode = "stream"
          code = 203
          msg = "stream it"
        } else {
          // other modes not implemented
          code = 501
          msg = "mode not implemented"
        }
      } else {
        code = 500
        msg = "syntax error"
      }

      self.txtconn.PrintfLine("%d %s", code, msg)
    } else if self.info.mode == "stream" { // we are in stream mode
      if cmd == "TAKETHIS" {
        var newsgroup string
        if len(commands) == 2 {
          article := commands[1]
          if ValidMessageID(article) {
            code := 239
            file := d.store.CreateTempFile(article)
            for {
              var line string
              line, err = self.ReadLine()
              if err != nil {
                log.Println("error reading", article, err)
                break
              }
              if line == "." {
                break
              } else {
                // newsgroup header
                if strings.HasPrefix(strings.ToLower(line), "Newsgroup: ") {
                  if len(newsgroup) == 0 {
                    newsgroup := line[11:]
                    if ! newsgroupValidFormat(newsgroup) {
                      // bad newsgroup
                      code = 439
                    }
                  }
                }
                file.Write([]byte(line))
                file.Write([]byte("\n"))
              }
            }
            file.Close()
            // tell them our result
            self.txtconn.PrintfLine("%d %s", code, article)
            // the send was good
            if code == 239 {
              log.Println(self.conn.RemoteAddr(), "got article", article)
              // inform daemon
              d.infeed_load <- article
            } else {
              // delete unaccepted article
              _ = d.store.GetTempFilename(article)
            }
          } else {
            self.txtconn.PrintfLine("439 %s", article)
          }
        }
      }
      // check command
      if cmd == "CHECK" {
        if len(commands) == 2 {
          // check syntax
          // send error if needed
          article := commands[1]
          if ! ValidMessageID(article) {
            self.txtconn.PrintfLine("501 bad message id")
            continue
          }
          // do we already have this article?
          if d.store.HasArticle(article) {
            // ya, we got it already
            // tell them to not send it
            self.txtconn.PrintfLine("438 %s we have this article", article)
          } else {
            // nope, we do not have it
            // tell them to send it
            self.txtconn.PrintfLine("238 %s we want this article please give it", article)
          }
        }
      }
    }
  }
  self.Close()
}

func (self *NNTPConnection) sendCapabilities() {
  wr := self.txtconn.DotWriter()
  io.WriteString(wr, "101 we can haz do things\r\n")
  io.WriteString(wr, "VERSION 2\r\n")
  io.WriteString(wr, "IMPLEMENTATION srndv2 better than SRNd\r\n")
  io.WriteString(wr, "STREAMING\r\n")
  io.WriteString(wr, "READER\r\n")
  wr.Close()
}

func (self *NNTPConnection) Quit() {
  if ! self.inbound {
    self.txtconn.PrintfLine("QUIT")
  }
  self.Close()
}

func (self *NNTPConnection) ReadLine() (string, error) {
  line, err := self.txtconn.ReadLine()
  if err != nil {
    log.Println("error reading line in feed", err)
    return "", err
  }
  if self.debug {
    log.Println(self.conn.RemoteAddr(), "recv line", line)
  }
  return line, nil
}

// close the connection
func (self *NNTPConnection) Close() {
  err := self.conn.Close()
  if err != nil {
    log.Println(self.conn.RemoteAddr(), err)
  }
  log.Println(self.conn.RemoteAddr(), "Closed Connection")
}
<|endoftext|>


<commit_before>package db

import (
	"bytes"
	"encoding/binary"

	wirefs "bazil.org/bazil/fs/wire"
	"bazil.org/fuse"
	"github.com/boltdb/bolt"
	"github.com/golang/protobuf/proto"
)

type Dirs struct {
	b *bolt.Bucket
}

func dirKey(parentInode uint64, name string) []byte {
	buf := make([]byte, 8+len(name))
	binary.BigEndian.PutUint64(buf, parentInode)
	copy(buf[8:], name)
	return buf
}

func basename(dirKey []byte) []byte {
	return dirKey[8:]
}

// Get the entry in parent directory with the given name.
//
// Returned value is valid after the transaction.
func (b *Dirs) Get(parentInode uint64, name string) (*wirefs.Dirent, error) {
	key := dirKey(parentInode, name)
	buf := b.b.Get(key)
	if buf == nil {
		return nil, fuse.ENOENT
	}

	var de wirefs.Dirent
	if err := proto.Unmarshal(buf, &de); err != nil {
		return nil, err
	}
	return &de, nil
}

// Put an entry in parent directory with the given name.
func (b *Dirs) Put(parentInode uint64, name string, de *wirefs.Dirent) error {
	buf, err := proto.Marshal(de)
	if err != nil {
		return err
	}
	key := dirKey(parentInode, name)
	if err := b.b.Put(key, buf); err != nil {
		return err
	}
	return nil
}

// Delete the entry in parent directory with the given name.
//
// Returns fuse.ENOENT if an entry does not exist.
func (b *Dirs) Delete(parentInode uint64, name string) error {
	key := dirKey(parentInode, name)
	if b.b.Get(key) == nil {
		return fuse.ENOENT
	}
	if err := b.b.Delete(key); err != nil {
		return err
	}
	return nil
}

// Rename renames an entry in the parent directory from oldName to
// newName.
//
// Returns the overwritten entry, or nil.
func (b *Dirs) Rename(parentInode uint64, oldName string, newName string) (*DirEntry, error) {
	kOld := dirKey(parentInode, oldName)
	kNew := dirKey(parentInode, newName)

	bufOld := b.b.Get(kOld)
	if bufOld == nil {
		return nil, fuse.ENOENT
	}

	// the file getting overwritten
	var loser *DirEntry
	if buf := b.b.Get(kNew); buf != nil {
		// overwriting
		loser = &DirEntry{
			name: basename(kNew),
			data: buf,
		}
	}

	if err := b.b.Put(kNew, bufOld); err != nil {
		return nil, err
	}
	if err := b.b.Delete(kOld); err != nil {
		return nil, err
	}

	return loser, nil
}

func (b *Dirs) List(inode uint64) *DirsCursor {
	c := b.b.Cursor()
	prefix := dirKey(inode, "")
	return &DirsCursor{
		inode:  inode,
		prefix: prefix,
		c:      c,
	}
}

type DirsCursor struct {
	inode  uint64
	prefix []byte
	c      *bolt.Cursor
}

func (c *DirsCursor) First() *DirEntry {
	return c.item(c.c.Seek(c.prefix))
}

func (c *DirsCursor) Next() *DirEntry {
	return c.item(c.c.Next())
}

func (c *DirsCursor) item(k, v []byte) *DirEntry {
	if !bytes.HasPrefix(k, c.prefix) {
		// past the end of the directory
		return nil
	}
	name := k[len(c.prefix):]
	return &DirEntry{name: name, data: v}
}

type DirEntry struct {
	name []byte
	data []byte
}

// Name returns the basename of this directory entry.
//
// name is valid after the transaction.
func (e *DirEntry) Name() string {
	return string(e.name)
}

// Unmarshal the directory entry to out.
//
// out is valid after the transaction.
func (e *DirEntry) Unmarshal(out *wirefs.Dirent) error {
	if err := proto.Unmarshal(e.data, out); err != nil {
		return err
	}
	return nil
}
<commit_msg>db: Support seeking in directories<commit_after>package db

import (
	"bytes"
	"encoding/binary"

	wirefs "bazil.org/bazil/fs/wire"
	"bazil.org/fuse"
	"github.com/boltdb/bolt"
	"github.com/golang/protobuf/proto"
)

type Dirs struct {
	b *bolt.Bucket
}

func dirKey(parentInode uint64, name string) []byte {
	buf := make([]byte, 8+len(name))
	binary.BigEndian.PutUint64(buf, parentInode)
	copy(buf[8:], name)
	return buf
}

func basename(dirKey []byte) []byte {
	return dirKey[8:]
}

// Get the entry in parent directory with the given name.
//
// Returned value is valid after the transaction.
func (b *Dirs) Get(parentInode uint64, name string) (*wirefs.Dirent, error) {
	key := dirKey(parentInode, name)
	buf := b.b.Get(key)
	if buf == nil {
		return nil, fuse.ENOENT
	}

	var de wirefs.Dirent
	if err := proto.Unmarshal(buf, &de); err != nil {
		return nil, err
	}
	return &de, nil
}

// Put an entry in parent directory with the given name.
func (b *Dirs) Put(parentInode uint64, name string, de *wirefs.Dirent) error {
	buf, err := proto.Marshal(de)
	if err != nil {
		return err
	}
	key := dirKey(parentInode, name)
	if err := b.b.Put(key, buf); err != nil {
		return err
	}
	return nil
}

// Delete the entry in parent directory with the given name.
//
// Returns fuse.ENOENT if an entry does not exist.
func (b *Dirs) Delete(parentInode uint64, name string) error {
	key := dirKey(parentInode, name)
	if b.b.Get(key) == nil {
		return fuse.ENOENT
	}
	if err := b.b.Delete(key); err != nil {
		return err
	}
	return nil
}

// Rename renames an entry in the parent directory from oldName to
// newName.
//
// Returns the overwritten entry, or nil.
func (b *Dirs) Rename(parentInode uint64, oldName string, newName string) (*DirEntry, error) {
	kOld := dirKey(parentInode, oldName)
	kNew := dirKey(parentInode, newName)

	bufOld := b.b.Get(kOld)
	if bufOld == nil {
		return nil, fuse.ENOENT
	}

	// the file getting overwritten
	var loser *DirEntry
	if buf := b.b.Get(kNew); buf != nil {
		// overwriting
		loser = &DirEntry{
			name: basename(kNew),
			data: buf,
		}
	}

	if err := b.b.Put(kNew, bufOld); err != nil {
		return nil, err
	}
	if err := b.b.Delete(kOld); err != nil {
		return nil, err
	}

	return loser, nil
}

func (b *Dirs) List(inode uint64) *DirsCursor {
	c := b.b.Cursor()
	prefix := dirKey(inode, "")
	return &DirsCursor{
		inode:  inode,
		prefix: prefix,
		c:      c,
	}
}

type DirsCursor struct {
	inode  uint64
	prefix []byte
	c      *bolt.Cursor
}

func (c *DirsCursor) First() *DirEntry {
	return c.item(c.c.Seek(c.prefix))
}

func (c *DirsCursor) Next() *DirEntry {
	return c.item(c.c.Next())
}

// Seek to first name equal to name, or the next one if exact match is
// not found.
//
// Passing an empty name seeks to the beginning of the directory.
func (c *DirsCursor) Seek(name string) *DirEntry {
	k := make([]byte, 0, len(c.prefix)+len(name))
	k = append(k, c.prefix...)
	k = append(k, name...)
	return c.item(c.c.Seek(k))
}

func (c *DirsCursor) item(k, v []byte) *DirEntry {
	if !bytes.HasPrefix(k, c.prefix) {
		// past the end of the directory
		return nil
	}
	name := k[len(c.prefix):]
	return &DirEntry{name: name, data: v}
}

type DirEntry struct {
	name []byte
	data []byte
}

// Name returns the basename of this directory entry.
//
// name is valid after the transaction.
func (e *DirEntry) Name() string {
	return string(e.name)
}

// Unmarshal the directory entry to out.
//
// out is valid after the transaction.
func (e *DirEntry) Unmarshal(out *wirefs.Dirent) error {
	if err := proto.Unmarshal(e.data, out); err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package cli

import (
	"fmt"

	"mynewt.apache.org/newt/newtmgr/config"
	"mynewt.apache.org/newt/newtmgr/protocol"
	"mynewt.apache.org/newt/newtmgr/transport"

	"github.com/spf13/cobra"
)

func echoRunCmd(cmd *cobra.Command, args []string) {
	cpm, err := config.NewConnProfileMgr()
	if err != nil {
		nmUsage(cmd, err)
	}

	profile, err := cpm.GetConnProfile(ConnProfileName)
	if err != nil {
		nmUsage(cmd, err)
	}

	conn, err := transport.NewConn(profile)
	if err != nil {
		nmUsage(cmd, err)
	}

	runner, err := protocol.NewCmdRunner(conn)
	if err != nil {
		nmUsage(cmd, err)
	}

	echo, err := protocol.NewEcho()
	if err != nil {
		nmUsage(cmd, err)
	}

	echo.Message = args[0]

	nmr, err := echo.EncodeWriteRequest()
	if err != nil {
		nmUsage(cmd, err)
	}

	if err := runner.WriteReq(nmr); err != nil {
		nmUsage(cmd, err)
	}

	rsp, err := runner.ReadResp()
	if err != nil {
		nmUsage(cmd, err)
	}

	ersp, err := protocol.DecodeEchoResponse(rsp.Data)
	if err != nil {
		nmUsage(cmd, err)
	}
	fmt.Println(ersp.Message)
}

func echoCmd() *cobra.Command {
	echoCmd := &cobra.Command{
		Use:   "echo",
		Short: "Send data to remote endpoint using newtmgr, and receive data back",
		Run:   echoRunCmd,
	}

	return echoCmd
}
<commit_msg>newtmgr echo; print out usage help if user passes no arguments.<commit_after>/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package cli

import (
	"fmt"

	"mynewt.apache.org/newt/newtmgr/config"
	"mynewt.apache.org/newt/newtmgr/protocol"
	"mynewt.apache.org/newt/newtmgr/transport"

	"github.com/spf13/cobra"
)

func echoRunCmd(cmd *cobra.Command, args []string) {
	cpm, err := config.NewConnProfileMgr()
	if err != nil {
		nmUsage(cmd, err)
	}

	profile, err := cpm.GetConnProfile(ConnProfileName)
	if err != nil {
		nmUsage(cmd, err)
	}

	conn, err := transport.NewConn(profile)
	if err != nil {
		nmUsage(cmd, err)
	}

	runner, err := protocol.NewCmdRunner(conn)
	if err != nil {
		nmUsage(cmd, err)
	}

	echo, err := protocol.NewEcho()
	if err != nil {
		nmUsage(cmd, err)
	}

	if len(args) != 1 {
		nmUsage(cmd, nil);
	}
	echo.Message = args[0]

	nmr, err := echo.EncodeWriteRequest()
	if err != nil {
		nmUsage(cmd, err)
	}

	if err := runner.WriteReq(nmr); err != nil {
		nmUsage(cmd, err)
	}

	rsp, err := runner.ReadResp()
	if err != nil {
		nmUsage(cmd, err)
	}

	ersp, err := protocol.DecodeEchoResponse(rsp.Data)
	if err != nil {
		nmUsage(cmd, err)
	}
	fmt.Println(ersp.Message)
}

func echoCmd() *cobra.Command {
	echoCmd := &cobra.Command{
		Use:   "echo",
		Short: "Send data to remote endpoint using newtmgr, and receive data back",
		Run:   echoRunCmd,
	}

	return echoCmd
}
<|endoftext|>


<commit_before>package gc_test

import (
	"context"
	"fmt"

	sq "github.com/Masterminds/squirrel"
	"github.com/concourse/concourse/atc"
	"github.com/concourse/concourse/atc/db"
	"github.com/concourse/concourse/atc/gc"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("ResourceCacheCollector", func() {
	var collector GcCollector
	var buildCollector GcCollector

	BeforeEach(func() {
		collector = gc.NewResourceCacheCollector(resourceCacheLifecycle)
		buildCollector = gc.NewBuildCollector(buildFactory)
	})

	Describe("Run", func() {
		Describe("resource caches", func() {
			var resourceCacheUseCollector GcCollector

			var oneOffBuild db.Build
			var jobBuild db.Build

			var oneOffCache db.UsedResourceCache
			var jobCache db.UsedResourceCache

			var resource db.Resource

			BeforeEach(func() {
				resourceCacheUseCollector = gc.NewResourceCacheUseCollector(resourceCacheLifecycle)

				oneOffBuild, err = defaultTeam.CreateOneOffBuild()
				Expect(err).ToNot(HaveOccurred())

				oneOffCache, err = resourceCacheFactory.FindOrCreateResourceCache(
					db.ForBuild(oneOffBuild.ID()),
					"some-base-type",
					atc.Version{"some": "version"},
					atc.Source{
						"some": "source",
					},
					nil,
					atc.VersionedResourceTypes{},
				)
				Expect(err).NotTo(HaveOccurred())

				jobBuild, err = defaultJob.CreateBuild()
				Expect(err).ToNot(HaveOccurred())

				jobCache, err = resourceCacheFactory.FindOrCreateResourceCache(
					db.ForBuild(jobBuild.ID()),
					"some-base-type",
					atc.Version{"some": "version"},
					atc.Source{
						"some": "source",
					},
					nil,
					atc.VersionedResourceTypes{},
				)
				Expect(err).NotTo(HaveOccurred())

				var found bool
				resource, found, err = defaultPipeline.Resource("some-resource")
				Expect(err).ToNot(HaveOccurred())
				Expect(found).To(BeTrue())

				_, err = resource.SetResourceConfig(
					atc.Source{
						"some": "source",
					},
					atc.VersionedResourceTypes{})
				Expect(err).ToNot(HaveOccurred())
			})

			resourceCacheExists := func(resourceCache db.UsedResourceCache) bool {
				var count int
				err = psql.Select("COUNT(*)").
					From("resource_caches").
					Where(sq.Eq{"id": resourceCache.ID()}).
					RunWith(dbConn).
					QueryRow().
					Scan(&count)
				Expect(err).NotTo(HaveOccurred())
				return count == 1
			}

			JustBeforeEach(func() {
				Expect(buildCollector.Run(context.TODO())).To(Succeed())
				Expect(resourceCacheUseCollector.Run(context.TODO())).To(Succeed())
				Expect(collector.Run(context.TODO())).To(Succeed())
			})

			Context("when the resource cache is still in use", func() {
				It("does not delete the cache", func() {
					Expect(collector.Run(context.TODO())).To(Succeed())
					Expect(resourceCacheExists(oneOffCache)).To(BeTrue())
					Expect(resourceCacheExists(jobCache)).To(BeTrue())
				})
			})

			Context("when the cache is no longer in use", func() {
				BeforeEach(func() {
					Expect(oneOffBuild.Finish(db.BuildStatusSucceeded)).To(Succeed())
					Expect(jobBuild.Finish(db.BuildStatusSucceeded)).To(Succeed())
				})

				Context("when the cache is an input to a job", func() {
					BeforeEach(func() {
						var versionMD5 string
						version := `{"some":"version"}`
						err = psql.Insert("resource_config_versions").
							Columns("version", "version_md5", "metadata", "resource_config_scope_id").
							Values(version, sq.Expr(fmt.Sprintf("md5('%s')", version)), `null`, jobCache.ResourceConfig().ID()).
							Suffix("RETURNING version_md5").
							RunWith(dbConn).QueryRow().Scan(&versionMD5)
						Expect(err).NotTo(HaveOccurred())

						Expect(defaultJob.SaveNextInputMapping(db.InputMapping{
							"whatever": db.InputResult{
								Input: &db.AlgorithmInput{
									AlgorithmVersion: db.AlgorithmVersion{
										Version:    db.ResourceVersion(versionMD5),
										ResourceID: resource.ID(),
									},
								},
							},
						}, true)).To(Succeed())
					})

					Context("when pipeline is paused", func() {
						BeforeEach(func() {
							err := defaultPipeline.Pause()
							Expect(err).NotTo(HaveOccurred())
						})

						It("removes the cache", func() {
							Expect(resourceCacheExists(jobCache)).To(BeFalse())
						})
					})

					Context("when pipeline is not paused", func() {
						It("leaves it alone", func() {
							Expect(resourceCacheExists(jobCache)).To(BeTrue())
						})
					})
				})

				Context("when the cache is an image resource version for a job build", func() {
					BeforeEach(func() {
						err := jobBuild.SaveImageResourceVersion(jobCache)
						Expect(err).NotTo(HaveOccurred())
					})

					It("leaves it alone", func() {
						Expect(resourceCacheExists(jobCache)).To(BeTrue())
					})

					Context("when another build of the same job exists with a different image cache", func() {
						var secondJobBuild db.Build
						var secondJobCache db.UsedResourceCache

						BeforeEach(func() {
							secondJobBuild, err = defaultJob.CreateBuild()
							Expect(err).ToNot(HaveOccurred())

							secondJobCache, err = resourceCacheFactory.FindOrCreateResourceCache(
								db.ForBuild(secondJobBuild.ID()),
								"some-base-type",
								atc.Version{"some": "new-version"},
								atc.Source{
									"some": "source",
								},
								nil,
								atc.VersionedResourceTypes{},
							)
							Expect(err).NotTo(HaveOccurred())

							Expect(secondJobBuild.SaveImageResourceVersion(secondJobCache)).To(Succeed())
						})

						Context("when the second build succeeds", func() {
							BeforeEach(func() {
								Expect(secondJobBuild.Finish(db.BuildStatusSucceeded)).To(Succeed())
							})

							It("keeps the new cache and removes the old one", func() {
								Expect(resourceCacheExists(jobCache)).To(BeFalse())
								Expect(resourceCacheExists(secondJobCache)).To(BeTrue())
							})
						})

						Context("when the second build fails", func() {
							BeforeEach(func() {
								Expect(secondJobBuild.Finish(db.BuildStatusFailed)).To(Succeed())
							})

							It("keeps the new cache and the old one", func() {
								Expect(resourceCacheExists(jobCache)).To(BeTrue())
								Expect(resourceCacheExists(secondJobCache)).To(BeTrue())
							})
						})
					})

					Context("when another build of a different job exists with a different image cache", func() {
						var secondJobBuild db.Build
						var secondJobCache db.UsedResourceCache

						BeforeEach(func() {
							secondJob, found, err := defaultPipeline.Job("some-other-job")
							Expect(err).NotTo(HaveOccurred())
							Expect(found).To(BeTrue())

							secondJobBuild, err = secondJob.CreateBuild()
							Expect(err).ToNot(HaveOccurred())

							secondJobCache, err = resourceCacheFactory.FindOrCreateResourceCache(
								db.ForBuild(secondJobBuild.ID()),
								"some-base-type",
								atc.Version{"some": "new-version"},
								atc.Source{
									"some": "source",
								},
								nil,
								atc.VersionedResourceTypes{},
							)
							Expect(err).NotTo(HaveOccurred())

							Expect(secondJobBuild.SaveImageResourceVersion(secondJobCache)).To(Succeed())
						})

						Context("when the second build succeeds", func() {
							BeforeEach(func() {
								Expect(secondJobBuild.Finish(db.BuildStatusSucceeded)).To(Succeed())
							})

							It("keeps the new cache and the old one", func() {
								Expect(resourceCacheExists(jobCache)).To(BeTrue())
								Expect(resourceCacheExists(secondJobCache)).To(BeTrue())
							})
						})

						Context("when the second build fails", func() {
							BeforeEach(func() {
								Expect(secondJobBuild.Finish(db.BuildStatusFailed)).To(Succeed())
							})

							It("keeps the new cache and the old one", func() {
								Expect(resourceCacheExists(jobCache)).To(BeTrue())
								Expect(resourceCacheExists(secondJobCache)).To(BeTrue())
							})
						})
					})
				})

				Context("when the cache is an image resource version for a one-off build", func() {
					BeforeEach(func() {
						err := oneOffBuild.SaveImageResourceVersion(oneOffCache)
						Expect(err).NotTo(HaveOccurred())
					})

					Context("when the build finished recently", func() {
						It("leaves it alone", func() {
							Expect(resourceCacheExists(oneOffCache)).To(BeTrue())
						})
					})

					Context("when the build finished a day ago", func() {
						BeforeEach(func() {
							_, err := psql.Update("builds").
								Set("end_time", sq.Expr("NOW() - '25 hours'::interval")).
								Where(sq.Eq{"id": oneOffBuild.ID()}).
								RunWith(dbConn).
								Exec()
							Expect(err).ToNot(HaveOccurred())
						})

						It("removes the cache", func() {
							Expect(resourceCacheExists(oneOffCache)).To(BeFalse())
						})
					})
				})
			})
		})
	})
})
<commit_msg>remove SetResourceConfig from cache collector test<commit_after>package gc_test

import (
	"context"
	"fmt"

	sq "github.com/Masterminds/squirrel"
	"github.com/concourse/concourse/atc"
	"github.com/concourse/concourse/atc/db"
	"github.com/concourse/concourse/atc/db/dbtest"
	"github.com/concourse/concourse/atc/gc"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("ResourceCacheCollector", func() {
	var collector GcCollector
	var buildCollector GcCollector

	BeforeEach(func() {
		collector = gc.NewResourceCacheCollector(resourceCacheLifecycle)
		buildCollector = gc.NewBuildCollector(buildFactory)
	})

	Describe("Run", func() {
		Describe("resource caches", func() {
			var resourceCacheUseCollector GcCollector

			var oneOffBuild db.Build
			var jobBuild db.Build

			var oneOffCache db.UsedResourceCache
			var jobCache db.UsedResourceCache

			var scenario *dbtest.Scenario

			BeforeEach(func() {
				resourceCacheUseCollector = gc.NewResourceCacheUseCollector(resourceCacheLifecycle)

				scenario = dbtest.Setup(
					builder.WithPipeline(atc.Config{
						Resources: atc.ResourceConfigs{
							{
								Name:   "some-resource",
								Type:   "some-base-type",
								Source: atc.Source{"some": "source"},
							},
						},
						Jobs: atc.JobConfigs{
							{
								Name: "some-job",
							},
							{
								Name: "some-other-job",
							},
						},
					}),
					builder.WithResourceVersions("some-resource"),
				)

				oneOffBuild, err = scenario.Team.CreateOneOffBuild()
				Expect(err).ToNot(HaveOccurred())

				oneOffCache, err = resourceCacheFactory.FindOrCreateResourceCache(
					db.ForBuild(oneOffBuild.ID()),
					"some-base-type",
					atc.Version{"some": "version"},
					atc.Source{
						"some": "source",
					},
					nil,
					atc.VersionedResourceTypes{},
				)
				Expect(err).NotTo(HaveOccurred())

				jobBuild, err = scenario.Job("some-job").CreateBuild()
				Expect(err).ToNot(HaveOccurred())

				jobCache, err = resourceCacheFactory.FindOrCreateResourceCache(
					db.ForBuild(jobBuild.ID()),
					"some-base-type",
					atc.Version{"some": "version"},
					atc.Source{
						"some": "source",
					},
					nil,
					atc.VersionedResourceTypes{},
				)
				Expect(err).NotTo(HaveOccurred())
			})

			resourceCacheExists := func(resourceCache db.UsedResourceCache) bool {
				var count int
				err = psql.Select("COUNT(*)").
					From("resource_caches").
					Where(sq.Eq{"id": resourceCache.ID()}).
					RunWith(dbConn).
					QueryRow().
					Scan(&count)
				Expect(err).NotTo(HaveOccurred())
				return count == 1
			}

			JustBeforeEach(func() {
				Expect(buildCollector.Run(context.TODO())).To(Succeed())
				Expect(resourceCacheUseCollector.Run(context.TODO())).To(Succeed())
				Expect(collector.Run(context.TODO())).To(Succeed())
			})

			Context("when the resource cache is still in use", func() {
				It("does not delete the cache", func() {
					Expect(collector.Run(context.TODO())).To(Succeed())
					Expect(resourceCacheExists(oneOffCache)).To(BeTrue())
					Expect(resourceCacheExists(jobCache)).To(BeTrue())
				})
			})

			Context("when the cache is no longer in use", func() {
				BeforeEach(func() {
					Expect(oneOffBuild.Finish(db.BuildStatusSucceeded)).To(Succeed())
					Expect(jobBuild.Finish(db.BuildStatusSucceeded)).To(Succeed())
				})

				Context("when the cache is an input to a job", func() {
					BeforeEach(func() {
						var versionMD5 string
						version := `{"some":"version"}`
						err = psql.Insert("resource_config_versions").
							Columns("version", "version_md5", "metadata", "resource_config_scope_id").
							Values(version, sq.Expr(fmt.Sprintf("md5('%s')", version)), `null`, jobCache.ResourceConfig().ID()).
							Suffix("RETURNING version_md5").
							RunWith(dbConn).QueryRow().Scan(&versionMD5)
						Expect(err).NotTo(HaveOccurred())

						Expect(scenario.Job("some-job").SaveNextInputMapping(db.InputMapping{
							"whatever": db.InputResult{
								Input: &db.AlgorithmInput{
									AlgorithmVersion: db.AlgorithmVersion{
										Version:    db.ResourceVersion(versionMD5),
										ResourceID: scenario.Resource("some-resource").ID(),
									},
								},
							},
						}, true)).To(Succeed())
					})

					Context("when pipeline is paused", func() {
						BeforeEach(func() {
							err := scenario.Pipeline.Pause()
							Expect(err).NotTo(HaveOccurred())
						})

						It("removes the cache", func() {
							Expect(resourceCacheExists(jobCache)).To(BeFalse())
						})
					})

					Context("when pipeline is not paused", func() {
						It("leaves it alone", func() {
							Expect(resourceCacheExists(jobCache)).To(BeTrue())
						})
					})
				})

				Context("when the cache is an image resource version for a job build", func() {
					BeforeEach(func() {
						err := jobBuild.SaveImageResourceVersion(jobCache)
						Expect(err).NotTo(HaveOccurred())
					})

					It("leaves it alone", func() {
						Expect(resourceCacheExists(jobCache)).To(BeTrue())
					})

					Context("when another build of the same job exists with a different image cache", func() {
						var secondJobBuild db.Build
						var secondJobCache db.UsedResourceCache

						BeforeEach(func() {
							secondJobBuild, err = scenario.Job("some-job").CreateBuild()
							Expect(err).ToNot(HaveOccurred())

							secondJobCache, err = resourceCacheFactory.FindOrCreateResourceCache(
								db.ForBuild(secondJobBuild.ID()),
								"some-base-type",
								atc.Version{"some": "new-version"},
								atc.Source{
									"some": "source",
								},
								nil,
								atc.VersionedResourceTypes{},
							)
							Expect(err).NotTo(HaveOccurred())

							Expect(secondJobBuild.SaveImageResourceVersion(secondJobCache)).To(Succeed())
						})

						Context("when the second build succeeds", func() {
							BeforeEach(func() {
								Expect(secondJobBuild.Finish(db.BuildStatusSucceeded)).To(Succeed())
							})

							It("keeps the new cache and removes the old one", func() {
								Expect(resourceCacheExists(jobCache)).To(BeFalse())
								Expect(resourceCacheExists(secondJobCache)).To(BeTrue())
							})
						})

						Context("when the second build fails", func() {
							BeforeEach(func() {
								Expect(secondJobBuild.Finish(db.BuildStatusFailed)).To(Succeed())
							})

							It("keeps the new cache and the old one", func() {
								Expect(resourceCacheExists(jobCache)).To(BeTrue())
								Expect(resourceCacheExists(secondJobCache)).To(BeTrue())
							})
						})
					})

					Context("when another build of a different job exists with a different image cache", func() {
						var secondJobBuild db.Build
						var secondJobCache db.UsedResourceCache

						BeforeEach(func() {
							secondJobBuild, err = scenario.Job("some-other-job").CreateBuild()
							Expect(err).ToNot(HaveOccurred())

							secondJobCache, err = resourceCacheFactory.FindOrCreateResourceCache(
								db.ForBuild(secondJobBuild.ID()),
								"some-base-type",
								atc.Version{"some": "new-version"},
								atc.Source{
									"some": "source",
								},
								nil,
								atc.VersionedResourceTypes{},
							)
							Expect(err).NotTo(HaveOccurred())

							Expect(secondJobBuild.SaveImageResourceVersion(secondJobCache)).To(Succeed())
						})

						Context("when the second build succeeds", func() {
							BeforeEach(func() {
								Expect(secondJobBuild.Finish(db.BuildStatusSucceeded)).To(Succeed())
							})

							It("keeps the new cache and the old one", func() {
								Expect(resourceCacheExists(jobCache)).To(BeTrue())
								Expect(resourceCacheExists(secondJobCache)).To(BeTrue())
							})
						})

						Context("when the second build fails", func() {
							BeforeEach(func() {
								Expect(secondJobBuild.Finish(db.BuildStatusFailed)).To(Succeed())
							})

							It("keeps the new cache and the old one", func() {
								Expect(resourceCacheExists(jobCache)).To(BeTrue())
								Expect(resourceCacheExists(secondJobCache)).To(BeTrue())
							})
						})
					})
				})

				Context("when the cache is an image resource version for a one-off build", func() {
					BeforeEach(func() {
						err := oneOffBuild.SaveImageResourceVersion(oneOffCache)
						Expect(err).NotTo(HaveOccurred())
					})

					Context("when the build finished recently", func() {
						It("leaves it alone", func() {
							Expect(resourceCacheExists(oneOffCache)).To(BeTrue())
						})
					})

					Context("when the build finished a day ago", func() {
						BeforeEach(func() {
							_, err := psql.Update("builds").
								Set("end_time", sq.Expr("NOW() - '25 hours'::interval")).
								Where(sq.Eq{"id": oneOffBuild.ID()}).
								RunWith(dbConn).
								Exec()
							Expect(err).ToNot(HaveOccurred())
						})

						It("removes the cache", func() {
							Expect(resourceCacheExists(oneOffCache)).To(BeFalse())
						})
					})
				})
			})
		})
	})
})
<|endoftext|>