<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package heap provides heap operations for any type that implements
// heap.Interface. A heap is a tree with the property that each node is the
// minimum-valued node in its subtree.
//
// The minimum element in the tree is the root, at index 0.
//
// A heap is a common way to implement a priority queue. To build a priority
// queue, implement the Heap interface with the (negative) priority as the
// ordering for the Less method, so Push adds items while Pop removes the
// highest-priority item from the queue. The Examples include such an
// implementation; the file example_pq_test.go has the complete source.
//
package heap

import "sort"

// Any type that implements heap.Interface may be used as a
// min-heap with the following invariants (established after
// Init has been called or if the data is empty or sorted):
//
//	!h.Less(j, i) for 0 <= i < h.Len() and j = 2*i+1 or 2*i+2 and j < h.Len()
//
// Note that Push and Pop in this interface are for package heap's
// implementation to call.  To add and remove things from the heap,
// use heap.Push and heap.Pop.
type Interface interface {
	sort.Interface
	Push(x interface{}) // add x as element Len()
	Pop() interface{}   // remove and return element Len() - 1.
}

// A heap must be initialized before any of the heap operations
// can be used. Init is idempotent with respect to the heap invariants
// and may be called whenever the heap invariants may have been invalidated.
// Its complexity is O(n) where n = h.Len().
//
func Init(h Interface) {
	// heapify
	n := h.Len()
	for i := n/2 - 1; i >= 0; i-- {
		down(h, i, n)
	}
}

// Push pushes the element x onto the heap. The complexity is
// O(log(n)) where n = h.Len().
//
func Push(h Interface, x interface{}) {
	h.Push(x)
	up(h, h.Len()-1)
}

// Pop removes the minimum element (according to Less) from the heap
// and returns it. The complexity is O(log(n)) where n = h.Len().
// It is equivalent to Remove(h, 0).
//
func Pop(h Interface) interface{} {
	n := h.Len() - 1
	h.Swap(0, n)
	down(h, 0, n)
	return h.Pop()
}

// Remove removes the element at index i from the heap.
// The complexity is O(log(n)) where n = h.Len().
//
func Remove(h Interface, i int) interface{} {
	n := h.Len() - 1
	if n != i {
		h.Swap(i, n)
		down(h, i, n)
		up(h, i)
	}
	return h.Pop()
}

// Fix reestablishes the heap ordering after the element at index i has changed its value.
// Changing the value of the element at index i and then calling Fix is equivalent to,
// but less expensive than, calling Remove(h, i) followed by a Push of the new value.
// The complexity is O(log(n)) where n = h.Len().
func Fix(h Interface, i int) {
	down(h, i, h.Len())
	up(h, i)
}

func up(h Interface, j int) {
	for {
		i := (j - 1) / 2 // parent
		if i == j || !h.Less(j, i) {
			break
		}
		h.Swap(i, j)
		j = i
	}
}

func down(h Interface, i, n int) {
	for {
		j1 := 2*i + 1
		if j1 >= n || j1 < 0 { // j1 < 0 after int overflow
			break
		}
		j := j1 // left child
		if j2 := j1 + 1; j2 < n && !h.Less(j1, j2) {
			j = j2 // = 2*i + 2  // right child
		}
		if !h.Less(j, i) {
			break
		}
		h.Swap(i, j)
		i = j
	}
}
<commit_msg>container/heap: avoid and/or ambiguity in documentation<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package heap provides heap operations for any type that implements
// heap.Interface. A heap is a tree with the property that each node is the
// minimum-valued node in its subtree.
//
// The minimum element in the tree is the root, at index 0.
//
// A heap is a common way to implement a priority queue. To build a priority
// queue, implement the Heap interface with the (negative) priority as the
// ordering for the Less method, so Push adds items while Pop removes the
// highest-priority item from the queue. The Examples include such an
// implementation; the file example_pq_test.go has the complete source.
//
package heap

import "sort"

// Any type that implements heap.Interface may be used as a
// min-heap with the following invariants (established after
// Init has been called or if the data is empty or sorted):
//
//	!h.Less(j, i) for 0 <= i < h.Len() and 2*i+1 <= j <= 2*i+2 and j < h.Len()
//
// Note that Push and Pop in this interface are for package heap's
// implementation to call.  To add and remove things from the heap,
// use heap.Push and heap.Pop.
type Interface interface {
	sort.Interface
	Push(x interface{}) // add x as element Len()
	Pop() interface{}   // remove and return element Len() - 1.
}

// A heap must be initialized before any of the heap operations
// can be used. Init is idempotent with respect to the heap invariants
// and may be called whenever the heap invariants may have been invalidated.
// Its complexity is O(n) where n = h.Len().
//
func Init(h Interface) {
	// heapify
	n := h.Len()
	for i := n/2 - 1; i >= 0; i-- {
		down(h, i, n)
	}
}

// Push pushes the element x onto the heap. The complexity is
// O(log(n)) where n = h.Len().
//
func Push(h Interface, x interface{}) {
	h.Push(x)
	up(h, h.Len()-1)
}

// Pop removes the minimum element (according to Less) from the heap
// and returns it. The complexity is O(log(n)) where n = h.Len().
// It is equivalent to Remove(h, 0).
//
func Pop(h Interface) interface{} {
	n := h.Len() - 1
	h.Swap(0, n)
	down(h, 0, n)
	return h.Pop()
}

// Remove removes the element at index i from the heap.
// The complexity is O(log(n)) where n = h.Len().
//
func Remove(h Interface, i int) interface{} {
	n := h.Len() - 1
	if n != i {
		h.Swap(i, n)
		down(h, i, n)
		up(h, i)
	}
	return h.Pop()
}

// Fix reestablishes the heap ordering after the element at index i has changed its value.
// Changing the value of the element at index i and then calling Fix is equivalent to,
// but less expensive than, calling Remove(h, i) followed by a Push of the new value.
// The complexity is O(log(n)) where n = h.Len().
func Fix(h Interface, i int) {
	down(h, i, h.Len())
	up(h, i)
}

func up(h Interface, j int) {
	for {
		i := (j - 1) / 2 // parent
		if i == j || !h.Less(j, i) {
			break
		}
		h.Swap(i, j)
		j = i
	}
}

func down(h Interface, i, n int) {
	for {
		j1 := 2*i + 1
		if j1 >= n || j1 < 0 { // j1 < 0 after int overflow
			break
		}
		j := j1 // left child
		if j2 := j1 + 1; j2 < n && !h.Less(j1, j2) {
			j = j2 // = 2*i + 2  // right child
		}
		if !h.Less(j, i) {
			break
		}
		h.Swap(i, j)
		i = j
	}
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package net

import (
	"flag"
	"fmt"
	"io"
	"strings"
	"syscall"
	"testing"
)

// If an IPv6 tunnel is running, we can try dialing a real IPv6 address.
var ipv6 = flag.Bool("ipv6", false, "assume ipv6 tunnel is present")

// fd is already connected to the destination, port 80.
// Run an HTTP request to fetch the appropriate page.
func fetchGoogle(t *testing.T, fd Conn, network, addr string) {
	req := []byte("GET /intl/en/privacy/ HTTP/1.0\r\nHost: www.google.com\r\n\r\n")
	n, err := fd.Write(req)

	buf := make([]byte, 1000)
	n, err = io.ReadFull(fd, buf)

	if n < 1000 {
		t.Errorf("fetchGoogle: short HTTP read from %s %s - %v", network, addr, err)
		return
	}
}

func doDial(t *testing.T, network, addr string) {
	fd, err := Dial(network, addr)
	if err != nil {
		t.Errorf("Dial(%q, %q, %q) = _, %v", network, "", addr, err)
		return
	}
	fetchGoogle(t, fd, network, addr)
	fd.Close()
}

var googleaddrs = []string{
	"%d.%d.%d.%d:80",
	"www.google.com:80",
	"%d.%d.%d.%d:http",
	"www.google.com:http",
	"%03d.%03d.%03d.%03d:0080",
	"[::ffff:%d.%d.%d.%d]:80",
	"[::ffff:%02x%02x:%02x%02x]:80",
	"[0:0:0:0:0000:ffff:%d.%d.%d.%d]:80",
	"[0:0:0:0:000000:ffff:%d.%d.%d.%d]:80",
	"[0:0:0:0:0:ffff::%d.%d.%d.%d]:80",
	"[2001:4860:0:2001::68]:80", // ipv6.google.com; removed if ipv6 flag not set
}

func TestLookupCNAME(t *testing.T) {
	if testing.Short() {
		// Don't use external network.
		t.Logf("skipping external network test during -short")
		return
	}
	cname, err := LookupCNAME("www.google.com")
	if !strings.HasSuffix(cname, ".l.google.com.") || err != nil {
		t.Errorf(`LookupCNAME("www.google.com.") = %q, %v, want "*.l.google.com.", nil`, cname, err)
	}
}

func TestDialGoogle(t *testing.T) {
	if testing.Short() {
		// Don't use external network.
		t.Logf("skipping external network test during -short")
		return
	}
	// If no ipv6 tunnel, don't try the last address.
	if !*ipv6 {
		googleaddrs[len(googleaddrs)-1] = ""
	}

	// Insert an actual IPv4 address for google.com
	// into the table.
	addrs, err := LookupIP("www.google.com")
	if err != nil {
		t.Fatalf("lookup www.google.com: %v", err)
	}
	var ip IP
	for _, addr := range addrs {
		if x := addr.To4(); x != nil {
			ip = x
			break
		}
	}
	if ip == nil {
		t.Fatalf("no IPv4 addresses for www.google.com")
	}

	for i, s := range googleaddrs {
		if strings.Contains(s, "%") {
			googleaddrs[i] = fmt.Sprintf(s, ip[0], ip[1], ip[2], ip[3])
		}
	}

	for i := 0; i < len(googleaddrs); i++ {
		addr := googleaddrs[i]
		if addr == "" {
			continue
		}
		t.Logf("-- %s --", addr)
		doDial(t, "tcp", addr)
		if addr[0] != '[' {
			doDial(t, "tcp4", addr)

			if !preferIPv4 {
				// make sure preferIPv4 flag works.
				preferIPv4 = true
				syscall.SocketDisableIPv6 = true
				doDial(t, "tcp4", addr)
				syscall.SocketDisableIPv6 = false
				preferIPv4 = false
			}
		}

		// Only run tcp6 if the kernel will take it.
		if kernelSupportsIPv6() {
			doDial(t, "tcp6", addr)
		}
	}
}
<commit_msg>net: fix ipv6 test<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package net

import (
	"flag"
	"fmt"
	"io"
	"strings"
	"syscall"
	"testing"
)

// If an IPv6 tunnel is running, we can try dialing a real IPv6 address.
var ipv6 = flag.Bool("ipv6", false, "assume ipv6 tunnel is present")

// fd is already connected to the destination, port 80.
// Run an HTTP request to fetch the appropriate page.
func fetchGoogle(t *testing.T, fd Conn, network, addr string) {
	req := []byte("GET /intl/en/privacy/ HTTP/1.0\r\nHost: www.google.com\r\n\r\n")
	n, err := fd.Write(req)

	buf := make([]byte, 1000)
	n, err = io.ReadFull(fd, buf)

	if n < 1000 {
		t.Errorf("fetchGoogle: short HTTP read from %s %s - %v", network, addr, err)
		return
	}
}

func doDial(t *testing.T, network, addr string) {
	fd, err := Dial(network, addr)
	if err != nil {
		t.Errorf("Dial(%q, %q, %q) = _, %v", network, "", addr, err)
		return
	}
	fetchGoogle(t, fd, network, addr)
	fd.Close()
}

func TestLookupCNAME(t *testing.T) {
	if testing.Short() {
		// Don't use external network.
		t.Logf("skipping external network test during -short")
		return
	}
	cname, err := LookupCNAME("www.google.com")
	if !strings.HasSuffix(cname, ".l.google.com.") || err != nil {
		t.Errorf(`LookupCNAME("www.google.com.") = %q, %v, want "*.l.google.com.", nil`, cname, err)
	}
}

var googleaddrsipv4 = []string{
	"%d.%d.%d.%d:80",
	"www.google.com:80",
	"%d.%d.%d.%d:http",
	"www.google.com:http",
	"%03d.%03d.%03d.%03d:0080",
	"[::ffff:%d.%d.%d.%d]:80",
	"[::ffff:%02x%02x:%02x%02x]:80",
	"[0:0:0:0:0000:ffff:%d.%d.%d.%d]:80",
	"[0:0:0:0:000000:ffff:%d.%d.%d.%d]:80",
	"[0:0:0:0:0:ffff::%d.%d.%d.%d]:80",
}

func TestDialGoogleIPv4(t *testing.T) {
	if testing.Short() {
		// Don't use external network.
		t.Logf("skipping external network test during -short")
		return
	}

	// Insert an actual IPv4 address for google.com
	// into the table.
	addrs, err := LookupIP("www.google.com")
	if err != nil {
		t.Fatalf("lookup www.google.com: %v", err)
	}
	var ip IP
	for _, addr := range addrs {
		if x := addr.To4(); x != nil {
			ip = x
			break
		}
	}
	if ip == nil {
		t.Fatalf("no IPv4 addresses for www.google.com")
	}

	for i, s := range googleaddrsipv4 {
		if strings.Contains(s, "%") {
			googleaddrsipv4[i] = fmt.Sprintf(s, ip[0], ip[1], ip[2], ip[3])
		}
	}

	for i := 0; i < len(googleaddrsipv4); i++ {
		addr := googleaddrsipv4[i]
		if addr == "" {
			continue
		}
		t.Logf("-- %s --", addr)
		doDial(t, "tcp", addr)
		if addr[0] != '[' {
			doDial(t, "tcp4", addr)
			if !preferIPv4 {
				// make sure preferIPv4 flag works.
				preferIPv4 = true
				syscall.SocketDisableIPv6 = true
				doDial(t, "tcp", addr)
				doDial(t, "tcp4", addr)
				syscall.SocketDisableIPv6 = false
				preferIPv4 = false
			}
		}
	}
}

var googleaddrsipv6 = []string{
	"[%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x]:80",
	"ipv6.google.com:80",
	"[%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x]:http",
	"ipv6.google.com:http",
}

func TestDialGoogleIPv6(t *testing.T) {
	if testing.Short() {
		// Don't use external network.
		t.Logf("skipping external network test during -short")
		return
	}
	// Only run tcp6 if the kernel will take it.
	if !*ipv6 || !kernelSupportsIPv6() {
		return
	}

	// Insert an actual IPv6 address for ipv6.google.com
	// into the table.
	addrs, err := LookupIP("ipv6.google.com")
	if err != nil {
		t.Fatalf("lookup ipv6.google.com: %v", err)
	}
	var ip IP
	for _, addr := range addrs {
		if x := addr.To16(); x != nil {
			ip = x
			break
		}
	}
	if ip == nil {
		t.Fatalf("no IPv6 addresses for ipv6.google.com")
	}

	for i, s := range googleaddrsipv6 {
		if strings.Contains(s, "%") {
			googleaddrsipv6[i] = fmt.Sprintf(s, ip[0], ip[1], ip[2], ip[3], ip[4], ip[5], ip[6], ip[7], ip[8], ip[9], ip[10], ip[11], ip[12], ip[13], ip[14], ip[15])
		}
	}

	for i := 0; i < len(googleaddrsipv6); i++ {
		addr := googleaddrsipv6[i]
		if addr == "" {
			continue
		}
		t.Logf("-- %s --", addr)
		doDial(t, "tcp", addr)
		doDial(t, "tcp6", addr)
	}
}
<|endoftext|>


<commit_before>package lfs

import (
	"fmt"
	"os"
	"path/filepath"
)

// Uploadable describes a file that can be uploaded.
type Uploadable struct {
	oid      string
	OidPath  string
	Filename string
	size     int64
	object   *objectResource
}

// NewUploadable builds the Uploadable from the given information.
// "filename" can be empty if a raw object is pushed (see "object-id" flag in push command)/
func NewUploadable(oid, filename string) (*Uploadable, *WrappedError) {
	localMediaPath, err := LocalMediaPath(oid)
	if err != nil {
		return nil, Errorf(err, "Error uploading file %s (%s)", filename, oid)
	}

	var pathForStats string = localMediaPath
	if len(filename) > 0 {
		if err := ensureFile(filename, localMediaPath); err != nil {
			return nil, Errorf(err, "Error uploading file %s (%s)", filename, oid)
		}
		pathForStats = filename
	}

	fi, err := os.Stat(pathForStats)
	if err != nil {
		return nil, Errorf(err, "Error uploading file %s (%s)", filename, oid)
	}

	return &Uploadable{oid: oid, OidPath: localMediaPath, Filename: filename, size: fi.Size()}, nil
}

func (u *Uploadable) Check() (*objectResource, *WrappedError) {
	return UploadCheck(u.OidPath)
}

func (u *Uploadable) Transfer(cb CopyCallback) *WrappedError {
	wcb := func(total, read int64, current int) error {
		cb(total, read, current)
		return nil
	}

	return UploadObject(u.object, wcb)
}

func (u *Uploadable) Object() *objectResource {
	return u.object
}

func (u *Uploadable) Oid() string {
	return u.oid
}

func (u *Uploadable) Size() int64 {
	return u.size
}

func (u *Uploadable) Name() string {
	return u.Filename
}

func (u *Uploadable) SetObject(o *objectResource) {
	u.object = o
}

// NewUploadQueue builds an UploadQueue, allowing `workers` concurrent uploads.
func NewUploadQueue(workers, files int) *TransferQueue {
	q := newTransferQueue(workers, files)
	q.transferKind = "upload"
	return q
}

// ensureFile makes sure that the cleanPath exists before pushing it.  If it
// does not exist, it attempts to clean it by reading the file at smudgePath.
func ensureFile(smudgePath, cleanPath string) error {
	if _, err := os.Stat(cleanPath); err == nil {
		return nil
	}

	expectedOid := filepath.Base(cleanPath)
	localPath := filepath.Join(LocalWorkingDir, smudgePath)
	file, err := os.Open(localPath)
	if err != nil {
		return err
	}

	defer file.Close()

	stat, err := file.Stat()
	if err != nil {
		return err
	}

	cleaned, err := PointerClean(file, stat.Size(), nil)
	if err != nil {
		return err
	}

	cleaned.Close()

	if expectedOid != cleaned.Oid {
		return fmt.Errorf("Expected %s to have an OID of %s, got %s", smudgePath, expectedOid, cleaned.Oid)
	}

	return nil
}
<commit_msg>Use shorthand variable declaration.<commit_after>package lfs

import (
	"fmt"
	"os"
	"path/filepath"
)

// Uploadable describes a file that can be uploaded.
type Uploadable struct {
	oid      string
	OidPath  string
	Filename string
	size     int64
	object   *objectResource
}

// NewUploadable builds the Uploadable from the given information.
// "filename" can be empty if a raw object is pushed (see "object-id" flag in push command)/
func NewUploadable(oid, filename string) (*Uploadable, *WrappedError) {
	localMediaPath, err := LocalMediaPath(oid)
	if err != nil {
		return nil, Errorf(err, "Error uploading file %s (%s)", filename, oid)
	}

	pathForStats := localMediaPath
	if len(filename) > 0 {
		if err := ensureFile(filename, localMediaPath); err != nil {
			return nil, Errorf(err, "Error uploading file %s (%s)", filename, oid)
		}
		pathForStats = filename
	}

	fi, err := os.Stat(pathForStats)
	if err != nil {
		return nil, Errorf(err, "Error uploading file %s (%s)", filename, oid)
	}

	return &Uploadable{oid: oid, OidPath: localMediaPath, Filename: filename, size: fi.Size()}, nil
}

func (u *Uploadable) Check() (*objectResource, *WrappedError) {
	return UploadCheck(u.OidPath)
}

func (u *Uploadable) Transfer(cb CopyCallback) *WrappedError {
	wcb := func(total, read int64, current int) error {
		cb(total, read, current)
		return nil
	}

	return UploadObject(u.object, wcb)
}

func (u *Uploadable) Object() *objectResource {
	return u.object
}

func (u *Uploadable) Oid() string {
	return u.oid
}

func (u *Uploadable) Size() int64 {
	return u.size
}

func (u *Uploadable) Name() string {
	return u.Filename
}

func (u *Uploadable) SetObject(o *objectResource) {
	u.object = o
}

// NewUploadQueue builds an UploadQueue, allowing `workers` concurrent uploads.
func NewUploadQueue(workers, files int) *TransferQueue {
	q := newTransferQueue(workers, files)
	q.transferKind = "upload"
	return q
}

// ensureFile makes sure that the cleanPath exists before pushing it.  If it
// does not exist, it attempts to clean it by reading the file at smudgePath.
func ensureFile(smudgePath, cleanPath string) error {
	if _, err := os.Stat(cleanPath); err == nil {
		return nil
	}

	expectedOid := filepath.Base(cleanPath)
	localPath := filepath.Join(LocalWorkingDir, smudgePath)
	file, err := os.Open(localPath)
	if err != nil {
		return err
	}

	defer file.Close()

	stat, err := file.Stat()
	if err != nil {
		return err
	}

	cleaned, err := PointerClean(file, stat.Size(), nil)
	if err != nil {
		return err
	}

	cleaned.Close()

	if expectedOid != cleaned.Oid {
		return fmt.Errorf("Expected %s to have an OID of %s, got %s", smudgePath, expectedOid, cleaned.Oid)
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package runtime_test

import "testing"

const N = 20

func BenchmarkAppend(b *testing.B) {
	b.StopTimer()
	x := make([]int, 0, N)
	b.StartTimer()
	for i := 0; i < b.N; i++ {
		x = x[0:0]
		for j := 0; j < N; j++ {
			x = append(x, j)
		}
	}
}

func BenchmarkAppendSpecialCase(b *testing.B) {
	b.StopTimer()
	x := make([]int, 0, N)
	b.StartTimer()
	for i := 0; i < b.N; i++ {
		x = x[0:0]
		for j := 0; j < N; j++ {
			if len(x) < cap(x) {
				x = x[:len(x)+1]
				x[len(x)-1] = j
			} else {
				x = append(x, j)
			}
		}
	}
}

var x = make([]int, 0, 10)

func f() int {
	x[:1][0] = 3
	return 2
}

func TestSideEffectOrder(t *testing.T) {
	x = append(x, 1, f())
	if x[0] != 1 || x[1] != 2 {
		t.Error("append failed: ", x[0], x[1])
	}
}
<commit_msg>runtime: make TestSideEffectOrder work twice<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package runtime_test

import "testing"

const N = 20

func BenchmarkAppend(b *testing.B) {
	b.StopTimer()
	x := make([]int, 0, N)
	b.StartTimer()
	for i := 0; i < b.N; i++ {
		x = x[0:0]
		for j := 0; j < N; j++ {
			x = append(x, j)
		}
	}
}

func BenchmarkAppendSpecialCase(b *testing.B) {
	b.StopTimer()
	x := make([]int, 0, N)
	b.StartTimer()
	for i := 0; i < b.N; i++ {
		x = x[0:0]
		for j := 0; j < N; j++ {
			if len(x) < cap(x) {
				x = x[:len(x)+1]
				x[len(x)-1] = j
			} else {
				x = append(x, j)
			}
		}
	}
}

var x []int

func f() int {
	x[:1][0] = 3
	return 2
}

func TestSideEffectOrder(t *testing.T) {
	x = make([]int, 0, 10)
	x = append(x, 1, f())
	if x[0] != 1 || x[1] != 2 {
		t.Error("append failed: ", x[0], x[1])
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package source

import (
	"context"
	"fmt"

	"go.uber.org/zap"
	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/equality"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/client-go/dynamic"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/kubernetes/scheme"
	"k8s.io/client-go/tools/cache"
	"knative.dev/eventing/pkg/logging"
	duckv1 "knative.dev/pkg/apis/duck/v1"
	pkgreconciler "knative.dev/pkg/reconciler"

	kedav1aplha1 "github.com/kedacore/keda/pkg/apis/keda/v1alpha1"
	kedaclientset "github.com/kedacore/keda/pkg/generated/clientset/versioned"
	"github.com/zroubalik/autoscaler-keda/pkg/reconciler/awssqs"
	"github.com/zroubalik/autoscaler-keda/pkg/reconciler/kafka"
	"github.com/zroubalik/autoscaler-keda/pkg/reconciler/keda"

	awssqsv1alpha1 "knative.dev/eventing-contrib/awssqs/pkg/apis/sources/v1alpha1"
	kafkav1beta1 "knative.dev/eventing-contrib/kafka/source/pkg/apis/sources/v1beta1"
)

type Reconciler struct {
	kubeClient      kubernetes.Interface
	kedaClient      kedaclientset.Interface
	sourceInterface dynamic.NamespaceableResourceInterface
	sourceLister    cache.GenericLister
	gvk             schema.GroupVersionKind
	gvr             schema.GroupVersionResource
}

func (r *Reconciler) Reconcile(ctx context.Context, key string) error {

	logging.FromContext(ctx).Info("RECONCILE START")

	// Convert the namespace/name string into a distinct namespace and name
	namespace, name, err := cache.SplitMetaNamespaceKey(key)
	if err != nil {
		logging.FromContext(ctx).Error("invalid resource key")
		return nil
	}

	// Get the Source resource with this namespace/name
	runtimeObj, err := r.sourceLister.ByNamespace(namespace).Get(name)
	if err != nil {
		logging.FromContext(ctx).Error("not able to get runtime object")
		return nil
	}
	if apierrors.IsNotFound(err) {
		// The resource may no longer exist, in which case we stop processing.
		logging.FromContext(ctx).Error("Source in work queue no longer exists")
		return nil
	} else if err != nil {
		return err
	}

	logging.FromContext(ctx).Info("RECONCILE kind: " + runtimeObj.GetObjectKind().GroupVersionKind().String())

	var ok bool
	if _, ok = runtimeObj.(*duckv1.Source); !ok {
		logging.FromContext(ctx).Error("runtime object is not convertible to Source duck type")
		// Avoid re-enqueuing.
		return nil
	}

	if runtimeObj.GetObjectKind().GroupVersionKind() != r.gvk {
		logging.FromContext(ctx).Error("runtime object is GVK doesn't match GVK specified for Reconciler")
		// Avoid re-enqueuing.
		return nil
	}

	unstructuredSource, err := r.sourceInterface.Namespace(namespace).Get(ctx, name, metav1.GetOptions{})
	if err != nil {
		logging.FromContext(ctx).Error("Failed to get Unstructured Source:", zap.Error(err))
		return nil
	}

	logging.FromContext(ctx).Info("RECONCILE PRED RETURN, KIND: " + r.gvk.Kind)

	switch r.gvk.Kind {
	case "KafkaSource":
		logging.FromContext(ctx).Info("RECONCILE Kafka Source")
		var kafkaSource = &kafkav1beta1.KafkaSource{}
		// TODO move scheme register up
		kafkav1beta1.AddToScheme(scheme.Scheme)
		if err := scheme.Scheme.Convert(unstructuredSource, kafkaSource, nil); err != nil {
			logging.FromContext(ctx).Error("Failed to convert Unstructured Source to KafkaSource", zap.Error(err))
			return err
		}
		return r.reconcileKafkaSource(ctx, kafkaSource)
	case "AwsSqsSource":
		logging.FromContext(ctx).Info("RECONCILE AWS SQS Source")
		var awsSqsSource = &awssqsv1alpha1.AwsSqsSource{}
		// TODO move scheme register up
		awssqsv1alpha1.AddToScheme(scheme.Scheme)
		if err := scheme.Scheme.Convert(unstructuredSource, awsSqsSource, nil); err != nil {
			logging.FromContext(ctx).Error("Failed to convert Unstructured Source to AwsSqsSource", zap.Error(err))
			return err
		}
		return r.reconcileAwsSqsSource(ctx, awsSqsSource)
	}

	return nil
}

func (r *Reconciler) reconcileKafkaSource(ctx context.Context, src *kafkav1beta1.KafkaSource) error {
	triggers, err := kafka.GenerateScaleTriggers(src)
	if err != nil {
		return err
	}
	scaledObject, err := keda.GenerateScaledObject(src, r.gvk, kafka.GenerateScaleTargetName(src), triggers)
	if err != nil {
		return err
	}

	return r.reconcileScaledObject(ctx, scaledObject, src)
}

func (r *Reconciler) reconcileAwsSqsSource(ctx context.Context, src *awssqsv1alpha1.AwsSqsSource) error {
	scaletarget, err := awssqs.GenerateScaleTargetName(ctx, r.kubeClient, src)
	if err != nil {
		return err
	}
	triggers, err := awssqs.GenerateScaleTriggers(src)
	if err != nil {
		return err
	}
	scaledObject, err := keda.GenerateScaledObject(src, r.gvk, scaletarget, triggers)
	if err != nil {
		return err
	}

	return r.reconcileScaledObject(ctx, scaledObject, src)
}

func (r *Reconciler) reconcileScaledObject(ctx context.Context, expectedScaledObject *kedav1aplha1.ScaledObject, obj metav1.Object) error {
	scaledObject, err := r.kedaClient.KedaV1alpha1().ScaledObjects(expectedScaledObject.Namespace).Get(ctx, expectedScaledObject.Name, metav1.GetOptions{})
	if apierrors.IsNotFound(err) {
		scaledObject, err = r.kedaClient.KedaV1alpha1().ScaledObjects(expectedScaledObject.Namespace).Create(ctx, expectedScaledObject, metav1.CreateOptions{})
		if err != nil {
			return scaleObjectDeploymentFailed(scaledObject.Namespace, scaledObject.Name, err)
		}
		return scaleObjectCreated(scaledObject.Namespace, scaledObject.Name)
	} else if err != nil {
		logging.FromContext(ctx).Error("Unable to get an existing ScaledObject", zap.Error(err))
		return err
	} else if !metav1.IsControlledBy(scaledObject, obj) {
		return fmt.Errorf("ScaledObject %q is not owned by %q", scaledObject.Name, obj)
	} else if !equality.Semantic.DeepDerivative(scaledObject.Spec, expectedScaledObject.Spec) {
		scaledObject.Spec = expectedScaledObject.Spec
		if _, err = r.kedaClient.KedaV1alpha1().ScaledObjects(expectedScaledObject.Namespace).Update(ctx, scaledObject, metav1.UpdateOptions{}); err != nil {
			return err
		}
		return scaleObjectUpdated(scaledObject.Namespace, scaledObject.Name)
	} else {
		logging.FromContext(ctx).Debug("Reusing existing ScaledObject", zap.Any("ScaledObject", scaledObject))
	}

	return nil
}

// scaleObjectCreated makes a new reconciler event with event type Normal, and
// reason ScaledObjectCreated.
func scaleObjectCreated(namespace, name string) pkgreconciler.Event {
	return pkgreconciler.NewEvent(corev1.EventTypeNormal, "ScaledObjectCreated", "ScaledObject created: \"%s/%s\"", namespace, name)
}

// scaleObjectUpdated makes a new reconciler event with event type Normal, and
// reason ScaledObjectUpdated.
func scaleObjectUpdated(namespace, name string) pkgreconciler.Event {
	return pkgreconciler.NewEvent(corev1.EventTypeNormal, "ScaledObjectUpdated", "ScaledObject updated: \"%s/%s\"", namespace, name)
}

// scaleObjectDeploymentFailed makes a new reconciler event with event type Warning, and
// reason ScaleObjectDeploymentFailed.
func scaleObjectDeploymentFailed(namespace, name string, err error) pkgreconciler.Event {
	return pkgreconciler.NewEvent(corev1.EventTypeWarning, "ScaleObjectDeploymentFailed", "ScaledObject deployment failed to: \"%s/%s\", %w", namespace, name, err)
}
<commit_msg>logging cleanup<commit_after>/*
Copyright 2020 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package source

import (
	"context"
	"fmt"

	"go.uber.org/zap"
	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/equality"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/client-go/dynamic"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/kubernetes/scheme"
	"k8s.io/client-go/tools/cache"
	"knative.dev/eventing/pkg/logging"
	duckv1 "knative.dev/pkg/apis/duck/v1"
	pkgreconciler "knative.dev/pkg/reconciler"

	kedav1aplha1 "github.com/kedacore/keda/pkg/apis/keda/v1alpha1"
	kedaclientset "github.com/kedacore/keda/pkg/generated/clientset/versioned"
	"github.com/zroubalik/autoscaler-keda/pkg/reconciler/awssqs"
	"github.com/zroubalik/autoscaler-keda/pkg/reconciler/kafka"
	"github.com/zroubalik/autoscaler-keda/pkg/reconciler/keda"

	awssqsv1alpha1 "knative.dev/eventing-contrib/awssqs/pkg/apis/sources/v1alpha1"
	kafkav1beta1 "knative.dev/eventing-contrib/kafka/source/pkg/apis/sources/v1beta1"
)

type Reconciler struct {
	kubeClient      kubernetes.Interface
	kedaClient      kedaclientset.Interface
	sourceInterface dynamic.NamespaceableResourceInterface
	sourceLister    cache.GenericLister
	gvk             schema.GroupVersionKind
	gvr             schema.GroupVersionResource
}

func (r *Reconciler) Reconcile(ctx context.Context, key string) error {

	// Convert the namespace/name string into a distinct namespace and name
	namespace, name, err := cache.SplitMetaNamespaceKey(key)
	if err != nil {
		logging.FromContext(ctx).Error("invalid resource key")
		return nil
	}

	// Get the Source resource with this namespace/name
	runtimeObj, err := r.sourceLister.ByNamespace(namespace).Get(name)
	if err != nil {
		logging.FromContext(ctx).Error("not able to get runtime object")
		return nil
	}
	if apierrors.IsNotFound(err) {
		// The resource may no longer exist, in which case we stop processing.
		logging.FromContext(ctx).Error("Source in work queue no longer exists")
		return nil
	} else if err != nil {
		return err
	}

	logging.FromContext(ctx).Info("RECONCILE kind: " + runtimeObj.GetObjectKind().GroupVersionKind().String())

	var ok bool
	if _, ok = runtimeObj.(*duckv1.Source); !ok {
		logging.FromContext(ctx).Error("runtime object is not convertible to Source duck type")
		// Avoid re-enqueuing.
		return nil
	}

	if runtimeObj.GetObjectKind().GroupVersionKind() != r.gvk {
		logging.FromContext(ctx).Error("runtime object is GVK doesn't match GVK specified for Reconciler")
		// Avoid re-enqueuing.
		return nil
	}

	unstructuredSource, err := r.sourceInterface.Namespace(namespace).Get(ctx, name, metav1.GetOptions{})
	if err != nil {
		logging.FromContext(ctx).Error("Failed to get Unstructured Source:", zap.Error(err))
		return nil
	}

	switch r.gvk.Kind {
	case "KafkaSource":
		var kafkaSource = &kafkav1beta1.KafkaSource{}
		// TODO move scheme register up
		kafkav1beta1.AddToScheme(scheme.Scheme)
		if err := scheme.Scheme.Convert(unstructuredSource, kafkaSource, nil); err != nil {
			logging.FromContext(ctx).Error("Failed to convert Unstructured Source to KafkaSource", zap.Error(err))
			return err
		}
		return r.reconcileKafkaSource(ctx, kafkaSource)
	case "AwsSqsSource":
		var awsSqsSource = &awssqsv1alpha1.AwsSqsSource{}
		// TODO move scheme register up
		awssqsv1alpha1.AddToScheme(scheme.Scheme)
		if err := scheme.Scheme.Convert(unstructuredSource, awsSqsSource, nil); err != nil {
			logging.FromContext(ctx).Error("Failed to convert Unstructured Source to AwsSqsSource", zap.Error(err))
			return err
		}
		return r.reconcileAwsSqsSource(ctx, awsSqsSource)
	}

	return nil
}

func (r *Reconciler) reconcileKafkaSource(ctx context.Context, src *kafkav1beta1.KafkaSource) error {
	triggers, err := kafka.GenerateScaleTriggers(src)
	if err != nil {
		return err
	}
	scaledObject, err := keda.GenerateScaledObject(src, r.gvk, kafka.GenerateScaleTargetName(src), triggers)
	if err != nil {
		return err
	}

	return r.reconcileScaledObject(ctx, scaledObject, src)
}

func (r *Reconciler) reconcileAwsSqsSource(ctx context.Context, src *awssqsv1alpha1.AwsSqsSource) error {
	scaletarget, err := awssqs.GenerateScaleTargetName(ctx, r.kubeClient, src)
	if err != nil {
		return err
	}
	triggers, err := awssqs.GenerateScaleTriggers(src)
	if err != nil {
		return err
	}
	scaledObject, err := keda.GenerateScaledObject(src, r.gvk, scaletarget, triggers)
	if err != nil {
		return err
	}

	return r.reconcileScaledObject(ctx, scaledObject, src)
}

func (r *Reconciler) reconcileScaledObject(ctx context.Context, expectedScaledObject *kedav1aplha1.ScaledObject, obj metav1.Object) error {
	scaledObject, err := r.kedaClient.KedaV1alpha1().ScaledObjects(expectedScaledObject.Namespace).Get(ctx, expectedScaledObject.Name, metav1.GetOptions{})
	if apierrors.IsNotFound(err) {
		scaledObject, err = r.kedaClient.KedaV1alpha1().ScaledObjects(expectedScaledObject.Namespace).Create(ctx, expectedScaledObject, metav1.CreateOptions{})
		if err != nil {
			return scaleObjectDeploymentFailed(scaledObject.Namespace, scaledObject.Name, err)
		}
		return scaleObjectCreated(scaledObject.Namespace, scaledObject.Name)
	} else if err != nil {
		logging.FromContext(ctx).Error("Unable to get an existing ScaledObject", zap.Error(err))
		return err
	} else if !metav1.IsControlledBy(scaledObject, obj) {
		return fmt.Errorf("ScaledObject %q is not owned by %q", scaledObject.Name, obj)
	} else if !equality.Semantic.DeepDerivative(scaledObject.Spec, expectedScaledObject.Spec) {
		scaledObject.Spec = expectedScaledObject.Spec
		if _, err = r.kedaClient.KedaV1alpha1().ScaledObjects(expectedScaledObject.Namespace).Update(ctx, scaledObject, metav1.UpdateOptions{}); err != nil {
			return err
		}
		return scaleObjectUpdated(scaledObject.Namespace, scaledObject.Name)
	} else {
		logging.FromContext(ctx).Debug("Reusing existing ScaledObject", zap.Any("ScaledObject", scaledObject))
	}

	return nil
}

// scaleObjectCreated makes a new reconciler event with event type Normal, and
// reason ScaledObjectCreated.
func scaleObjectCreated(namespace, name string) pkgreconciler.Event {
	return pkgreconciler.NewEvent(corev1.EventTypeNormal, "ScaledObjectCreated", "ScaledObject created: \"%s/%s\"", namespace, name)
}

// scaleObjectUpdated makes a new reconciler event with event type Normal, and
// reason ScaledObjectUpdated.
func scaleObjectUpdated(namespace, name string) pkgreconciler.Event {
	return pkgreconciler.NewEvent(corev1.EventTypeNormal, "ScaledObjectUpdated", "ScaledObject updated: \"%s/%s\"", namespace, name)
}

// scaleObjectDeploymentFailed makes a new reconciler event with event type Warning, and
// reason ScaleObjectDeploymentFailed.
func scaleObjectDeploymentFailed(namespace, name string, err error) pkgreconciler.Event {
	return pkgreconciler.NewEvent(corev1.EventTypeWarning, "ScaleObjectDeploymentFailed", "ScaledObject deployment failed to: \"%s/%s\", %w", namespace, name, err)
}
<|endoftext|>


<commit_before>package migration

import (
	"database/sql"
	"fmt"
	"time"

	"code.cloudfoundry.org/lager"
	"github.com/concourse/atc/db/lock"
	"github.com/mattes/migrate"
	"github.com/mattes/migrate/database/postgres"
	"github.com/mattes/migrate/source"
	"github.com/mattes/migrate/source/go-bindata"

	_ "github.com/lib/pq"
	_ "github.com/mattes/migrate/source/file"
)

func Open(driver, dsn string) (*sql.DB, error) {
	return OpenWithLockFactory(driver, dsn, nil)
}

func OpenWithLockFactory(driver, dsn string, lockFactory lock.LockFactory) (*sql.DB, error) {
	db, err := sql.Open(driver, dsn)
	if err != nil {
		return nil, err
	}

	s, err := bindata.WithInstance(bindata.Resource(
		AssetNames(),
		func(name string) ([]byte, error) {
			return Asset(name)
		}),
	)

	dbConn, err := OpenWithMigrateDrivers(db, "go-bindata", s, lockFactory)
	if err != nil {
		_ = db.Close()
		return nil, err
	}

	return dbConn, nil
}

func OpenWithMigrateDrivers(db *sql.DB, sourceName string, s source.Driver, lockFactory lock.LockFactory) (*sql.DB, error) {

	logger := lager.NewLogger("migrations").Session("locks")
	for {

		if lockFactory != nil {
			lock, acquired, err := lockFactory.Acquire(logger, lock.NewDatabaseMigrationLockID())
			if err != nil {
				return nil, err
			}

			if !acquired {
				time.Sleep(1 * time.Second)
				continue
			}

			defer lock.Release()
		}

		forceVersion, err := checkMigrationVersion(db)

		if err != nil {
			return nil, err
		}

		d, err := postgres.WithInstance(db, &postgres.Config{})
		if err != nil {
			_ = db.Close()
			return nil, err
		}

		m, err := migrate.NewWithInstance(sourceName, s, "postgres", d)
		if err != nil {
			return nil, err
		}

		if forceVersion > 0 {
			if err = m.Force(forceVersion); err != nil {
				logger.Error("migrations", err)
				return nil, err
			}
		}

		if err = m.Up(); err != nil {
			if err.Error() != "no change" {
				logger.Error("migrations", err)
				return nil, err
			}
		}

		return db, nil
	}
}

func checkMigrationVersion(db *sql.DB) (int, error) {
	oldMigrationLastVersion := 189
	newMigrationStartVersion := 1510262030

	var err error
	var dbVersion int

	if err = db.QueryRow("SELECT version FROM migration_version").Scan(&dbVersion); err != nil {
		return -1, nil
	}

	if dbVersion != oldMigrationLastVersion {
		return -1, fmt.Errorf("Must upgrade from db version %d (concourse 3.6.0), current db version: %d", oldMigrationLastVersion, dbVersion)
	}

	if _, err = db.Exec("DROP TABLE IF EXISTS migration_version"); err != nil {
		return -1, nil
	}

	return newMigrationStartVersion, nil
}
<commit_msg>Remove double db close if migrate table check fails<commit_after>package migration

import (
	"database/sql"
	"fmt"
	"time"

	"code.cloudfoundry.org/lager"
	"github.com/concourse/atc/db/lock"
	"github.com/mattes/migrate"
	"github.com/mattes/migrate/database/postgres"
	"github.com/mattes/migrate/source"
	"github.com/mattes/migrate/source/go-bindata"

	_ "github.com/lib/pq"
	_ "github.com/mattes/migrate/source/file"
)

func Open(driver, dsn string) (*sql.DB, error) {
	return OpenWithLockFactory(driver, dsn, nil)
}

func OpenWithLockFactory(driver, dsn string, lockFactory lock.LockFactory) (*sql.DB, error) {
	db, err := sql.Open(driver, dsn)
	if err != nil {
		return nil, err
	}

	s, err := bindata.WithInstance(bindata.Resource(
		AssetNames(),
		func(name string) ([]byte, error) {
			return Asset(name)
		}),
	)

	dbConn, err := OpenWithMigrateDrivers(db, "go-bindata", s, lockFactory)
	if err != nil {
		_ = db.Close()
		return nil, err
	}

	return dbConn, nil
}

func OpenWithMigrateDrivers(db *sql.DB, sourceName string, s source.Driver, lockFactory lock.LockFactory) (*sql.DB, error) {

	logger := lager.NewLogger("migrations").Session("locks")
	for {

		if lockFactory != nil {
			lock, acquired, err := lockFactory.Acquire(logger, lock.NewDatabaseMigrationLockID())
			if err != nil {
				return nil, err
			}

			if !acquired {
				time.Sleep(1 * time.Second)
				continue
			}

			defer lock.Release()
		}

		forceVersion, err := checkMigrationVersion(db)

		if err != nil {
			return nil, err
		}

		d, err := postgres.WithInstance(db, &postgres.Config{})
		if err != nil {
			return nil, err
		}

		m, err := migrate.NewWithInstance(sourceName, s, "postgres", d)
		if err != nil {
			return nil, err
		}

		if forceVersion > 0 {
			if err = m.Force(forceVersion); err != nil {
				logger.Error("migrations", err)
				return nil, err
			}
		}

		if err = m.Up(); err != nil {
			if err.Error() != "no change" {
				logger.Error("migrations", err)
				return nil, err
			}
		}

		return db, nil
	}
}

func checkMigrationVersion(db *sql.DB) (int, error) {
	oldMigrationLastVersion := 189
	newMigrationStartVersion := 1510262030

	var err error
	var dbVersion int

	if err = db.QueryRow("SELECT version FROM migration_version").Scan(&dbVersion); err != nil {
		return -1, nil
	}

	if dbVersion != oldMigrationLastVersion {
		return -1, fmt.Errorf("Must upgrade from db version %d (concourse 3.6.0), current db version: %d", oldMigrationLastVersion, dbVersion)
	}

	if _, err = db.Exec("DROP TABLE IF EXISTS migration_version"); err != nil {
		return -1, err
	}

	return newMigrationStartVersion, nil
}
<|endoftext|>


<commit_before>package stream

import (
	"fmt"
	"sync"
	"time"

	"github.com/Code-Hex/go-generics-cache/policy/simple"

	"nimona.io/pkg/context"
	"nimona.io/pkg/did"
	"nimona.io/pkg/network"
	"nimona.io/pkg/object"
	"nimona.io/pkg/sqlobjectstore"
	"nimona.io/pkg/tilde"
)

var ErrInvalidRoot = fmt.Errorf("root object doesn't match stream's hash")

type (
	controller struct {
		lock sync.RWMutex
		// services
		network     network.Network
		objectStore *sqlobjectstore.Store
		// dag graph
		graph *Graph[tilde.Digest, object.Metadata]
		// state, not thread safe
		streamInfo *StreamInfo
		// subscriptions
		subscriptions *simple.Cache[did.DID, bool]
	}
)

func NewController(
	cid tilde.Digest,
	network network.Network,
	objectStore *sqlobjectstore.Store,
) Controller {
	c := &controller{
		graph:         NewGraph[tilde.Digest, object.Metadata](),
		network:       network,
		objectStore:   objectStore,
		streamInfo:    NewStreamInfo(),
		subscriptions: simple.NewCache[did.DID, bool](),
	}
	c.streamInfo.RootDigest = cid
	return c
}

// Insert an event to the stream.
// Can either accept an Object, or anything that can be marshalled into one.
// This method will make any necessary changes to the object to make it valid.
// - Will set the object's root if it's not set
// - Will set the object's parents if they are not set
// - Will set the object's sequence if it's not set
// Returns the updated object's hash.
func (s *controller) Insert(v interface{}) (tilde.Digest, error) {
	var o *object.Object
	switch vv := v.(type) {
	case *object.Object:
		o = vv
	default:
		var err error
		o, err = object.Marshal(o)
		if err != nil {
			return tilde.EmptyDigest, fmt.Errorf("failed to marshal object: %w", err)
		}
	}

	// verify that the object has the basic metadata
	if o.Type == "" {
		return tilde.EmptyDigest, fmt.Errorf("object type is required")
	}

	// if the object has no root, set it to the stream root
	if o.Metadata.Root.IsEmpty() && s.streamInfo.RootObject == nil {
		err := s.Apply(o)
		if err != nil {
			return tilde.EmptyDigest, fmt.Errorf("failed to apply object: %w", err)
		}
		return o.Hash(), nil
	}

	// verify or set the object's root
	if o.Metadata.Root.IsEmpty() {
		o.Metadata.Root = s.streamInfo.RootDigest
	} else if !o.Metadata.Root.Equal(s.streamInfo.RootDigest) {
		return tilde.EmptyDigest, fmt.Errorf("roots don't match")
	}

	// verify or set the object's parents
	if len(o.Metadata.Parents) == 0 {
		ps := s.graph.GetLeaves()
		if len(ps) > 0 {
			o.Metadata.Parents = object.Parents{
				"*": ps,
			}
		}
	}

	// gather all nodes until the graph's root
	pns := map[tilde.Digest]struct{}{}
	for _, pn := range o.Metadata.Parents.All() {
		pns[pn] = struct{}{}
		ps := s.graph.nodesToRoot(pn)
		for _, p := range ps {
			pns[p] = struct{}{}
		}
	}

	// verify or set the object's sequence
	if o.Metadata.Sequence == 0 {
		o.Metadata.Sequence = uint64(len(pns))
	}

	// get the object's hash
	h := o.Hash()

	// apply the event
	err := s.Apply(o)
	if err != nil {
		return tilde.EmptyDigest, fmt.Errorf("failed to apply object: %w", err)
	}

	// TODO: figure out how to move announcements to first-time applies

	// announce the event to subscribers
	subscribers := s.subscriptions.Keys()
	if len(subscribers) == 0 {
		return h, nil
	}

	announcement := &Announcement{
		Metadata: object.Metadata{
			Owner: s.network.GetPeerKey().PublicKey().DID(),
		},
		StreamHash:   s.streamInfo.RootDigest,
		ObjectHashes: []tilde.Digest{h},
	}

	announcementObject, err := object.Marshal(announcement)
	if err != nil {
		return tilde.EmptyDigest, fmt.Errorf("failed to marshal announcement: %w", err)
	}

	for _, sub := range subscribers {
		err := s.network.Send(
			context.New(
				context.WithTimeout(time.Second*2),
			),
			announcementObject,
			sub,
		)
		if err != nil {
			// TODO: log error
			continue
		}
	}

	return h, nil
}

// Apply an event to the stream.
// Can either accept an Object, or anything that can be marshalled into one.
func (s *controller) Apply(v interface{}) error {
	s.lock.Lock()
	defer s.lock.Unlock()

	var o *object.Object
	switch vv := v.(type) {
	case *object.Object:
		o = vv
	default:
		var err error
		o, err = object.Marshal(o)
		if err != nil {
			return fmt.Errorf("failed to marshal object: %w", err)
		}
	}

	// verify that the object has the basic metadata
	if o.Type == "" {
		return fmt.Errorf("object type is required")
	}

	// verify that the object has not been applied already
	digest := o.Hash()
	_, ok := s.streamInfo.Objects[digest]
	if ok {
		return nil
	}

	// check if we're applying the root object
	if o.Metadata.Root.IsEmpty() && s.streamInfo.RootObject == nil {
		h := o.Hash()
		// check the object's hash against the stream root
		if !h.Equal(s.streamInfo.RootDigest) {
			return ErrInvalidRoot
		}
		// store the object
		err := s.objectStore.Put(o)
		if err != nil {
			return fmt.Errorf("failed to store object: %w", err)
		}
		// update stream info
		s.streamInfo.RootType = o.Type
		s.streamInfo.RootObject = o
		// add the root to the graph
		s.graph.Add(digest, o.Metadata, nil)
		// add the object to the metadata list
		oi := GetObjectInfo(o)
		s.streamInfo.Objects[digest] = oi
		// return
		return nil
	}

	// verify the object's root
	if !o.Metadata.Root.Equal(s.streamInfo.RootDigest) {
		return fmt.Errorf("roots don't match")
	}

	// verify the object's parents
	if len(o.Metadata.Parents) == 0 {
		return fmt.Errorf("object has no parents")
	}

	// verify the object's sequence
	if o.Metadata.Sequence == 0 {
		return fmt.Errorf("object has no sequence")
	}

	// store the object
	err := s.objectStore.Put(o)
	if err != nil {
		return fmt.Errorf("failed to store object: %w", err)
	}

	// add it to the graph
	s.graph.Add(digest, o.Metadata, o.Metadata.Parents.All())

	// add the object to the metadata list
	oi := GetObjectInfo(o)
	s.streamInfo.Objects[oi.Digest] = oi

	// handle special objects
	switch o.Type {
	case SubscriptionType:
		// add the subscription to the subscriptions list
		sub := &Subscription{}
		err := object.Unmarshal(o, sub)
		// in case of error, just move on
		if err != nil {
			break
		}
		if sub.Metadata.Owner.IsEmpty() {
			break
		}
		s.subscriptions.Set(sub.Metadata.Owner, true)
	}

	return nil
}

func (s *controller) GetStreamInfo() StreamInfo {
	// TODO lock and copy
	return *s.streamInfo
}

func (s *controller) GetObjectDigests() ([]tilde.Digest, error) {
	// TODO lock
	return s.graph.TopologicalSort()
}

func (s *controller) GetStreamRoot() tilde.Digest {
	return tilde.EmptyDigest
}

func (s *controller) GetDigests() ([]tilde.Digest, error) {
	return s.graph.TopologicalSort()
}

func (s *controller) GetReader(ctx context.Context) (object.ReadCloser, error) {
	os := make(chan *object.Object)
	er := make(chan error)
	cl := make(chan struct{})
	or := object.NewReadCloser(ctx, os, er, cl)
	ds, err := s.graph.TopologicalSort()
	if err != nil {
		return nil, err
	}
	go func() {
		defer close(os)
		defer close(er)
		defer close(cl)
		for _, d := range ds {
			o, err := s.objectStore.Get(d)
			if err != nil {
				er <- err
				return
			}
			select {
			case os <- o:
			case <-ctx.Done():
				return
			case <-cl:
				return
			}
		}
	}()
	return or, nil
}

func (s *controller) GetSubscribers() ([]did.DID, error) {
	s.lock.RLock()
	defer s.lock.RUnlock()
	return s.subscriptions.Keys(), nil
}

func (s *controller) ContainsDigest(cid tilde.Digest) bool {
	s.lock.RLock()
	defer s.lock.RUnlock()
	return s.graph.Contains(cid)
}
<commit_msg>fix(stream): fix GetStreamRoot<commit_after>package stream

import (
	"fmt"
	"sync"
	"time"

	"github.com/Code-Hex/go-generics-cache/policy/simple"

	"nimona.io/pkg/context"
	"nimona.io/pkg/did"
	"nimona.io/pkg/network"
	"nimona.io/pkg/object"
	"nimona.io/pkg/sqlobjectstore"
	"nimona.io/pkg/tilde"
)

var ErrInvalidRoot = fmt.Errorf("root object doesn't match stream's hash")

type (
	controller struct {
		lock sync.RWMutex
		// services
		network     network.Network
		objectStore *sqlobjectstore.Store
		// dag graph
		graph *Graph[tilde.Digest, object.Metadata]
		// state, not thread safe
		streamInfo *StreamInfo
		// subscriptions
		subscriptions *simple.Cache[did.DID, bool]
	}
)

func NewController(
	cid tilde.Digest,
	network network.Network,
	objectStore *sqlobjectstore.Store,
) Controller {
	c := &controller{
		graph:         NewGraph[tilde.Digest, object.Metadata](),
		network:       network,
		objectStore:   objectStore,
		streamInfo:    NewStreamInfo(),
		subscriptions: simple.NewCache[did.DID, bool](),
	}
	c.streamInfo.RootDigest = cid
	return c
}

// Insert an event to the stream.
// Can either accept an Object, or anything that can be marshalled into one.
// This method will make any necessary changes to the object to make it valid.
// - Will set the object's root if it's not set
// - Will set the object's parents if they are not set
// - Will set the object's sequence if it's not set
// Returns the updated object's hash.
func (s *controller) Insert(v interface{}) (tilde.Digest, error) {
	var o *object.Object
	switch vv := v.(type) {
	case *object.Object:
		o = vv
	default:
		var err error
		o, err = object.Marshal(o)
		if err != nil {
			return tilde.EmptyDigest, fmt.Errorf("failed to marshal object: %w", err)
		}
	}

	// verify that the object has the basic metadata
	if o.Type == "" {
		return tilde.EmptyDigest, fmt.Errorf("object type is required")
	}

	// if the object has no root, set it to the stream root
	if o.Metadata.Root.IsEmpty() && s.streamInfo.RootObject == nil {
		err := s.Apply(o)
		if err != nil {
			return tilde.EmptyDigest, fmt.Errorf("failed to apply object: %w", err)
		}
		return o.Hash(), nil
	}

	// verify or set the object's root
	if o.Metadata.Root.IsEmpty() {
		o.Metadata.Root = s.streamInfo.RootDigest
	} else if !o.Metadata.Root.Equal(s.streamInfo.RootDigest) {
		return tilde.EmptyDigest, fmt.Errorf("roots don't match")
	}

	// verify or set the object's parents
	if len(o.Metadata.Parents) == 0 {
		ps := s.graph.GetLeaves()
		if len(ps) > 0 {
			o.Metadata.Parents = object.Parents{
				"*": ps,
			}
		}
	}

	// gather all nodes until the graph's root
	pns := map[tilde.Digest]struct{}{}
	for _, pn := range o.Metadata.Parents.All() {
		pns[pn] = struct{}{}
		ps := s.graph.nodesToRoot(pn)
		for _, p := range ps {
			pns[p] = struct{}{}
		}
	}

	// verify or set the object's sequence
	if o.Metadata.Sequence == 0 {
		o.Metadata.Sequence = uint64(len(pns))
	}

	// get the object's hash
	h := o.Hash()

	// apply the event
	err := s.Apply(o)
	if err != nil {
		return tilde.EmptyDigest, fmt.Errorf("failed to apply object: %w", err)
	}

	// TODO: figure out how to move announcements to first-time applies

	// announce the event to subscribers
	subscribers := s.subscriptions.Keys()
	if len(subscribers) == 0 {
		return h, nil
	}

	announcement := &Announcement{
		Metadata: object.Metadata{
			Owner: s.network.GetPeerKey().PublicKey().DID(),
		},
		StreamHash:   s.streamInfo.RootDigest,
		ObjectHashes: []tilde.Digest{h},
	}

	announcementObject, err := object.Marshal(announcement)
	if err != nil {
		return tilde.EmptyDigest, fmt.Errorf("failed to marshal announcement: %w", err)
	}

	for _, sub := range subscribers {
		err := s.network.Send(
			context.New(
				context.WithTimeout(time.Second*2),
			),
			announcementObject,
			sub,
		)
		if err != nil {
			// TODO: log error
			continue
		}
	}

	return h, nil
}

// Apply an event to the stream.
// Can either accept an Object, or anything that can be marshalled into one.
func (s *controller) Apply(v interface{}) error {
	s.lock.Lock()
	defer s.lock.Unlock()

	var o *object.Object
	switch vv := v.(type) {
	case *object.Object:
		o = vv
	default:
		var err error
		o, err = object.Marshal(o)
		if err != nil {
			return fmt.Errorf("failed to marshal object: %w", err)
		}
	}

	// verify that the object has the basic metadata
	if o.Type == "" {
		return fmt.Errorf("object type is required")
	}

	// verify that the object has not been applied already
	digest := o.Hash()
	_, ok := s.streamInfo.Objects[digest]
	if ok {
		return nil
	}

	// check if we're applying the root object
	if o.Metadata.Root.IsEmpty() && s.streamInfo.RootObject == nil {
		h := o.Hash()
		// check the object's hash against the stream root
		if !h.Equal(s.streamInfo.RootDigest) {
			return ErrInvalidRoot
		}
		// store the object
		err := s.objectStore.Put(o)
		if err != nil {
			return fmt.Errorf("failed to store object: %w", err)
		}
		// update stream info
		s.streamInfo.RootType = o.Type
		s.streamInfo.RootObject = o
		// add the root to the graph
		s.graph.Add(digest, o.Metadata, nil)
		// add the object to the metadata list
		oi := GetObjectInfo(o)
		s.streamInfo.Objects[digest] = oi
		// return
		return nil
	}

	// verify the object's root
	if !o.Metadata.Root.Equal(s.streamInfo.RootDigest) {
		return fmt.Errorf("roots don't match")
	}

	// verify the object's parents
	if len(o.Metadata.Parents) == 0 {
		return fmt.Errorf("object has no parents")
	}

	// verify the object's sequence
	if o.Metadata.Sequence == 0 {
		return fmt.Errorf("object has no sequence")
	}

	// store the object
	err := s.objectStore.Put(o)
	if err != nil {
		return fmt.Errorf("failed to store object: %w", err)
	}

	// add it to the graph
	s.graph.Add(digest, o.Metadata, o.Metadata.Parents.All())

	// add the object to the metadata list
	oi := GetObjectInfo(o)
	s.streamInfo.Objects[oi.Digest] = oi

	// handle special objects
	switch o.Type {
	case SubscriptionType:
		// add the subscription to the subscriptions list
		sub := &Subscription{}
		err := object.Unmarshal(o, sub)
		// in case of error, just move on
		if err != nil {
			break
		}
		if sub.Metadata.Owner.IsEmpty() {
			break
		}
		s.subscriptions.Set(sub.Metadata.Owner, true)
	}

	return nil
}

func (s *controller) GetStreamInfo() StreamInfo {
	// TODO lock and copy
	return *s.streamInfo
}

func (s *controller) GetObjectDigests() ([]tilde.Digest, error) {
	// TODO lock
	return s.graph.TopologicalSort()
}

func (s *controller) GetStreamRoot() tilde.Digest {
	return s.streamInfo.RootDigest
}

func (s *controller) GetDigests() ([]tilde.Digest, error) {
	return s.graph.TopologicalSort()
}

func (s *controller) GetReader(ctx context.Context) (object.ReadCloser, error) {
	os := make(chan *object.Object)
	er := make(chan error)
	cl := make(chan struct{})
	or := object.NewReadCloser(ctx, os, er, cl)
	ds, err := s.graph.TopologicalSort()
	if err != nil {
		return nil, err
	}
	go func() {
		defer close(os)
		defer close(er)
		defer close(cl)
		for _, d := range ds {
			o, err := s.objectStore.Get(d)
			if err != nil {
				er <- err
				return
			}
			select {
			case os <- o:
			case <-ctx.Done():
				return
			case <-cl:
				return
			}
		}
	}()
	return or, nil
}

func (s *controller) GetSubscribers() ([]did.DID, error) {
	s.lock.RLock()
	defer s.lock.RUnlock()
	return s.subscriptions.Keys(), nil
}

func (s *controller) ContainsDigest(cid tilde.Digest) bool {
	s.lock.RLock()
	defer s.lock.RUnlock()
	return s.graph.Contains(cid)
}
<|endoftext|>


<commit_before>package dbTest

import (
	"fmt"
	"testing"

	"time"

	"github.com/Everlag/poeitemstore/cmd"
	"github.com/Everlag/poeitemstore/db"
	"github.com/Everlag/poeitemstore/stash"
	"github.com/boltdb/bolt"
)

// MultiModSearchToIndexQuery converts a MultiModSearch
// into an IndexQuery. It also returns the league because
// you usually need that...
func MultiModSearchToIndexQuery(search cmd.MultiModSearch,
	bdb *bolt.DB, t *testing.T) (db.IndexQuery, db.LeagueHeapID) {

	if len(search.MinValues) != len(search.Mods) {
		t.Fatalf("each mod must have a minvalue")
	}

	// Lookup the root, flavor, and mod
	strings := []string{search.RootType, search.RootFlavor}
	ids, err := db.GetStrings(strings, bdb)
	if err != nil {
		t.Fatalf("failed to fetch rootType or RootFlavor id, err=%s\n", err)
	}
	modIds, err := db.GetStrings(search.Mods, bdb)
	if err != nil {
		t.Fatalf("failed to fetch mod id, err=%s\n", err)
	}

	// And we we need to fetch the league
	leagueIDs, err := db.GetLeagues([]string{search.League}, bdb)
	if err != nil {
		t.Fatalf("failed to fetch league, err=%s\n", err)
	}

	return db.NewIndexQuery(ids[0], ids[1],
		modIds, search.MinValues, leagueIDs[0], search.MaxDesired), leagueIDs[0]

}

// QueryResultsToItems converts the provided IDs to their
// inflated stash forms
func QueryResultsToItems(ids []db.ID, league db.LeagueHeapID,
	bdb *bolt.DB, t *testing.T) []stash.Item {

	// Fetch the items so we can grab their GGGIDs
	compact := make([]db.Item, 0)
	err := bdb.View(func(tx *bolt.Tx) error {
		for _, id := range ids {
			item, err := db.GetItemByID(id, league, tx)
			if err != nil {
				return fmt.Errorf("failed to find item, err=%s", err)
			}
			compact = append(compact, item)
		}

		return nil
	})
	if err != nil {
		t.Fatalf("failed to find queried item in database")
	}
	foundItems := make([]stash.Item, 0)
	for _, tiny := range compact {
		foundItems = append(foundItems, tiny.Inflate(bdb))
	}

	return foundItems
}

// IndexQueryWithResultsToItemStoreQuery converts a MultiModSearch
// into an ItemStoreQuery while attempting to preserve the semantics
// of an IndexQuery in the resulting ItemStoreQuery
//
// IndexQuery has results ordered by highest values
// while ItemStoreQuery has results ordered by latest additions
// with minimum values.
func IndexQueryWithResultsToItemStoreQuery(search cmd.MultiModSearch,
	prevResults []stash.Item,
	bdb *bolt.DB, t *testing.T) db.ItemStoreQuery {

	if len(search.MinValues) != len(search.Mods) {
		t.Fatalf("each mod must have a minvalue")
	}

	// Setup a interestedMap so we can have constant time lookup
	// for which mods we are interesed in
	interestedMap := make(map[string]struct{})
	for _, mod := range search.Mods {
		interestedMap[mod] = struct{}{}
	}

	// Setup the minValue map, this will determine the real minimum
	// values which the ItemStoreQuery will need to find
	minValueMap := make(map[string]uint16)
	for _, item := range prevResults {
		for _, mod := range item.GetMods() {
			// Check if we are about this mod
			_, ok := interestedMap[string(mod.Template)]
			if !ok {
				continue
			}

			// Update the minValues as necessary
			prev, ok := minValueMap[string(mod.Template)]
			if !ok {
				prev = mod.Values[0]
			}
			if prev >= mod.Values[0] {
				minValueMap[string(mod.Template)] = mod.Values[0]
			}
		}
	}

	// Overwrite the search with the new minimum values
	prevLength := len(search.Mods) // Store old length for later
	search.Mods = make([]string, 0)
	search.MinValues = make([]uint16, 0)
	for mod, min := range minValueMap {
		search.Mods = append(search.Mods, mod)
		search.MinValues = append(search.MinValues, min)
	}
	if len(search.Mods) != prevLength {
		t.Fatalf("bad ItemStoreQuery translation: mismatched #mods")
	}

	t.Logf("Generated ItemStoreQuery:\n %s", search.String())

	// Lookup the root, flavor, and mod
	strings := []string{search.RootType, search.RootFlavor}
	ids, err := db.GetStrings(strings, bdb)
	if err != nil {
		t.Fatalf("failed to fetch rootType or RootFlavor id, err=%s\n", err)
	}
	modIds, err := db.GetStrings(search.Mods, bdb)
	if err != nil {
		t.Fatalf("failed to fetch mod id, err=%s\n", err)
	}

	// And we we need to fetch the league
	leagueIDs, err := db.GetLeagues([]string{search.League}, bdb)
	if err != nil {
		t.Fatalf("failed to fetch league, err=%s\n", err)
	}

	return db.NewItemStoreQuery(ids[0], ids[1],
		modIds, search.MinValues, leagueIDs[0], search.MaxDesired)

}

// Test as searching across multiple stash updates
func TestIndexQuery48UpdatesMovespeedFireResist(t *testing.T) {

	// t.Parallel()

	bdb := NewTempDatabase(t)

	// Define our search up here, it will be constant for all of
	// our sub-tests
	search := cmd.MultiModSearch{
		MaxDesired: 4,
		RootType:   "Armour",
		RootFlavor: "Boots",
		League:     "Legacy",
		Mods: []string{
			"#% increased Movement Speed",
			"+#% to Fire Resistance",
		},
		MinValues: []uint16{
			24,
			27,
		},
	}

	// Fetch the changes we need
	set := GetChangeSet("data/tesetSet - 24 updates.msgp", t)
	if len(set.Changes) != 24 {
		t.Fatalf("wrong number of changes, expected 24 got %d",
			len(set.Changes))
	}

	inverter := GetChangeSetInverter(set)

	for i, comp := range set.Changes {
		id := inverter[i]
		start := time.Now()
		resp, err := comp.Decompress()
		if err != nil {
			t.Fatalf("failed to decompress stash.Compressed, changeID=%s err=%s",
				id, err)
		}

		t.Logf("processing changeID=%s", id)

		cStashes, cItems, err := db.StashStashToCompact(resp.Stashes, bdb)
		if err != nil {
			t.Fatalf("failed to convert fat stashes to compact, err=%s\n", err)
		}

		_, err = db.AddStashes(cStashes, cItems, bdb)
		if err != nil {
			t.Fatalf("failed to AddStashes, err=%s", err)
		}

		// Translate the query now, after we are more likely
		// to have the desired mods available on the StringHeap
		indexQuery, league := MultiModSearchToIndexQuery(search, bdb, t)

		indexResult, err := indexQuery.Run(bdb)
		if err != nil {
			t.Fatalf("failed IndexQuery.Run, err=%s", err)
		}

		// Inflate items for easier mod checking
		indexItems := QueryResultsToItems(indexResult, league, bdb, t)

		// Convert to equivalent ItemStoreQuery
		itemStoreQuery := IndexQueryWithResultsToItemStoreQuery(search,
			indexItems, bdb, t)

		// Perform the ItemStoreQuery to generate reference
		itemStoreResults, err := itemStoreQuery.Run(bdb)
		if err != nil {
			t.Fatalf("failed ItemStoreQuery.Run, err=%s", err)
		}

		// Translate :|
		itemStoreResultsGGG := IDsToGGGID(itemStoreResults, league, bdb, t)

		// Ensure they match
		CompareQueryResultsToExpected(indexResult, league, itemStoreResultsGGG,
			bdb, t)

		end := time.Now()
		t.Logf("%d stashes, took %s", len(resp.Stashes), end.Sub(start))
	}

}
<commit_msg>dbTest simplify IndexQueryWithResultsToItemStoreQuery<commit_after>package dbTest

import (
	"fmt"
	"testing"

	"github.com/Everlag/poeitemstore/cmd"
	"github.com/Everlag/poeitemstore/db"
	"github.com/Everlag/poeitemstore/stash"
	"github.com/boltdb/bolt"
)

// MultiModSearchToIndexQuery converts a MultiModSearch
// into an IndexQuery. It also returns the league because
// you usually need that...
func MultiModSearchToIndexQuery(search cmd.MultiModSearch,
	bdb *bolt.DB, t *testing.T) (db.IndexQuery, db.LeagueHeapID) {

	if len(search.MinValues) != len(search.Mods) {
		t.Fatalf("each mod must have a minvalue")
	}

	// Lookup the root, flavor, and mod
	strings := []string{search.RootType, search.RootFlavor}
	ids, err := db.GetStrings(strings, bdb)
	if err != nil {
		t.Fatalf("failed to fetch rootType or RootFlavor id, err=%s\n", err)
	}
	modIds, err := db.GetStrings(search.Mods, bdb)
	if err != nil {
		t.Fatalf("failed to fetch mod id, err=%s\n", err)
	}

	// And we we need to fetch the league
	leagueIDs, err := db.GetLeagues([]string{search.League}, bdb)
	if err != nil {
		t.Fatalf("failed to fetch league, err=%s\n", err)
	}

	return db.NewIndexQuery(ids[0], ids[1],
		modIds, search.MinValues, leagueIDs[0], search.MaxDesired), leagueIDs[0]

}

// QueryResultsToItems converts the provided IDs to their
// inflated stash forms
func QueryResultsToItems(ids []db.ID, league db.LeagueHeapID,
	bdb *bolt.DB, t *testing.T) []stash.Item {

	// Fetch the items so we can grab their GGGIDs
	compact := make([]db.Item, 0)
	err := bdb.View(func(tx *bolt.Tx) error {
		for _, id := range ids {
			item, err := db.GetItemByID(id, league, tx)
			if err != nil {
				return fmt.Errorf("failed to find item, err=%s", err)
			}
			compact = append(compact, item)
		}

		return nil
	})
	if err != nil {
		t.Fatalf("failed to find queried item in database")
	}
	foundItems := make([]stash.Item, 0)
	for _, tiny := range compact {
		foundItems = append(foundItems, tiny.Inflate(bdb))
	}

	return foundItems
}

// IndexQueryWithResultsToItemStoreQuery converts a MultiModSearch
// into an ItemStoreQuery while attempting to preserve the semantics
// of an IndexQuery in the resulting ItemStoreQuery
//
// IndexQuery has results ordered by highest values
// while ItemStoreQuery has results ordered by latest additions
// with minimum values.
func IndexQueryWithResultsToItemStoreQuery(search cmd.MultiModSearch,
	prevResults []stash.Item,
	bdb *bolt.DB, t *testing.T) db.ItemStoreQuery {

	if len(search.MinValues) != len(search.Mods) {
		t.Fatalf("each mod must have a minvalue")
	}

	// Setup a interestedMap so we can have constant time lookup
	// for which mods we are interesed in
	interestedMap := make(map[string]struct{})
	for _, mod := range search.Mods {
		interestedMap[mod] = struct{}{}
	}

	// Setup the minValue map, this will determine the real minimum
	// values which the ItemStoreQuery will need to find
	minValueMap := make(map[string]uint16)
	for _, item := range prevResults {
		for _, mod := range item.GetMods() {
			// Check if we are about this mod
			_, ok := interestedMap[string(mod.Template)]
			if !ok {
				continue
			}

			// Update the minValues as necessary
			prev, ok := minValueMap[string(mod.Template)]
			if !ok {
				prev = mod.Values[0]
			}
			if prev >= mod.Values[0] {
				minValueMap[string(mod.Template)] = mod.Values[0]
			}
		}
	}

	// Overwrite the search with the new minimum values
	prevLength := len(search.Mods) // Store old length for later
	search.Mods = make([]string, 0)
	search.MinValues = make([]uint16, 0)
	for mod, min := range minValueMap {
		search.Mods = append(search.Mods, mod)
		search.MinValues = append(search.MinValues, min)
	}
	if len(search.Mods) != prevLength {
		t.Fatalf("bad MultiModSearch translation: mismatched #mods")
	}

	t.Logf("Generated MultiModSearch:\n %s", search.String())

	itemStoreSearch, _ := MultiModSearchToItemStoreQuery(search, bdb, t)
	return itemStoreSearch

}

// Test as searching across multiple stash updates
func TestIndexQuery48UpdatesMovespeedFireResist(t *testing.T) {

	// t.Parallel()

	bdb := NewTempDatabase(t)

	// Define our search up here, it will be constant for all of
	// our sub-tests
	search := cmd.MultiModSearch{
		MaxDesired: 4,
		RootType:   "Armour",
		RootFlavor: "Boots",
		League:     "Legacy",
		Mods: []string{
			"#% increased Movement Speed",
			"+#% to Fire Resistance",
		},
		MinValues: []uint16{
			24,
			27,
		},
	}

	// Fetch the changes we need
	set := GetChangeSet("data/tesetSet - 24 updates.msgp", t)
	if len(set.Changes) != 24 {
		t.Fatalf("wrong number of changes, expected 24 got %d",
			len(set.Changes))
	}

	inverter := GetChangeSetInverter(set)

	for i, comp := range set.Changes {
		id := inverter[i]
		resp, err := comp.Decompress()
		if err != nil {
			t.Fatalf("failed to decompress stash.Compressed, changeID=%s err=%s",
				id, err)
		}

		t.Logf("processing changeID=%s", id)

		cStashes, cItems, err := db.StashStashToCompact(resp.Stashes, bdb)
		if err != nil {
			t.Fatalf("failed to convert fat stashes to compact, err=%s\n", err)
		}

		_, err = db.AddStashes(cStashes, cItems, bdb)
		if err != nil {
			t.Fatalf("failed to AddStashes, err=%s", err)
		}

		// Translate the query now, after we are more likely
		// to have the desired mods available on the StringHeap
		indexQuery, league := MultiModSearchToIndexQuery(search, bdb, t)

		indexResult, err := indexQuery.Run(bdb)
		if err != nil {
			t.Fatalf("failed IndexQuery.Run, err=%s", err)
		}

		// Inflate items for easier mod checking
		indexItems := QueryResultsToItems(indexResult, league, bdb, t)

		// Convert to equivalent ItemStoreQuery
		itemStoreQuery := IndexQueryWithResultsToItemStoreQuery(search,
			indexItems, bdb, t)

		// Perform the ItemStoreQuery to generate reference
		itemStoreResults, err := itemStoreQuery.Run(bdb)
		if err != nil {
			t.Fatalf("failed ItemStoreQuery.Run, err=%s", err)
		}

		// Translate :|
		itemStoreResultsGGG := IDsToGGGID(itemStoreResults, league, bdb, t)

		// Ensure they match
		CompareQueryResultsToExpected(indexResult, league, itemStoreResultsGGG,
			bdb, t)

	}

}
<|endoftext|>


<commit_before>/*
	Atlas types are used to define how to map Go values into refmt token streams.

	Atlas information may be autogenerated based on struct tags automatically,
	but you can also specify custom AtlasEntry info to use advanced features
	and define custom transformations.
*/
package atlas

import (
	"reflect"
)

type Atlas struct {
	// Map typeinfo to a static description of how that type should be handled.
	// (The internal machinery that will wield this information, and has memory of
	// progress as it does so, is configured using the AtlasEntry, but allocated separately.
	// The machinery is stateful and mutable; the AtlasEntry is not.)
	//
	// We use 'var rtid uintptr = reflect.ValueOf(rt).Pointer()' -- pointer of the
	// value of the reflect.Type info -- as an index.
	// This is both unique and correctly converges when recomputed, and much
	// faster to compare against than reflect.Type (which is an interface that
	// tends to contain fairly large structures).
	mappings map[uintptr]*AtlasEntry

	// Mapping of tag ints to atlasEntry for quick lookups when the
	// unmarshaller hits a tag.  Values are a subset of `mappings`.
	tagMappings map[int]*AtlasEntry
}

// Gets the AtlasEntry for a typeID.  Used by obj package, not meant for user facing.
func (atl Atlas) Get(rtid uintptr) (*AtlasEntry, bool) {
	ent, ok := atl.mappings[rtid]
	return ent, ok
}

// Gets the AtlasEntry for a tag int.  Used by obj package, not meant for user facing.
func (atl Atlas) GetEntryByTag(tag int) (*AtlasEntry, bool) {
	ent, ok := atl.tagMappings[tag]
	return ent, ok
}

/*
	The AtlasEntry is a declarative roadmap of what we should do for
	marshal and unmarshal of a single object, keyed by type.

	There are a lot of paths your mappings might want to take:

	  - For a struct type, you may simply want to specify some alternate keys, or some to leave out, etc.
	  - For an interface type, you probably want to specify one of our interface muxing strategies
	     with a mapping between enumstr:typeinfo (and, what to do if we get a struct we don't recognize).
	  - For a string, int, or other primitive, you don't need to say anything: defaults will DTRT.
	  - For a typedef'd string, int, or other primitive, you *still* don't need to say anything: but,
	     if you want custom behavior (say, transform the string to an int at the last second, and back again),
		 you can specify transformer functions for that.
	  - For a struct type that you want to turn into a whole different kind (like a string): use
	     those same transform functions.  (You'll no longer need a FieldMap.)
	  - For the most esoteric needs, you can fall all the way back to providing a custom MarshalMachine
	     (but avoid that; it's a lot of work, and one of these other transform methods should suffice).
*/
type AtlasEntry struct {
	// The reflect info of the type this morphism is regarding.
	Type reflect.Type

	// --------------------------------------------------------
	// The big escape valves: wanna map to some other kind completely?
	// --------------------------------------------------------

	// Transforms the value we reached by walking (the 'live' value -- which
	// must be of `this.Type`) into another value (the 'serialable' value --
	// which will be of `this.MarshalTransformTargetType`).
	//
	// The target type may be anything, even of a completely different Kind!
	//
	// This transform func runs first, then the resulting value is
	// serialized (by running through the path through Atlas again, so
	// chaining of transform funcs is supported, though not recommended).
	MarshalTransformFunc MarshalTransformFunc
	// The type of value we expect after using the MarshalTransformFunc.
	//
	// The match between transform func and target type should be checked
	// during construction of this AtlasEntry.
	MarshalTransformTargetType reflect.Type

	// Expects a different type (the 'serialable' value -- which will be of
	// 'this.UnmarshalTransformTargetType') than the value we reached by
	// walking (the 'live' value -- which must be of `this.Type`).
	//
	// The target type may be anything, even of a completely different Kind!
	//
	// The unmarshal of that target type will be run first, then the
	// resulting value is fed through this function to produce the real value,
	// which is then placed correctly into bigger mid-unmarshal object tree.
	//
	// For non-primitives, unmarshal of the target type will always target
	// an empty pointer or empty slice, roughly as per if it was
	// operating on a value produced by `TargetType.New()`.
	UnmarshalTransformFunc UnmarshalTransformFunc
	// The type of value we will manufacture an instance of and unmarshal
	// into, then when done provide to the UnmarshalTransformFunc.
	//
	// The match between transform func and target type should be checked
	// during construction of this AtlasEntry.
	UnmarshalTransformTargetType reflect.Type

	// --------------------------------------------------------
	// Standard options for how to map (varies by Kind)
	// --------------------------------------------------------

	// A "tag" to emit when marshalling this type of value;
	// and when unmarshalling, this tag will cause unmarshal to pick
	// this atlas (and if there's conflicting type info, error).
	Tag int
	// Flag for whether the Tag feature should be used (zero is a valid tag).
	Tagged bool

	// A mapping of fields in a struct to serial keys.
	// Only valid if `this.Type.Kind() == Struct`.
	StructMap *StructMap

	// FUTURE: enum-ish primitives, multiplexers for interfaces,
	//  lots of such things will belong here.

	// --------------------------------------------------------
	// Hooks, validate helpers
	// --------------------------------------------------------

	// A validation function which will be called for the whole value
	// after unmarshalling reached the end of the object.
	// If it returns an error, the entire unmarshal will error.
	//
	// Not used in marshalling.
	// Not reachable if an UnmarshalTransform is set.
	ValidateFn func(v interface{}) error
}
<commit_msg>obj/atlas: more package docs.<commit_after>/*
	Atlas types are used to define how to map Go values into refmt token streams.

	Atlas information may be autogenerated based on struct tags automatically,
	but you can also specify custom AtlasEntry info to use advanced features
	and define custom transformations.

	An Atlas is a collection of AtlasEntry (plus some internal indexing).
	Typical usage is to declare an AtlasEntry for your structs (often near by the
	struct definition), then

	Building an AtlasEntry for some type called `Formula` looks like this:

		atlas.BuildEntry(Formula{}).StructMap().Autogenerate().Complete()

	Building an AtlasEntry always starts with `atlas.BuildEntry(x)` where `x` is
	a dummy object used to convey type information.
	The next function in the chain declares what kind of behavior we're going
	to use to turn that type of object into its serial form.
	(In the above example, we're declaring that we want refmt to see the `Formula`
	type as a struct and traverse its fields.  There are many other options!)
	Subsequent functions are specific to what kind of walking and mapping we've
	chosen.  For struct walking, this may involve declaring fields and custom serial
	names to map them to; for a "Transform" we'd instead have to provide callbacks
	to do the transformation from the `Formula` type to some other type; etcetera.
	The final function in the chain is always called `Complete`, and returns
	a ready-to-use AtlasEntry.

	Building a complete Atlas for a whole suite of serializable types is as
	easy as putting a bunch of them together:

		atlas.Build(
			atlas.BuildEntry(Foo{}).StructMap().Autogenerate().Complete(),
			atlas.BuildEntry(Bar{}).StructMap().Autogenerate().Complete(),
			atlas.BuildEntry(Baz{}).StructMap().Autogenerate().Complete(),
		)

	You can put your entire protocol into one Atlas.
	It's also possible to build several different Atlases each with different
	sets of AtlasEntry.  This may be useful if you have a protocol where some
	messages are not valid during some phases of communication, and you would
	like to use the Atlas as a form of whitelisting for what can be
	marshalled/unmarshalled.
*/
package atlas

import (
	"reflect"
)

type Atlas struct {
	// Map typeinfo to a static description of how that type should be handled.
	// (The internal machinery that will wield this information, and has memory of
	// progress as it does so, is configured using the AtlasEntry, but allocated separately.
	// The machinery is stateful and mutable; the AtlasEntry is not.)
	//
	// We use 'var rtid uintptr = reflect.ValueOf(rt).Pointer()' -- pointer of the
	// value of the reflect.Type info -- as an index.
	// This is both unique and correctly converges when recomputed, and much
	// faster to compare against than reflect.Type (which is an interface that
	// tends to contain fairly large structures).
	mappings map[uintptr]*AtlasEntry

	// Mapping of tag ints to atlasEntry for quick lookups when the
	// unmarshaller hits a tag.  Values are a subset of `mappings`.
	tagMappings map[int]*AtlasEntry
}

// Gets the AtlasEntry for a typeID.  Used by obj package, not meant for user facing.
func (atl Atlas) Get(rtid uintptr) (*AtlasEntry, bool) {
	ent, ok := atl.mappings[rtid]
	return ent, ok
}

// Gets the AtlasEntry for a tag int.  Used by obj package, not meant for user facing.
func (atl Atlas) GetEntryByTag(tag int) (*AtlasEntry, bool) {
	ent, ok := atl.tagMappings[tag]
	return ent, ok
}

/*
	The AtlasEntry is a declarative roadmap of what we should do for
	marshal and unmarshal of a single object, keyed by type.

	There are a lot of paths your mappings might want to take:

	  - For a struct type, you may simply want to specify some alternate keys, or some to leave out, etc.
	  - For an interface type, you probably want to specify one of our interface muxing strategies
	     with a mapping between enumstr:typeinfo (and, what to do if we get a struct we don't recognize).
	  - For a string, int, or other primitive, you don't need to say anything: defaults will DTRT.
	  - For a typedef'd string, int, or other primitive, you *still* don't need to say anything: but,
	     if you want custom behavior (say, transform the string to an int at the last second, and back again),
		 you can specify transformer functions for that.
	  - For a struct type that you want to turn into a whole different kind (like a string): use
	     those same transform functions.  (You'll no longer need a FieldMap.)
	  - For the most esoteric needs, you can fall all the way back to providing a custom MarshalMachine
	     (but avoid that; it's a lot of work, and one of these other transform methods should suffice).
*/
type AtlasEntry struct {
	// The reflect info of the type this morphism is regarding.
	Type reflect.Type

	// --------------------------------------------------------
	// The big escape valves: wanna map to some other kind completely?
	// --------------------------------------------------------

	// Transforms the value we reached by walking (the 'live' value -- which
	// must be of `this.Type`) into another value (the 'serialable' value --
	// which will be of `this.MarshalTransformTargetType`).
	//
	// The target type may be anything, even of a completely different Kind!
	//
	// This transform func runs first, then the resulting value is
	// serialized (by running through the path through Atlas again, so
	// chaining of transform funcs is supported, though not recommended).
	MarshalTransformFunc MarshalTransformFunc
	// The type of value we expect after using the MarshalTransformFunc.
	//
	// The match between transform func and target type should be checked
	// during construction of this AtlasEntry.
	MarshalTransformTargetType reflect.Type

	// Expects a different type (the 'serialable' value -- which will be of
	// 'this.UnmarshalTransformTargetType') than the value we reached by
	// walking (the 'live' value -- which must be of `this.Type`).
	//
	// The target type may be anything, even of a completely different Kind!
	//
	// The unmarshal of that target type will be run first, then the
	// resulting value is fed through this function to produce the real value,
	// which is then placed correctly into bigger mid-unmarshal object tree.
	//
	// For non-primitives, unmarshal of the target type will always target
	// an empty pointer or empty slice, roughly as per if it was
	// operating on a value produced by `TargetType.New()`.
	UnmarshalTransformFunc UnmarshalTransformFunc
	// The type of value we will manufacture an instance of and unmarshal
	// into, then when done provide to the UnmarshalTransformFunc.
	//
	// The match between transform func and target type should be checked
	// during construction of this AtlasEntry.
	UnmarshalTransformTargetType reflect.Type

	// --------------------------------------------------------
	// Standard options for how to map (varies by Kind)
	// --------------------------------------------------------

	// A "tag" to emit when marshalling this type of value;
	// and when unmarshalling, this tag will cause unmarshal to pick
	// this atlas (and if there's conflicting type info, error).
	Tag int
	// Flag for whether the Tag feature should be used (zero is a valid tag).
	Tagged bool

	// A mapping of fields in a struct to serial keys.
	// Only valid if `this.Type.Kind() == Struct`.
	StructMap *StructMap

	// FUTURE: enum-ish primitives, multiplexers for interfaces,
	//  lots of such things will belong here.

	// --------------------------------------------------------
	// Hooks, validate helpers
	// --------------------------------------------------------

	// A validation function which will be called for the whole value
	// after unmarshalling reached the end of the object.
	// If it returns an error, the entire unmarshal will error.
	//
	// Not used in marshalling.
	// Not reachable if an UnmarshalTransform is set.
	ValidateFn func(v interface{}) error
}
<|endoftext|>


<commit_before>package routing

import (
	"encoding/json"
	"fmt"
	"os/exec"
	"strconv"
	"strings"
	"time"

	"github.com/cloudfoundry-incubator/cf-test-helpers/cf"
	"github.com/cloudfoundry-incubator/cf-test-helpers/generator"
	"github.com/cloudfoundry-incubator/cf-test-helpers/helpers"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"

	"testing"
)

type AppResource struct {
	Metadata struct {
		Url string
	}
}
type AppsResponse struct {
	Resources []AppResource
}
type Stat struct {
	Stats struct {
		Host string
		Port int
	}
}
type StatsResponse map[string]Stat

func PushApp(asset string) string {
	app := generator.PrefixedRandomName("RATS-APP-")
	Expect(cf.Cf("push", app, "-p", asset).Wait(CF_PUSH_TIMEOUT)).To(Exit(0))
	return app
}

func ScaleAppInstances(appName string, instances int) {
	Expect(cf.Cf("scale", appName, "-i", strconv.Itoa(instances)).Wait(DEFAULT_TIMEOUT)).To(Exit(0))
	Eventually(func() string {
		return string(cf.Cf("app", appName).Wait(DEFAULT_TIMEOUT).Out.Contents())
	}, DEFAULT_TIMEOUT*2, 2*time.Second).
		Should(ContainSubstring(fmt.Sprintf("instances: %d/%d", instances, instances)))
}

func DeleteApp(appName string) {
	Expect(cf.Cf("delete", appName, "-f").Wait(DEFAULT_TIMEOUT)).To(Exit(0))
}

func GetAppGuid(appName string) string {
	appGuid := cf.Cf("app", appName, "--guid").Wait(DEFAULT_TIMEOUT).Out.Contents()
	return strings.TrimSpace(string(appGuid))
}

func MapRouteToApp(domain, path, app string) {
	spaceGuid, domainGuid := GetSpaceAndDomainGuids(app)

	routeGuid := CreateRoute(domain, path, spaceGuid, domainGuid)
	appGuid := GetAppGuid(app)

	Expect(cf.Cf("curl", "/v2/apps/"+appGuid+"/routes/"+routeGuid, "-X", "PUT").Wait(CF_PUSH_TIMEOUT)).To(Exit(0))
}

func CreateRoute(domainName, contextPath, spaceGuid, domainGuid string) string {
	jsonBody := "{\"host\":\"" + domainName + "\", \"path\":\"" + contextPath + "\", \"domain_guid\":\"" + domainGuid + "\",\"space_guid\":\"" + spaceGuid + "\"}"
	routePostResponseBody := cf.Cf("curl", "/v2/routes", "-X", "POST", "-d", jsonBody).Wait(CF_PUSH_TIMEOUT).Out.Contents()

	var routeResponseJSON struct {
		Metadata struct {
			Guid string `json:"guid"`
		} `json:"metadata"`
	}
	json.Unmarshal([]byte(routePostResponseBody), &routeResponseJSON)
	return routeResponseJSON.Metadata.Guid
}

func GetSpaceAndDomainGuids(app string) (string, string) {
	getRoutePath := fmt.Sprintf("/v2/routes?q=host:%s", app)
	routeBody := cf.Cf("curl", getRoutePath).Wait(DEFAULT_TIMEOUT).Out.Contents()
	var routeJSON struct {
		Resources []struct {
			Entity struct {
				SpaceGuid  string `json:"space_guid"`
				DomainGuid string `json:"domain_guid"`
			} `json:"entity"`
		} `json:"resources"`
	}
	json.Unmarshal([]byte(routeBody), &routeJSON)

	spaceGuid := routeJSON.Resources[0].Entity.SpaceGuid
	domainGuid := routeJSON.Resources[0].Entity.DomainGuid

	return spaceGuid, domainGuid
}

func GetAppInfo(appName string) (host, port string) {
	var appsResponse AppsResponse
	var statsResponse StatsResponse

	cfResponse := cf.Cf("curl", fmt.Sprintf("/v2/apps?q=name:%s", appName)).Wait(DEFAULT_TIMEOUT).Out.Contents()
	json.Unmarshal(cfResponse, &appsResponse)
	serverAppUrl := appsResponse.Resources[0].Metadata.Url

	cfResponse = cf.Cf("curl", fmt.Sprintf("%s/stats", serverAppUrl)).Wait(DEFAULT_TIMEOUT).Out.Contents()
	json.Unmarshal(cfResponse, &statsResponse)

	appIp := statsResponse["0"].Stats.Host
	appPort := fmt.Sprintf("%d", statsResponse["0"].Stats.Port)
	return appIp, appPort
}

func RegisterRoute(appRoute string, ip string, port string, routeServiceName string) {
	systemDomain := config.SystemDomain
	oauthPassword := config.ClientSecret
	oauthUrl := config.Protocol() + "uaa." + systemDomain
	routingApiEndpoint := config.Protocol() + "routing-api." + systemDomain

	appsDomain := config.AppsDomain
	routeServiceRoute := config.Protocol() + routeServiceName + "." + appsDomain
	route := appRoute + "." + appsDomain
	routeJSON := `[{"route":"` + route + `","port":` + port + `,"ip":"` + ip + `","ttl":60, "route_service_url":"` + routeServiceRoute + `"}]`

	args := []string{"register", routeJSON, "--api", routingApiEndpoint, "--client-id", "gorouter", "--client-secret", oauthPassword, "--oauth-url", oauthUrl}
	session := Rtr(args...)
	Eventually(session.Out).Should(Say("Successfully registered routes"))
}

func Rtr(args ...string) *Session {
	session, err := Start(exec.Command("rtr", args...), GinkgoWriter, GinkgoWriter)
	Expect(err).NotTo(HaveOccurred())

	return session
}

const (
	DEFAULT_TIMEOUT = 30 * time.Second
	CF_PUSH_TIMEOUT = 2 * time.Minute
)

var config helpers.Config

func TestRouting(t *testing.T) {
	RegisterFailHandler(Fail)

	config = helpers.LoadConfig()

	componentName := "Routing"

	rs := []Reporter{}

	context := helpers.NewContext(config)
	environment := helpers.NewEnvironment(context)

	BeforeSuite(func() {
		Expect(config.SystemDomain).ToNot(Equal(""), "Must provide a system domain for the routing suite")
		Expect(config.ClientSecret).ToNot(Equal(""), "Must provide a client secret for the routing suite")
		environment.Setup()
	})

	AfterSuite(func() {
		environment.Teardown()
	})

	if config.ArtifactsDirectory != "" {
		helpers.EnableCFTrace(config, componentName)
		rs = append(rs, helpers.NewJUnitReporter(config, componentName))
	}

	RunSpecsWithDefaultAndCustomReporters(t, componentName, rs)
}
<commit_msg>Revert https changing to routing spec<commit_after>package routing

import (
	"encoding/json"
	"fmt"
	"os/exec"
	"strconv"
	"strings"
	"time"

	"github.com/cloudfoundry-incubator/cf-test-helpers/cf"
	"github.com/cloudfoundry-incubator/cf-test-helpers/generator"
	"github.com/cloudfoundry-incubator/cf-test-helpers/helpers"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"

	"testing"
)

type AppResource struct {
	Metadata struct {
		Url string
	}
}
type AppsResponse struct {
	Resources []AppResource
}
type Stat struct {
	Stats struct {
		Host string
		Port int
	}
}
type StatsResponse map[string]Stat

func PushApp(asset string) string {
	app := generator.PrefixedRandomName("RATS-APP-")
	Expect(cf.Cf("push", app, "-p", asset).Wait(CF_PUSH_TIMEOUT)).To(Exit(0))
	return app
}

func ScaleAppInstances(appName string, instances int) {
	Expect(cf.Cf("scale", appName, "-i", strconv.Itoa(instances)).Wait(DEFAULT_TIMEOUT)).To(Exit(0))
	Eventually(func() string {
		return string(cf.Cf("app", appName).Wait(DEFAULT_TIMEOUT).Out.Contents())
	}, DEFAULT_TIMEOUT*2, 2*time.Second).
		Should(ContainSubstring(fmt.Sprintf("instances: %d/%d", instances, instances)))
}

func DeleteApp(appName string) {
	Expect(cf.Cf("delete", appName, "-f").Wait(DEFAULT_TIMEOUT)).To(Exit(0))
}

func GetAppGuid(appName string) string {
	appGuid := cf.Cf("app", appName, "--guid").Wait(DEFAULT_TIMEOUT).Out.Contents()
	return strings.TrimSpace(string(appGuid))
}

func MapRouteToApp(domain, path, app string) {
	spaceGuid, domainGuid := GetSpaceAndDomainGuids(app)

	routeGuid := CreateRoute(domain, path, spaceGuid, domainGuid)
	appGuid := GetAppGuid(app)

	Expect(cf.Cf("curl", "/v2/apps/"+appGuid+"/routes/"+routeGuid, "-X", "PUT").Wait(CF_PUSH_TIMEOUT)).To(Exit(0))
}

func CreateRoute(domainName, contextPath, spaceGuid, domainGuid string) string {
	jsonBody := "{\"host\":\"" + domainName + "\", \"path\":\"" + contextPath + "\", \"domain_guid\":\"" + domainGuid + "\",\"space_guid\":\"" + spaceGuid + "\"}"
	routePostResponseBody := cf.Cf("curl", "/v2/routes", "-X", "POST", "-d", jsonBody).Wait(CF_PUSH_TIMEOUT).Out.Contents()

	var routeResponseJSON struct {
		Metadata struct {
			Guid string `json:"guid"`
		} `json:"metadata"`
	}
	json.Unmarshal([]byte(routePostResponseBody), &routeResponseJSON)
	return routeResponseJSON.Metadata.Guid
}

func GetSpaceAndDomainGuids(app string) (string, string) {
	getRoutePath := fmt.Sprintf("/v2/routes?q=host:%s", app)
	routeBody := cf.Cf("curl", getRoutePath).Wait(DEFAULT_TIMEOUT).Out.Contents()
	var routeJSON struct {
		Resources []struct {
			Entity struct {
				SpaceGuid  string `json:"space_guid"`
				DomainGuid string `json:"domain_guid"`
			} `json:"entity"`
		} `json:"resources"`
	}
	json.Unmarshal([]byte(routeBody), &routeJSON)

	spaceGuid := routeJSON.Resources[0].Entity.SpaceGuid
	domainGuid := routeJSON.Resources[0].Entity.DomainGuid

	return spaceGuid, domainGuid
}

func GetAppInfo(appName string) (host, port string) {
	var appsResponse AppsResponse
	var statsResponse StatsResponse

	cfResponse := cf.Cf("curl", fmt.Sprintf("/v2/apps?q=name:%s", appName)).Wait(DEFAULT_TIMEOUT).Out.Contents()
	json.Unmarshal(cfResponse, &appsResponse)
	serverAppUrl := appsResponse.Resources[0].Metadata.Url

	cfResponse = cf.Cf("curl", fmt.Sprintf("%s/stats", serverAppUrl)).Wait(DEFAULT_TIMEOUT).Out.Contents()
	json.Unmarshal(cfResponse, &statsResponse)

	appIp := statsResponse["0"].Stats.Host
	appPort := fmt.Sprintf("%d", statsResponse["0"].Stats.Port)
	return appIp, appPort
}

func RegisterRoute(appRoute string, ip string, port string, routeServiceName string) {
	systemDomain := config.SystemDomain
	oauthPassword := config.ClientSecret
	oauthUrl := config.Protocol() + "uaa." + systemDomain
	routingApiEndpoint := config.Protocol() + "routing-api." + systemDomain

	appsDomain := config.AppsDomain
	routeServiceRoute := "https://" + routeServiceName + "." + appsDomain
	route := appRoute + "." + appsDomain
	routeJSON := `[{"route":"` + route + `","port":` + port + `,"ip":"` + ip + `","ttl":60, "route_service_url":"` + routeServiceRoute + `"}]`

	args := []string{"register", routeJSON, "--api", routingApiEndpoint, "--client-id", "gorouter", "--client-secret", oauthPassword, "--oauth-url", oauthUrl}
	session := Rtr(args...)
	Eventually(session.Out).Should(Say("Successfully registered routes"))
}

func Rtr(args ...string) *Session {
	session, err := Start(exec.Command("rtr", args...), GinkgoWriter, GinkgoWriter)
	Expect(err).NotTo(HaveOccurred())

	return session
}

const (
	DEFAULT_TIMEOUT = 30 * time.Second
	CF_PUSH_TIMEOUT = 2 * time.Minute
)

var config helpers.Config

func TestRouting(t *testing.T) {
	RegisterFailHandler(Fail)

	config = helpers.LoadConfig()

	componentName := "Routing"

	rs := []Reporter{}

	context := helpers.NewContext(config)
	environment := helpers.NewEnvironment(context)

	BeforeSuite(func() {
		Expect(config.SystemDomain).ToNot(Equal(""), "Must provide a system domain for the routing suite")
		Expect(config.ClientSecret).ToNot(Equal(""), "Must provide a client secret for the routing suite")
		environment.Setup()
	})

	AfterSuite(func() {
		environment.Teardown()
	})

	if config.ArtifactsDirectory != "" {
		helpers.EnableCFTrace(config, componentName)
		rs = append(rs, helpers.NewJUnitReporter(config, componentName))
	}

	RunSpecsWithDefaultAndCustomReporters(t, componentName, rs)
}
<|endoftext|>