<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package os

import (
	"io"
	"runtime"
	"syscall"
)

// MkdirAll creates a directory named path,
// along with any necessary parents, and returns nil,
// or else returns an error.
// The permission bits perm (before umask) are used for all
// directories that MkdirAll creates.
// If path is already a directory, MkdirAll does nothing
// and returns nil.
func MkdirAll(path string, perm FileMode) error {
	// Fast path: if we can tell whether path is a directory or file, stop with success or error.
	dir, err := Stat(path)
	if err == nil {
		if dir.IsDir() {
			return nil
		}
		return &PathError{"mkdir", path, syscall.ENOTDIR}
	}

	// Slow path: make sure parent exists and then call Mkdir for path.
	i := len(path)
	for i > 0 && IsPathSeparator(path[i-1]) { // Skip trailing path separator.
		i--
	}

	j := i
	for j > 0 && !IsPathSeparator(path[j-1]) { // Scan backward over element.
		j--
	}

	if j > 1 {
		// Create parent.
		err = MkdirAll(fixRootDirectory(path[:j-1]), perm)
		if err != nil {
			return err
		}
	}

	// Parent now exists; invoke Mkdir and use its result.
	err = Mkdir(path, perm)
	if err != nil {
		// Handle arguments like "foo/." by
		// double-checking that directory doesn't exist.
		dir, err1 := Lstat(path)
		if err1 == nil && dir.IsDir() {
			return nil
		}
		return err
	}
	return nil
}

// RemoveAll removes path and any children it contains.
// It removes everything it can but returns the first error
// it encounters. If the path does not exist, RemoveAll
// returns nil (no error).
func RemoveAll(path string) error {
	// Simple case: if Remove works, we're done.
	err := Remove(path)
	if err == nil || IsNotExist(err) {
		return nil
	}

	// Otherwise, is this a directory we need to recurse into?
	dir, serr := Lstat(path)
	if serr != nil {
		if serr, ok := serr.(*PathError); ok && (IsNotExist(serr.Err) || serr.Err == syscall.ENOTDIR) {
			return nil
		}
		return serr
	}
	if !dir.IsDir() {
		// Not a directory; return the error from Remove.
		return err
	}

	// Directory.
	fd, err := Open(path)
	if err != nil {
		if IsNotExist(err) {
			// Race. It was deleted between the Lstat and Open.
			// Return nil per RemoveAll's docs.
			return nil
		}
		return err
	}

	// Remove contents & return first error.
	err = nil
	for {
		if err == nil && (runtime.GOOS == "plan9" || runtime.GOOS == "nacl") {
			// Reset read offset after removing directory entries.
			// See golang.org/issue/22572.
			fd.Seek(0, 0)
		}
		names, err1 := fd.Readdirnames(100)
		for _, name := range names {
			err1 := RemoveAll(path + string(PathSeparator) + name)
			if err == nil {
				err = err1
			}
		}
		if err1 == io.EOF {
			break
		}
		// If Readdirnames returned an error, use it.
		if err == nil {
			err = err1
		}
		if len(names) == 0 {
			break
		}
	}

	// Close directory, because windows won't remove opened directory.
	fd.Close()

	// Remove directory.
	err1 := Remove(path)
	if err1 == nil || IsNotExist(err1) {
		return nil
	}
	if err == nil {
		err = err1
	}
	return err
}
<commit_msg>os: when looping in RemoveAll, close and re-open directory<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package os

import (
	"io"
	"syscall"
)

// MkdirAll creates a directory named path,
// along with any necessary parents, and returns nil,
// or else returns an error.
// The permission bits perm (before umask) are used for all
// directories that MkdirAll creates.
// If path is already a directory, MkdirAll does nothing
// and returns nil.
func MkdirAll(path string, perm FileMode) error {
	// Fast path: if we can tell whether path is a directory or file, stop with success or error.
	dir, err := Stat(path)
	if err == nil {
		if dir.IsDir() {
			return nil
		}
		return &PathError{"mkdir", path, syscall.ENOTDIR}
	}

	// Slow path: make sure parent exists and then call Mkdir for path.
	i := len(path)
	for i > 0 && IsPathSeparator(path[i-1]) { // Skip trailing path separator.
		i--
	}

	j := i
	for j > 0 && !IsPathSeparator(path[j-1]) { // Scan backward over element.
		j--
	}

	if j > 1 {
		// Create parent.
		err = MkdirAll(fixRootDirectory(path[:j-1]), perm)
		if err != nil {
			return err
		}
	}

	// Parent now exists; invoke Mkdir and use its result.
	err = Mkdir(path, perm)
	if err != nil {
		// Handle arguments like "foo/." by
		// double-checking that directory doesn't exist.
		dir, err1 := Lstat(path)
		if err1 == nil && dir.IsDir() {
			return nil
		}
		return err
	}
	return nil
}

// RemoveAll removes path and any children it contains.
// It removes everything it can but returns the first error
// it encounters. If the path does not exist, RemoveAll
// returns nil (no error).
func RemoveAll(path string) error {
	// Simple case: if Remove works, we're done.
	err := Remove(path)
	if err == nil || IsNotExist(err) {
		return nil
	}

	// Otherwise, is this a directory we need to recurse into?
	dir, serr := Lstat(path)
	if serr != nil {
		if serr, ok := serr.(*PathError); ok && (IsNotExist(serr.Err) || serr.Err == syscall.ENOTDIR) {
			return nil
		}
		return serr
	}
	if !dir.IsDir() {
		// Not a directory; return the error from Remove.
		return err
	}

	// Remove contents & return first error.
	err = nil
	for {
		fd, err := Open(path)
		if err != nil {
			if IsNotExist(err) {
				// Already deleted by someone else.
				return nil
			}
			return err
		}

		const request = 1024
		names, err1 := fd.Readdirnames(request)

		// Removing files from the directory may have caused
		// the OS to reshuffle it. Simply calling Readdirnames
		// again may skip some entries. The only reliable way
		// to avoid this is to close and re-open the
		// directory. See issue 20841.
		fd.Close()

		for _, name := range names {
			err1 := RemoveAll(path + string(PathSeparator) + name)
			if err == nil {
				err = err1
			}
		}

		if err1 == io.EOF {
			break
		}
		// If Readdirnames returned an error, use it.
		if err == nil {
			err = err1
		}
		if len(names) == 0 {
			break
		}

		// We don't want to re-open unnecessarily, so if we
		// got fewer than request names from Readdirnames, try
		// simply removing the directory now. If that
		// succeeds, we are done.
		if len(names) < request {
			err1 := Remove(path)
			if err1 == nil || IsNotExist(err1) {
				return nil
			}

			if err != nil {
				// We got some error removing the
				// directory contents, and since we
				// read fewer names than we requested
				// there probably aren't more files to
				// remove. Don't loop around to read
				// the directory again. We'll probably
				// just get the same error.
				return err
			}
		}
	}

	// Remove directory.
	err1 := Remove(path)
	if err1 == nil || IsNotExist(err1) {
		return nil
	}
	if err == nil {
		err = err1
	}
	return err
}
<|endoftext|>


<commit_before>package location

import (
	"fmt"
	"net/http"

	"github.com/MG-RAST/Shock/shock-server/conf"
	e "github.com/MG-RAST/Shock/shock-server/errors"
	"github.com/MG-RAST/Shock/shock-server/logger"
	"github.com/MG-RAST/Shock/shock-server/node"
	"github.com/MG-RAST/Shock/shock-server/request"
	"github.com/MG-RAST/Shock/shock-server/responder"
	"github.com/MG-RAST/golib/stretchr/goweb/context"
	"gopkg.in/mgo.v2/bson"
)

// Load location
func Load(locID string) (loc *conf.LocationConfig, err error) {
	loc, ok := conf.LocationsMap[locID]
	if !ok {
		err = fmt.Errorf("(Location->Load) Location %s not found", locID)
		return
	}
	return
}

// GET, /location/{loc}/{function}, specify -H "Content-Type: application/json"
func LocRequest(ctx context.Context) {

	locationID := ctx.PathValue("loc")
	function := ctx.PathValue("function")

	fmt.Printf("LocRequest received locationID: %s, function: %s\n", locationID, function)
	logger.Debug(2, "(LocRequest) received locationID: %s, function: %s", locationID, function)

	rmeth := ctx.HttpRequest().Method

	u, err := request.Authenticate(ctx.HttpRequest())
	if err != nil && err.Error() != e.NoAuth {
		request.AuthError(err, ctx)
		return
	}

	// public user cannot use this
	if (u == nil) && conf.USE_AUTH {
		errMsg := "admin required"
		//errMsg := e.UnAuth
		responder.RespondWithError(ctx, http.StatusUnauthorized, errMsg)
		return
	}

	if (u != nil) && (!u.Admin) && conf.USE_AUTH {
		errMsg := e.UnAuth
		logger.Debug(2, "(LocRequest) attempt to use as non admin (user: %s)", u.Username)
		responder.RespondWithError(ctx, http.StatusInternalServerError, errMsg)
		return
	}

	if rmeth != "GET" {
		errMsg := fmt.Sprintf("(LocRequest) %s not supported", rmeth)
		responder.RespondWithError(ctx, http.StatusInternalServerError, errMsg)
		return
	}

	fmt.Printf("LocRequest passed auth bits and rmeth \n")

	locConf, err := Load(locationID)
	if err != nil {
		responder.RespondWithError(ctx, http.StatusInternalServerError, err.Error())
		fmt.Printf("LocRequest LOAD error \n")

		return
	}

	fmt.Printf("LocRequest worked \n")

	// ensure we only list nodes with Priority higher or equal to the one defined for the location

	MinPrio := locConf.MinPriority
	MinPrio = 0 // for debugging only

	// find Node Types with Priority > MinPrio
	nodes := node.Nodes{}
	matchesminprioquery := bson.M{"priority": bson.M{"$ge": MinPrio}}

	switch function {

	case "missing":
		// we ensure we only list nodes with Priority higher or equal to the one defined for the location
		nolocationquery := bson.M{"locations.id": bson.M{"$ne": locationID}}
		locationstoredfalsequery := bson.M{"locations.stored": bson.M{"$ne": "true"}}

		// check either node priority or priority of the data type
		// nodes with Priority encoded in JSON directly
		aquery := bson.M{"$and": []bson.M{nolocationquery, matchesminprioquery}}
		bquery := bson.M{"$and": []bson.M{locationstoredfalsequery, matchesminprioquery}}
		query := bson.M{"$and": []bson.M{aquery, bquery}}

		// nodes with no JSON priority but Attr.Type that has a priority
		nodes.GetAll(query)
		//spew.Dump(nodes)
		// list all nodes without Location set or marked as Location.stored==false  MongoDB
		responder.RespondWithData(ctx, nodes)
		return

	// 	list all nodes marked as stored==true in Location in MongoDB
	case "present":
		query := bson.M{"locations.stored": bson.M{"$ne": "true"}}
		nodes.GetAll(query)
		responder.RespondWithData(ctx, nodes)
		return

		// // list all nodes marked as Location.stored==false and priority
	case "inflight":
		locationstoredfalsequery := bson.M{"locations.stored": bson.M{"$eq": "false"}}
		query := bson.M{"$and": []bson.M{locationstoredfalsequery, matchesminprioquery}}
		nodes.GetAll(query)
		responder.RespondWithData(ctx, nodes)
		responder.RespondWithData(ctx, nodes)
		return

	//return config info for Node.
	case "info":
		// spew.Dump(locConf)
		responder.RespondWithData(ctx, locConf)
		return

	default:

		errMsg := fmt.Sprintf("(Location) %s not supported", function)
		responder.RespondWithError(ctx, http.StatusInternalServerError, errMsg)

	}

	// if locationID == "" { // /node/{nid}/locations
	// 	locations := n.GetLocations()
	// 	responder.RespondWithData(ctx, locations)
	// 	return
	// } else { // /node/{nid}/locations/{loc}
	// 	var thisLocation Location
	// 	thisLocation, err = n.GetLocation(locationID)
	// 	if err != nil {
	// 		responder.RespondWithError(ctx, http.StatusInternalServerError, err.Error())
	// 		return
	// 	}
	// 	responder.RespondWithData(ctx, thisLocation)
	// 	return

	// }

	// if locationID != "" { // /node/{nid}/locations/{loc}
	// 	err = n.DeleteLocation(locationID)
	// 	if err != nil {
	// 		responder.RespondWithError(ctx, http.StatusInternalServerError, err.Error())
	// 		return
	// 	}
	// 	n.Save()
	// 	responder.RespondWithData(ctx, n.Locations)
	// } else { // /node/{nid}/locations

	// 	err = n.DeleteLocations()
	// 	if err != nil {
	// 		responder.RespondWithError(ctx, http.StatusInternalServerError, err.Error())
	// 		return
	// 	}
	// 	n.Save()
	// 	responder.RespondOK(ctx)

	// }

	// }

	return
}
<commit_msg>query clean-up<commit_after>package location

import (
	"fmt"
	"net/http"

	"github.com/MG-RAST/Shock/shock-server/conf"
	e "github.com/MG-RAST/Shock/shock-server/errors"
	"github.com/MG-RAST/Shock/shock-server/logger"
	"github.com/MG-RAST/Shock/shock-server/node"
	"github.com/MG-RAST/Shock/shock-server/request"
	"github.com/MG-RAST/Shock/shock-server/responder"
	"github.com/MG-RAST/golib/stretchr/goweb/context"
	"gopkg.in/mgo.v2/bson"
)

// Load location
func Load(locID string) (loc *conf.LocationConfig, err error) {
	loc, ok := conf.LocationsMap[locID]
	if !ok {
		err = fmt.Errorf("(Location->Load) Location %s not found", locID)
		return
	}
	return
}

// GET, /location/{loc}/{function}, specify -H "Content-Type: application/json"
func LocRequest(ctx context.Context) {

	locationID := ctx.PathValue("loc")
	function := ctx.PathValue("function")

	fmt.Printf("LocRequest received locationID: %s, function: %s\n", locationID, function)
	logger.Debug(2, "(LocRequest) received locationID: %s, function: %s", locationID, function)

	rmeth := ctx.HttpRequest().Method

	u, err := request.Authenticate(ctx.HttpRequest())
	if err != nil && err.Error() != e.NoAuth {
		request.AuthError(err, ctx)
		return
	}

	// public user cannot use this
	if (u == nil) && conf.USE_AUTH {
		errMsg := "admin required"
		//errMsg := e.UnAuth
		responder.RespondWithError(ctx, http.StatusUnauthorized, errMsg)
		return
	}

	if (u != nil) && (!u.Admin) && conf.USE_AUTH {
		errMsg := e.UnAuth
		logger.Debug(2, "(LocRequest) attempt to use as non admin (user: %s)", u.Username)
		responder.RespondWithError(ctx, http.StatusInternalServerError, errMsg)
		return
	}

	if rmeth != "GET" {
		errMsg := fmt.Sprintf("(LocRequest) %s not supported", rmeth)
		responder.RespondWithError(ctx, http.StatusInternalServerError, errMsg)
		return
	}

	fmt.Printf("LocRequest passed auth bits and rmeth \n")

	locConf, err := Load(locationID)
	if err != nil {
		responder.RespondWithError(ctx, http.StatusInternalServerError, err.Error())
		fmt.Printf("LocRequest LOAD error \n")

		return
	}

	fmt.Printf("LocRequest worked \n")

	// ensure we only list nodes with Priority higher or equal to the one defined for the location

	MinPrio := locConf.MinPriority
	MinPrio = 0 // for debugging only

	// find Node Types with Priority > MinPrio
	nodes := node.Nodes{}
	matchesminprioquery := bson.M{"priority": bson.M{"$ge": MinPrio}}

	switch function {

	case "missing":

		// 1) find nodes that have specific locationID but are not stored
		locationStoredFalseQuery := bson.M{"locations": bson.M{"id": "anltsm", "stored": false}}

		// 2) array with locations but without specific location
		noLocationQuery := bson.M{"locations.id": bson.M{"$ne": locationID}}

		// 3) array is missing completely
		locationArrayMissing := bson.M{"locations": bson.M{"$exists": "false"}}

		// combine 1-3) we ensure we only list nodes with Priority higher or equal to the one defined for the location
		allNodesLocationMissingQuery := bson.M{"$or": []bson.M{locationStoredFalseQuery, noLocationQuery, locationArrayMissing}}

		// add priority
		missingNodesWithHighPriorityQuery := bson.M{"$and": []bson.M{allNodesLocationMissingQuery, matchesminprioquery}}

		//aquery := bson.M{"$and": []bson.M{nolocationquery, matchesminprioquery}}
		//bquery := bson.M{"$and": []bson.M{locationstoredfalsequery, matchesminprioquery}}
		//query := bson.M{"$and": []bson.M{aquery, bquery}}

		// nodes with no JSON priority but Attr.Type that has a priority
		nodes.GetAll(missingNodesWithHighPriorityQuery)
		//spew.Dump(nodes)
		// list all nodes without Location set or marked as Location.stored==false  MongoDB
		responder.RespondWithData(ctx, nodes)
		return

	// 	list all nodes marked as stored==true in Location in MongoDB
	case "present":
		query := bson.M{"locations.stored": bson.M{"$ne": "true"}}
		nodes.GetAll(query)
		responder.RespondWithData(ctx, nodes)
		return

		// // list all nodes marked as Location.stored==false and priority
	case "inflight":
		locationstoredfalsequery := bson.M{"locations.stored": bson.M{"$eq": "false"}}
		query := bson.M{"$and": []bson.M{locationstoredfalsequery, matchesminprioquery}}
		nodes.GetAll(query)
		responder.RespondWithData(ctx, nodes)
		responder.RespondWithData(ctx, nodes)
		return

	//return config info for Node.
	case "info":
		// spew.Dump(locConf)
		responder.RespondWithData(ctx, locConf)
		return

	default:

		errMsg := fmt.Sprintf("(Location) %s not supported", function)
		responder.RespondWithError(ctx, http.StatusInternalServerError, errMsg)

	}

	// if locationID == "" { // /node/{nid}/locations
	// 	locations := n.GetLocations()
	// 	responder.RespondWithData(ctx, locations)
	// 	return
	// } else { // /node/{nid}/locations/{loc}
	// 	var thisLocation Location
	// 	thisLocation, err = n.GetLocation(locationID)
	// 	if err != nil {
	// 		responder.RespondWithError(ctx, http.StatusInternalServerError, err.Error())
	// 		return
	// 	}
	// 	responder.RespondWithData(ctx, thisLocation)
	// 	return

	// }

	// if locationID != "" { // /node/{nid}/locations/{loc}
	// 	err = n.DeleteLocation(locationID)
	// 	if err != nil {
	// 		responder.RespondWithError(ctx, http.StatusInternalServerError, err.Error())
	// 		return
	// 	}
	// 	n.Save()
	// 	responder.RespondWithData(ctx, n.Locations)
	// } else { // /node/{nid}/locations

	// 	err = n.DeleteLocations()
	// 	if err != nil {
	// 		responder.RespondWithError(ctx, http.StatusInternalServerError, err.Error())
	// 		return
	// 	}
	// 	n.Save()
	// 	responder.RespondOK(ctx)

	// }

	// }

	return
}
<|endoftext|>


<commit_before>package smoke

import (
	"errors"
	"fmt"
	"os"
	"regexp"
	"strconv"
	"strings"
	"testing"
	"time"

	"path/filepath"

	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	utilerrors "k8s.io/apimachinery/pkg/util/errors"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/pkg/api/v1"
	"k8s.io/client-go/tools/clientcmd"
	cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"
	"k8s.io/kubernetes/pkg/kubectl/resource"
)

const (
	// networkingEnv is the environment variable that specifies if calico is running.
	networkingEnv = "SMOKE_NETWORKING"
	// nodeCountEnv is the environment variable that specifies the node count.
	nodeCountEnv = "SMOKE_NODE_COUNT"
	// manifestPathsEnv is the environment variable that defines the paths to the manifests that are deployed on the cluster.
	manifestPathsEnv = "SMOKE_MANIFEST_PATHS"
)

var (
	// defaultIgnoredManifests represents the manifests that are ignored by
	// testAllResourcesCreated by default.
	defaultIgnoredManifests = []string{
		"bootstrap",
		"kco-config.yaml",
		"tnco-config.yaml",
		// TODO: temporary disabling this for OpenTonic
		"tectonic/security/priviledged-scc-tectonic.yaml",
	}

	// equivalentKindRemapping is used by resourceIdentifier to map different
	// Kubernetes object kinds, that can be considered equivalent when checking
	// resource existence, with the same identifier.
	equivalentKindRemapping = map[string]string{
		"extensions/v1beta1:DaemonSet":  "extensions/v1beta1:DeploymentOrDaemonSet",
		"extensions/v1beta1:Deployment": "extensions/v1beta1:DeploymentOrDaemonSet",
		"apps/v1beta2:DaemonSet":        "apps/v1beta2:DeploymentOrDaemonSet",
		"apps/v1beta2:Deployment":       "apps/v1beta2:DeploymentOrDaemonSet",
	}

	// decodeErrorRegexp defines the format of the error returned by Kubernetes' resource mapper.
	decodeErrorRegexp = regexp.MustCompile(`unable to (?P<Type>decode|recognize) "(?P<Manifest>.*)": (?P<Message>.*)`)
)

func testCluster(t *testing.T) {
	// wait for all nodes to become available
	t.Run("AllNodesRunning", testAllNodesRunning)
	t.Run("AllResourcesCreated", testAllResourcesCreated)
	t.Run("AllPodsRunning", testAllPodsRunning)
	t.Run("KillAPIServer", testKillAPIServer)
}

func testAllPodsRunning(t *testing.T) {
	err := retry(allPodsRunning, t, 3*time.Second, 10*time.Minute)
	if err != nil {
		t.Fatalf("Timed out waiting for pods to be ready.")
	}
	t.Log("All pods are ready.")
}

func allPodsRunning(t *testing.T) error {
	err := checkPodsRunning(t)
	if err != nil {
		return err
	}

	// check one more time, because in case of crashloop we might get the transition
	time.Sleep(5 * time.Second)
	err = checkPodsRunning(t)
	if err != nil {
		return err
	}
	return nil
}

func checkPodsRunning(t *testing.T) error {
	c, _ := newClient(t)
	pods, err := c.Core().Pods("").List(meta_v1.ListOptions{})
	if err != nil {
		return fmt.Errorf("could not list pods: %v", err)
	}

	allReady := len(pods.Items) != 0
	for _, p := range pods.Items {
		if p.Status.Phase != v1.PodRunning || p.Status.ContainerStatuses[0].State.Running == nil {
			allReady = false
			t.Logf("pod %s/%s not running", p.Namespace, p.Name)
		}
	}

	if !allReady {
		return errors.New("pods are not all ready")
	}

	return nil
}

func allNodesRunning(expected int) func(t *testing.T) error {
	return func(t *testing.T) error {
		c, _ := newClient(t)
		nodes, err := c.Core().Nodes().List(meta_v1.ListOptions{})
		if err != nil {
			return fmt.Errorf("failed to list nodes: %v", err)
		}
		allReady := len(nodes.Items) != 0
		for _, node := range nodes.Items {
			if nodeReady(node) {
				t.Logf("node %s ready", node.Name)
				continue
			}
			allReady = false
			t.Logf("node %s not ready", node.Name)
		}
		if got := len(nodes.Items); got != expected {
			return fmt.Errorf("expected %d nodes, got %d", expected, got)
		}
		if !allReady {
			return errors.New("nodes are not all ready")
		}
		return nil
	}
}

func testAllNodesRunning(t *testing.T) {
	nodeCount, err := strconv.Atoi(os.Getenv(nodeCountEnv))
	if err != nil {
		t.Fatalf("failed to get number of expected nodes from environment variable %s: %v", nodeCountEnv, err)
	}

	max := 10 * time.Minute
	err = retry(allNodesRunning(nodeCount), t, 10*time.Second, max)
	if err != nil {
		t.Fatalf("Failed to find %d ready nodes in %v.", nodeCount, max)
	}
	t.Logf("Successfully found %d ready nodes.", nodeCount)
}

func testKillAPIServer(t *testing.T) {
	c, _ := newClient(t)
	pods, err := getAPIServers(c)
	if err != nil {
		t.Fatalf("Failed to get API server pod: %v", err)
	}

	oldPod := map[string]bool{}

	// Nuke all API servers.
	for _, pod := range pods.Items {
		if err := c.Core().Pods(pod.Namespace).Delete(pod.Name, nil); err != nil {
			t.Fatalf("Failed to delete API server pod %s: %v", pod.Name, err)
		}
		oldPod[pod.Name] = true
	}

	// API servers and temp API servers come in and out. Ensure
	// that the API server we detect is running for a couple
	// iterations.
	runningLastTime := false

	apiServerUp := func(t *testing.T) error {
		pods, err := getAPIServers(c)
		if err != nil {
			return fmt.Errorf("failed to get API server pod: %v", err)
		}

		for _, pod := range pods.Items {
			if oldPod[pod.Name] {
				return fmt.Errorf("old API server %s still running", pod.Name)
			}
		}

		allReady := len(pods.Items) != 0
		for _, p := range pods.Items {
			if p.Status.Phase != v1.PodRunning {
				allReady = false
			}
		}

		if allReady {
			if runningLastTime {
				return nil
			}
			runningLastTime = true
		}
		return fmt.Errorf("API server has not yet been running for more than one check")
	}

	max := 9 * time.Minute
	err = retry(apiServerUp, t, 15*time.Second, max)
	if err != nil {
		t.Fatalf("Failed waiting for API server pods to be ready in %v.", max)
	}
	t.Log("API server pods successfully came back up.")
}

// allResourcesCreated reads all the manifests recursively from the given paths (except the ones that are explictely
// ignored), and ensures that all the associated resources have been properly created.
func allResourcesCreated(manifestsPaths, ignoredManifests []string) func(t *testing.T) error {
	return func(t *testing.T) error {
		t.Logf("looking for resources defined by the provided manifests...")

		_, cc := newClient(t)
		failed := false

		// Walk recursively through the provided folders to find manifests, decode them and
		// ensure they exist on the cluster.
		resourcesToManifests := make(map[string][]string)
		resourcesCreated := make(map[string]bool)
		errs := walkPathForObjects(cc, manifestsPaths, func(info *resource.Info, walkErr error) error {
			if walkErr != nil {
				return walkErr
			}

			resourceIdentifier := resourceIdentifier(info)
			manifest := stripPathPrefixes(info.Source, manifestsPaths)

			if containsAnyOfStrings(ignoredManifests, manifest) {
				// The manifest is ignored.
				return nil
			}

			resourcesToManifests[resourceIdentifier] = append(resourcesToManifests[resourceIdentifier], manifest)
			resourcesCreated[resourceIdentifier] = resourcesCreated[resourceIdentifier] || (info.Get() == nil)
			if resourcesCreated[resourceIdentifier] {
				t.Logf("OK : %s - %v", resourceIdentifier, manifest)
			}

			return nil
		})

		// Each resource that is not present on the server, but declared by one or
		// multiple manifests is treated as a failure.
		//
		// TODO: This might be ineffective if resources are clearly different types
		// and purpose (i.e. DaemonSet / ConfigMap) are named the same. To work around
		// this, we could have a mapping of types we consider to be similar. For now,
		// this does not seem to be an issue and the effort is not justified.
		for resourceIdentifier, resourceCreated := range resourcesCreated {
			if !resourceCreated {
				t.Logf("MISSING : %s - %v", resourceIdentifier, resourcesToManifests[resourceIdentifier])
				failed = true
			}
		}

		// Each manifests that failed to get decoded (e.g. unknown type) is treated as
		// a failure. This typically means that the TPR kind, or the operator that is
		// responsible for creating the TPR kind, does not exist.
		for _, err := range errs {
			if containsAnyOfStrings(ignoredManifests, err.Error()) {
				// The manifest is ignored.
				continue
			}
			t.Log(err)
			failed = true
		}

		if failed {
			return errors.New("all defined resources were not present")
		}

		t.Logf("all resources defined by the provided manifests are present")
		return nil
	}
}

func testAllResourcesCreated(t *testing.T) {
	// Read configuration from environment.
	manifestPaths := os.Getenv(manifestPathsEnv)
	manifestsPathsSp := strings.Split(manifestPaths, ",")
	if len(manifestsPathsSp) == 0 {
		t.Skipf("no manifest paths in environment variable %s, skipping", manifestPathsEnv)
	}

	max := 10 * time.Minute
	err := retry(allResourcesCreated(manifestsPathsSp, defaultIgnoredManifests), t, 30*time.Second, max)
	if err != nil {
		t.Fatalf("timed out waiting for all manifests to be created after %v", max)
	}
}

func getAPIServers(client *kubernetes.Clientset) (*v1.PodList, error) {
	pods, err := client.Core().Pods(kubeSystemNamespace).List(meta_v1.ListOptions{LabelSelector: apiServerSelector})
	if err != nil {
		return nil, err
	}
	if len(pods.Items) == 0 {
		return nil, fmt.Errorf("no pods matched the label selector %q in the %s namespace", apiServerSelector, kubeSystemNamespace)
	}
	return pods, nil
}

func nodeReady(node v1.Node) (ok bool) {
	for _, cond := range node.Status.Conditions {
		if cond.Type == v1.NodeReady {
			return cond.Status == v1.ConditionTrue
		}
	}
	return false
}

// walkPathForObjects is a helper that calls the given resource.VisitorFunc function for each decoded Kubernetes
// manifest present in the given paths. Any decoding or parsing errors are aggregated.
func walkPathForObjects(cfg clientcmd.ClientConfig, paths []string, fn resource.VisitorFunc) (errs []error) {
	f := cmdutil.NewFactory(cfg)

	schema, err := f.Validator(false, "")
	if err != nil {
		return []error{err}
	}

	mapper, typer, err := f.UnstructuredObject()
	if err != nil {
		return []error{err}
	}

	result := resource.NewBuilder(mapper, f.CategoryExpander(), typer, resource.ClientMapperFunc(f.UnstructuredClientForMapping), unstructured.UnstructuredJSONScheme).
		ContinueOnError().
		Schema(schema).
		FilenameParam(false, &resource.FilenameOptions{Recursive: true, Filenames: paths}).
		Flatten().
		Do()

	err = result.Err()
	if err != nil && !strings.HasPrefix(err.Error(), "you must provide one or more resources") {
		return []error{err}
	}

	if err := result.Visit(fn); err != nil {
		for _, err := range err.(utilerrors.Aggregate).Errors() {
			if manifest, message, ok := parseMapperDecodingError(err.Error()); ok {
				errs = append(errs, fmt.Errorf("manifest %q not recognized: %s (this is likely due to a missing TPR kind / Operator)", stripPathPrefixes(manifest, paths), message))
			} else {
				errs = append(errs, fmt.Errorf("failed to parse manifest: %s (syntax?)", err))
			}
		}
	}
	return errs
}

// parseMapperDecodingError extracts information from a Kubernetes' mapper
// error.
func parseMapperDecodingError(err string) (manifest, message string, ok bool) {
	tokens := decodeErrorRegexp.FindStringSubmatch(err)
	if tokens == nil {
		return "", "", false
	}

	for i, name := range decodeErrorRegexp.SubexpNames() {
		if name == "Manifest" {
			manifest = tokens[i]
		} else if name == "Message" {
			message = tokens[i]
		}
	}

	return manifest, message, true
}

// stripPathPrefixes attempts to remove a prefix from the given path
// using the provided lists of potential prefixes. If none of the provided
// prefixes matched, the original path is returned.
func stripPathPrefixes(path string, prefixes []string) string {
	for _, prefix := range prefixes {
		if rel, err := filepath.Rel(prefix, path); err == nil {
			return rel
		}
	}
	return path
}

// containsAnyOfStrings returns whether one of the needles is
// contained within the haystack.
func containsAnyOfStrings(needles []string, haystack string) bool {
	for _, needle := range needles {
		if strings.Contains(haystack, needle) {
			return true
		}
	}
	return false
}

// resourcesIdentifier returns a string that can be used to identify and map
// a Kubernetes resource easily. Some object kinds are treated equivalently
// (see equivalentKindRemapping) in order to ease executing presence tests.
func resourceIdentifier(info *resource.Info) string {
	kindObject := info.VersionedObject.GetObjectKind().GroupVersionKind()
	kind := fmt.Sprintf("%s/%s:%s", kindObject.Group, kindObject.Version, kindObject.Kind)
	if equivalentKind, ok := equivalentKindRemapping[kind]; ok {
		kind = equivalentKind
	}
	return fmt.Sprintf("[Kind: %s | Namespace: %s | Name: %s]", kind, info.Namespace, info.Name)
}
<commit_msg>extend timeout of KillAPI test<commit_after>package smoke

import (
	"errors"
	"fmt"
	"os"
	"regexp"
	"strconv"
	"strings"
	"testing"
	"time"

	"path/filepath"

	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	utilerrors "k8s.io/apimachinery/pkg/util/errors"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/pkg/api/v1"
	"k8s.io/client-go/tools/clientcmd"
	cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"
	"k8s.io/kubernetes/pkg/kubectl/resource"
)

const (
	// networkingEnv is the environment variable that specifies if calico is running.
	networkingEnv = "SMOKE_NETWORKING"
	// nodeCountEnv is the environment variable that specifies the node count.
	nodeCountEnv = "SMOKE_NODE_COUNT"
	// manifestPathsEnv is the environment variable that defines the paths to the manifests that are deployed on the cluster.
	manifestPathsEnv = "SMOKE_MANIFEST_PATHS"
)

var (
	// defaultIgnoredManifests represents the manifests that are ignored by
	// testAllResourcesCreated by default.
	defaultIgnoredManifests = []string{
		"bootstrap",
		"kco-config.yaml",
		"tnco-config.yaml",
		// TODO: temporary disabling this for OpenTonic
		"tectonic/security/priviledged-scc-tectonic.yaml",
	}

	// equivalentKindRemapping is used by resourceIdentifier to map different
	// Kubernetes object kinds, that can be considered equivalent when checking
	// resource existence, with the same identifier.
	equivalentKindRemapping = map[string]string{
		"extensions/v1beta1:DaemonSet":  "extensions/v1beta1:DeploymentOrDaemonSet",
		"extensions/v1beta1:Deployment": "extensions/v1beta1:DeploymentOrDaemonSet",
		"apps/v1beta2:DaemonSet":        "apps/v1beta2:DeploymentOrDaemonSet",
		"apps/v1beta2:Deployment":       "apps/v1beta2:DeploymentOrDaemonSet",
	}

	// decodeErrorRegexp defines the format of the error returned by Kubernetes' resource mapper.
	decodeErrorRegexp = regexp.MustCompile(`unable to (?P<Type>decode|recognize) "(?P<Manifest>.*)": (?P<Message>.*)`)
)

func testCluster(t *testing.T) {
	// wait for all nodes to become available
	t.Run("AllNodesRunning", testAllNodesRunning)
	t.Run("AllResourcesCreated", testAllResourcesCreated)
	t.Run("AllPodsRunning", testAllPodsRunning)
	t.Run("KillAPIServer", testKillAPIServer)
}

func testAllPodsRunning(t *testing.T) {
	err := retry(allPodsRunning, t, 3*time.Second, 10*time.Minute)
	if err != nil {
		t.Fatalf("Timed out waiting for pods to be ready.")
	}
	t.Log("All pods are ready.")
}

func allPodsRunning(t *testing.T) error {
	err := checkPodsRunning(t)
	if err != nil {
		return err
	}

	// check one more time, because in case of crashloop we might get the transition
	time.Sleep(5 * time.Second)
	err = checkPodsRunning(t)
	if err != nil {
		return err
	}
	return nil
}

func checkPodsRunning(t *testing.T) error {
	c, _ := newClient(t)
	pods, err := c.Core().Pods("").List(meta_v1.ListOptions{})
	if err != nil {
		return fmt.Errorf("could not list pods: %v", err)
	}

	allReady := len(pods.Items) != 0
	for _, p := range pods.Items {
		if p.Status.Phase != v1.PodRunning || p.Status.ContainerStatuses[0].State.Running == nil {
			allReady = false
			t.Logf("pod %s/%s not running", p.Namespace, p.Name)
		}
	}

	if !allReady {
		return errors.New("pods are not all ready")
	}

	return nil
}

func allNodesRunning(expected int) func(t *testing.T) error {
	return func(t *testing.T) error {
		c, _ := newClient(t)
		nodes, err := c.Core().Nodes().List(meta_v1.ListOptions{})
		if err != nil {
			return fmt.Errorf("failed to list nodes: %v", err)
		}
		allReady := len(nodes.Items) != 0
		for _, node := range nodes.Items {
			if nodeReady(node) {
				t.Logf("node %s ready", node.Name)
				continue
			}
			allReady = false
			t.Logf("node %s not ready", node.Name)
		}
		if got := len(nodes.Items); got != expected {
			return fmt.Errorf("expected %d nodes, got %d", expected, got)
		}
		if !allReady {
			return errors.New("nodes are not all ready")
		}
		return nil
	}
}

func testAllNodesRunning(t *testing.T) {
	nodeCount, err := strconv.Atoi(os.Getenv(nodeCountEnv))
	if err != nil {
		t.Fatalf("failed to get number of expected nodes from environment variable %s: %v", nodeCountEnv, err)
	}

	max := 10 * time.Minute
	err = retry(allNodesRunning(nodeCount), t, 10*time.Second, max)
	if err != nil {
		t.Fatalf("Failed to find %d ready nodes in %v.", nodeCount, max)
	}
	t.Logf("Successfully found %d ready nodes.", nodeCount)
}

func testKillAPIServer(t *testing.T) {
	c, _ := newClient(t)
	pods, err := getAPIServers(c)
	if err != nil {
		t.Fatalf("Failed to get API server pod: %v", err)
	}

	oldPod := map[string]bool{}

	// Nuke all API servers.
	for _, pod := range pods.Items {
		if err := c.Core().Pods(pod.Namespace).Delete(pod.Name, nil); err != nil {
			t.Fatalf("Failed to delete API server pod %s: %v", pod.Name, err)
		}
		oldPod[pod.Name] = true
	}

	// API servers and temp API servers come in and out. Ensure
	// that the API server we detect is running for a couple
	// iterations.
	runningLastTime := false

	apiServerUp := func(t *testing.T) error {
		pods, err := getAPIServers(c)
		if err != nil {
			return fmt.Errorf("failed to get API server pod: %v", err)
		}

		for _, pod := range pods.Items {
			if oldPod[pod.Name] {
				return fmt.Errorf("old API server %s still running", pod.Name)
			}
		}

		allReady := len(pods.Items) != 0
		for _, p := range pods.Items {
			if p.Status.Phase != v1.PodRunning {
				allReady = false
			}
		}

		if allReady {
			if runningLastTime {
				return nil
			}
			runningLastTime = true
		}
		return fmt.Errorf("API server has not yet been running for more than one check")
	}

	max := 15 * time.Minute
	err = retry(apiServerUp, t, 15*time.Second, max)
	if err != nil {
		t.Fatalf("Failed waiting for API server pods to be ready in %v.", max)
	}
	t.Log("API server pods successfully came back up.")
}

// allResourcesCreated reads all the manifests recursively from the given paths (except the ones that are explictely
// ignored), and ensures that all the associated resources have been properly created.
func allResourcesCreated(manifestsPaths, ignoredManifests []string) func(t *testing.T) error {
	return func(t *testing.T) error {
		t.Logf("looking for resources defined by the provided manifests...")

		_, cc := newClient(t)
		failed := false

		// Walk recursively through the provided folders to find manifests, decode them and
		// ensure they exist on the cluster.
		resourcesToManifests := make(map[string][]string)
		resourcesCreated := make(map[string]bool)
		errs := walkPathForObjects(cc, manifestsPaths, func(info *resource.Info, walkErr error) error {
			if walkErr != nil {
				return walkErr
			}

			resourceIdentifier := resourceIdentifier(info)
			manifest := stripPathPrefixes(info.Source, manifestsPaths)

			if containsAnyOfStrings(ignoredManifests, manifest) {
				// The manifest is ignored.
				return nil
			}

			resourcesToManifests[resourceIdentifier] = append(resourcesToManifests[resourceIdentifier], manifest)
			resourcesCreated[resourceIdentifier] = resourcesCreated[resourceIdentifier] || (info.Get() == nil)
			if resourcesCreated[resourceIdentifier] {
				t.Logf("OK : %s - %v", resourceIdentifier, manifest)
			}

			return nil
		})

		// Each resource that is not present on the server, but declared by one or
		// multiple manifests is treated as a failure.
		//
		// TODO: This might be ineffective if resources are clearly different types
		// and purpose (i.e. DaemonSet / ConfigMap) are named the same. To work around
		// this, we could have a mapping of types we consider to be similar. For now,
		// this does not seem to be an issue and the effort is not justified.
		for resourceIdentifier, resourceCreated := range resourcesCreated {
			if !resourceCreated {
				t.Logf("MISSING : %s - %v", resourceIdentifier, resourcesToManifests[resourceIdentifier])
				failed = true
			}
		}

		// Each manifests that failed to get decoded (e.g. unknown type) is treated as
		// a failure. This typically means that the TPR kind, or the operator that is
		// responsible for creating the TPR kind, does not exist.
		for _, err := range errs {
			if containsAnyOfStrings(ignoredManifests, err.Error()) {
				// The manifest is ignored.
				continue
			}
			t.Log(err)
			failed = true
		}

		if failed {
			return errors.New("all defined resources were not present")
		}

		t.Logf("all resources defined by the provided manifests are present")
		return nil
	}
}

func testAllResourcesCreated(t *testing.T) {
	// Read configuration from environment.
	manifestPaths := os.Getenv(manifestPathsEnv)
	manifestsPathsSp := strings.Split(manifestPaths, ",")
	if len(manifestsPathsSp) == 0 {
		t.Skipf("no manifest paths in environment variable %s, skipping", manifestPathsEnv)
	}

	max := 10 * time.Minute
	err := retry(allResourcesCreated(manifestsPathsSp, defaultIgnoredManifests), t, 30*time.Second, max)
	if err != nil {
		t.Fatalf("timed out waiting for all manifests to be created after %v", max)
	}
}

func getAPIServers(client *kubernetes.Clientset) (*v1.PodList, error) {
	pods, err := client.Core().Pods(kubeSystemNamespace).List(meta_v1.ListOptions{LabelSelector: apiServerSelector})
	if err != nil {
		return nil, err
	}
	if len(pods.Items) == 0 {
		return nil, fmt.Errorf("no pods matched the label selector %q in the %s namespace", apiServerSelector, kubeSystemNamespace)
	}
	return pods, nil
}

func nodeReady(node v1.Node) (ok bool) {
	for _, cond := range node.Status.Conditions {
		if cond.Type == v1.NodeReady {
			return cond.Status == v1.ConditionTrue
		}
	}
	return false
}

// walkPathForObjects is a helper that calls the given resource.VisitorFunc function for each decoded Kubernetes
// manifest present in the given paths. Any decoding or parsing errors are aggregated.
func walkPathForObjects(cfg clientcmd.ClientConfig, paths []string, fn resource.VisitorFunc) (errs []error) {
	f := cmdutil.NewFactory(cfg)

	schema, err := f.Validator(false, "")
	if err != nil {
		return []error{err}
	}

	mapper, typer, err := f.UnstructuredObject()
	if err != nil {
		return []error{err}
	}

	result := resource.NewBuilder(mapper, f.CategoryExpander(), typer, resource.ClientMapperFunc(f.UnstructuredClientForMapping), unstructured.UnstructuredJSONScheme).
		ContinueOnError().
		Schema(schema).
		FilenameParam(false, &resource.FilenameOptions{Recursive: true, Filenames: paths}).
		Flatten().
		Do()

	err = result.Err()
	if err != nil && !strings.HasPrefix(err.Error(), "you must provide one or more resources") {
		return []error{err}
	}

	if err := result.Visit(fn); err != nil {
		for _, err := range err.(utilerrors.Aggregate).Errors() {
			if manifest, message, ok := parseMapperDecodingError(err.Error()); ok {
				errs = append(errs, fmt.Errorf("manifest %q not recognized: %s (this is likely due to a missing TPR kind / Operator)", stripPathPrefixes(manifest, paths), message))
			} else {
				errs = append(errs, fmt.Errorf("failed to parse manifest: %s (syntax?)", err))
			}
		}
	}
	return errs
}

// parseMapperDecodingError extracts information from a Kubernetes' mapper
// error.
func parseMapperDecodingError(err string) (manifest, message string, ok bool) {
	tokens := decodeErrorRegexp.FindStringSubmatch(err)
	if tokens == nil {
		return "", "", false
	}

	for i, name := range decodeErrorRegexp.SubexpNames() {
		if name == "Manifest" {
			manifest = tokens[i]
		} else if name == "Message" {
			message = tokens[i]
		}
	}

	return manifest, message, true
}

// stripPathPrefixes attempts to remove a prefix from the given path
// using the provided lists of potential prefixes. If none of the provided
// prefixes matched, the original path is returned.
func stripPathPrefixes(path string, prefixes []string) string {
	for _, prefix := range prefixes {
		if rel, err := filepath.Rel(prefix, path); err == nil {
			return rel
		}
	}
	return path
}

// containsAnyOfStrings returns whether one of the needles is
// contained within the haystack.
func containsAnyOfStrings(needles []string, haystack string) bool {
	for _, needle := range needles {
		if strings.Contains(haystack, needle) {
			return true
		}
	}
	return false
}

// resourcesIdentifier returns a string that can be used to identify and map
// a Kubernetes resource easily. Some object kinds are treated equivalently
// (see equivalentKindRemapping) in order to ease executing presence tests.
func resourceIdentifier(info *resource.Info) string {
	kindObject := info.VersionedObject.GetObjectKind().GroupVersionKind()
	kind := fmt.Sprintf("%s/%s:%s", kindObject.Group, kindObject.Version, kindObject.Kind)
	if equivalentKind, ok := equivalentKindRemapping[kind]; ok {
		kind = equivalentKind
	}
	return fmt.Sprintf("[Kind: %s | Namespace: %s | Name: %s]", kind, info.Namespace, info.Name)
}
<|endoftext|>


<commit_before>package moredis

import (
	"bytes"
	"encoding/json"
	"fmt"

	"github.com/Clever/moredis/logger"
	"github.com/garyburd/redigo/redis"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

// Params holds the params the user passes in for substitution into config templates.
type Params map[string]string

// Set parses a Param object from a string into the method receiver.
func (p *Params) Set(value string) error {
	if err := json.Unmarshal([]byte(value), p); err != nil {
		return err
	}
	return nil
}

// String gives a string representation of a Params object
func (p *Params) String() string {
	return fmt.Sprintf("%v", *p)
}

// Bson makes a copy of the Params object in a bson map for
// interaction with mgo.
func (p *Params) Bson() bson.M {
	ret := bson.M{}
	for key, val := range *p {
		ret[key] = val
	}
	return ret
}

// BuildCache builds a redis cache according to the passed in config.
func BuildCache(cacheConfig Config, params Params, redisURL string, mongoURL string) error {
	logger.Info("Populating cache.", logger.M{"cache": cacheConfig.Name})

	// set up mongo/redis connections
	mongoDb, redisConn, err := SetupDbs(mongoURL, redisURL)
	if err != nil {
		logger.Error("Failed to connect to dbs", err)
		return err
	}
	defer mongoDb.Session.Close()
	defer redisConn.Close()

	if err := processCollections(cacheConfig, params, mongoDb, redisConn); err != nil {
		return err
	}
	return nil
}

func processCollections(cacheConfig Config, params Params, mongoDb *mgo.Database, redisConn redis.Conn) error {
	redisWriter := NewRedisWriter(redisConn)
	for _, collection := range cacheConfig.Collections {
		query, err := ParseTemplatedJSON(collection.Query, params)
		if err != nil {
			logger.Error("Failed to parse query", err)
			return err
		}

		var iter MongoIter
		var projection map[string]interface{}
		if collection.Projection != "" {
			var err error
			projection, err = ParseTemplatedJSON(collection.Projection, params)
			if err != nil {
				logger.Error("Error applying projection template", err)
			}
			iter = mongoDb.C(collection.Collection).Find(query).Select(projection).Iter()
		} else {
			iter = mongoDb.C(collection.Collection).Find(query).Iter()
		}

		if err := SetRedisHashKeys(redisConn, &collection); err != nil {
			logger.Error("Error setting up redis map keys", err)
			return err
		}

		if err := ParseTemplates(&collection); err != nil {
			logger.Error("Error parsing templates", err)
			return err
		}

		logger.Info("Processing query for collection", logger.M{
			"query":      query,
			"collection": collection.Collection,
			"projection": projection,
		})
		if err := ProcessQuery(redisWriter, iter, collection.Maps); err != nil {
			logger.Error("Error processing query", err)
			return err
		}
		if err := redisWriter.Flush(); err != nil {
			logger.Error("Error flushing redis conn", err)
			return err
		}

		for _, rmap := range collection.Maps {
			if err := UpdateRedisMapReference(redisConn, params, rmap); err != nil {
				logger.Error("Failed to update map reference", err)
				return err
			}
		}
	}
	logger.Info("Completed populating cache", logger.M{"cache": cacheConfig.Name})
	return nil
}

// ProcessQuery iterates through all of the documents contained within iter, and maps
// keys to values in a redis hash according to your mapping config.
func ProcessQuery(writer RedisWriter, iter MongoIter, maps []MapConfig) error {
	processed := 0
	var result bson.M
	var b bytes.Buffer
	for iter.Next(&result) {
		for _, rmap := range maps {
			if err := rmap.KeyTemplate.Execute(&b, result); err != nil {
				logger.Error("Could not execute key template", err)
				return err
			}
			key := b.String()
			b.Reset()

			if key == "" || key == "<no value>" {
				continue
			}

			if err := rmap.ValueTemplate.Execute(&b, result); err != nil {
				logger.Error("Could not execute value template", err)
				return err
			}
			val := b.String()
			b.Reset()

			if err := writer.Send("HSET", rmap.HashKey, key, val); err != nil {
				logger.Error("Could not send HSET", err)
				return err
			}
		}
		processed++
	}
	if err := iter.Err(); err != nil {
		logger.Error("Iteration error", err)
	}
	if err := iter.Close(); err != nil {
		logger.Error("Iter.Close() error", err)
		return err
	}
	if err := writer.Flush(); err != nil {
		logger.Error("Error flushing", err)
		return err
	}
	logger.Info("Processed all documents for query", logger.M{"processed": processed})
	return nil
}

// SetRedisHashKeys determines the correct keys to use for the redis hashes that
// will be created to store the mapped values.  These keys are generated in an atomic
// fashion and will not interfere with any other running instances of moredis
func SetRedisHashKeys(conn redis.Conn, collection *CollectionConfig) error {
	for ix := range collection.Maps {
		tempKey, err := redis.Int64(conn.Do("INCR", "moredis:mapindexcounter"))
		if err != nil {
			return err
		}
		collection.Maps[ix].HashKey = fmt.Sprintf("moredis:maps:%d", tempKey)
	}
	return nil
}

// UpdateRedisMapReference updates the map specified in redis to point to the newly populated hashes,
// then deletes the previously referenced hash.  The hash reference is updated atomically.
func UpdateRedisMapReference(conn redis.Conn, params Params, mapConfig MapConfig) error {
	mapName, err := ApplyTemplate(mapConfig.Name, params.Bson())
	if err != nil {
		return err
	}
	oldMap, err := redis.String(conn.Do("GETSET", mapName, mapConfig.HashKey))
	logger.Info("Updating map reference", logger.M{"map": mapName, "oldref": oldMap, "newref": mapConfig.HashKey})
	if err == redis.ErrNil {
		// no old map, just return
		return nil
	}
	if err != nil {
		return err
	}

	logger.Info("Deleting old referenced map", logger.M{"map": oldMap})
	if _, err := conn.Do("DEL", oldMap); err != nil {
		return err
	}
	return nil
}
<commit_msg>actually return error<commit_after>package moredis

import (
	"bytes"
	"encoding/json"
	"fmt"

	"github.com/Clever/moredis/logger"
	"github.com/garyburd/redigo/redis"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

// Params holds the params the user passes in for substitution into config templates.
type Params map[string]string

// Set parses a Param object from a string into the method receiver.
func (p *Params) Set(value string) error {
	if err := json.Unmarshal([]byte(value), p); err != nil {
		return err
	}
	return nil
}

// String gives a string representation of a Params object
func (p *Params) String() string {
	return fmt.Sprintf("%v", *p)
}

// Bson makes a copy of the Params object in a bson map for
// interaction with mgo.
func (p *Params) Bson() bson.M {
	ret := bson.M{}
	for key, val := range *p {
		ret[key] = val
	}
	return ret
}

// BuildCache builds a redis cache according to the passed in config.
func BuildCache(cacheConfig Config, params Params, redisURL string, mongoURL string) error {
	logger.Info("Populating cache.", logger.M{"cache": cacheConfig.Name})

	// set up mongo/redis connections
	mongoDb, redisConn, err := SetupDbs(mongoURL, redisURL)
	if err != nil {
		logger.Error("Failed to connect to dbs", err)
		return err
	}
	defer mongoDb.Session.Close()
	defer redisConn.Close()

	if err := processCollections(cacheConfig, params, mongoDb, redisConn); err != nil {
		return err
	}
	return nil
}

func processCollections(cacheConfig Config, params Params, mongoDb *mgo.Database, redisConn redis.Conn) error {
	redisWriter := NewRedisWriter(redisConn)
	for _, collection := range cacheConfig.Collections {
		query, err := ParseTemplatedJSON(collection.Query, params)
		if err != nil {
			logger.Error("Failed to parse query", err)
			return err
		}

		var iter MongoIter
		var projection map[string]interface{}
		if collection.Projection != "" {
			var err error
			projection, err = ParseTemplatedJSON(collection.Projection, params)
			if err != nil {
				logger.Error("Error applying projection template", err)
			}
			iter = mongoDb.C(collection.Collection).Find(query).Select(projection).Iter()
		} else {
			iter = mongoDb.C(collection.Collection).Find(query).Iter()
		}

		if err := SetRedisHashKeys(redisConn, &collection); err != nil {
			logger.Error("Error setting up redis map keys", err)
			return err
		}

		if err := ParseTemplates(&collection); err != nil {
			logger.Error("Error parsing templates", err)
			return err
		}

		logger.Info("Processing query for collection", logger.M{
			"query":      query,
			"collection": collection.Collection,
			"projection": projection,
		})
		if err := ProcessQuery(redisWriter, iter, collection.Maps); err != nil {
			logger.Error("Error processing query", err)
			return err
		}
		if err := redisWriter.Flush(); err != nil {
			logger.Error("Error flushing redis conn", err)
			return err
		}

		for _, rmap := range collection.Maps {
			if err := UpdateRedisMapReference(redisConn, params, rmap); err != nil {
				logger.Error("Failed to update map reference", err)
				return err
			}
		}
	}
	logger.Info("Completed populating cache", logger.M{"cache": cacheConfig.Name})
	return nil
}

// ProcessQuery iterates through all of the documents contained within iter, and maps
// keys to values in a redis hash according to your mapping config.
func ProcessQuery(writer RedisWriter, iter MongoIter, maps []MapConfig) error {
	processed := 0
	var result bson.M
	var b bytes.Buffer
	for iter.Next(&result) {
		for _, rmap := range maps {
			if err := rmap.KeyTemplate.Execute(&b, result); err != nil {
				logger.Error("Could not execute key template", err)
				return err
			}
			key := b.String()
			b.Reset()

			if key == "" || key == "<no value>" {
				continue
			}

			if err := rmap.ValueTemplate.Execute(&b, result); err != nil {
				logger.Error("Could not execute value template", err)
				return err
			}
			val := b.String()
			b.Reset()

			if err := writer.Send("HSET", rmap.HashKey, key, val); err != nil {
				logger.Error("Could not send HSET", err)
				return err
			}
		}
		processed++
	}
	if err := iter.Err(); err != nil {
		logger.Error("Iteration error", err)
		return err
	}
	if err := iter.Close(); err != nil {
		logger.Error("Iter.Close() error", err)
		return err
	}
	if err := writer.Flush(); err != nil {
		logger.Error("Error flushing", err)
		return err
	}
	logger.Info("Processed all documents for query", logger.M{"processed": processed})
	return nil
}

// SetRedisHashKeys determines the correct keys to use for the redis hashes that
// will be created to store the mapped values.  These keys are generated in an atomic
// fashion and will not interfere with any other running instances of moredis
func SetRedisHashKeys(conn redis.Conn, collection *CollectionConfig) error {
	for ix := range collection.Maps {
		tempKey, err := redis.Int64(conn.Do("INCR", "moredis:mapindexcounter"))
		if err != nil {
			return err
		}
		collection.Maps[ix].HashKey = fmt.Sprintf("moredis:maps:%d", tempKey)
	}
	return nil
}

// UpdateRedisMapReference updates the map specified in redis to point to the newly populated hashes,
// then deletes the previously referenced hash.  The hash reference is updated atomically.
func UpdateRedisMapReference(conn redis.Conn, params Params, mapConfig MapConfig) error {
	mapName, err := ApplyTemplate(mapConfig.Name, params.Bson())
	if err != nil {
		return err
	}
	oldMap, err := redis.String(conn.Do("GETSET", mapName, mapConfig.HashKey))
	logger.Info("Updating map reference", logger.M{"map": mapName, "oldref": oldMap, "newref": mapConfig.HashKey})
	if err == redis.ErrNil {
		// no old map, just return
		return nil
	}
	if err != nil {
		return err
	}

	logger.Info("Deleting old referenced map", logger.M{"map": oldMap})
	if _, err := conn.Do("DEL", oldMap); err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package mutable

import (
	"fmt"
	"math"
	"os"
	"time"

	"github.com/jacobsa/timeutil"
	"golang.org/x/net/context"
)

// A mutable view on some content. Created with an initial read-only view,
// which then can be modified by the user and read back. Keeps track of which
// portion of the content has been dirtied.
//
// External synchronization is required.
type Content interface {
	// Panic if any internal invariants are violated.
	CheckInvariants()

	// Destroy any state used by the object, putting it into an indeterminate
	// state. The object must not be used again.
	Destroy()

	// Read part of the content, with semantics equivalent to io.ReaderAt aside
	// from context support.
	ReadAt(ctx context.Context, buf []byte, offset int64) (n int, err error)

	// Return information about the current state of the content.
	Stat(ctx context.Context) (sr StatResult, err error)

	// Write into the content, with semantics equivalent to io.WriterAt aside from
	// context support.
	WriteAt(ctx context.Context, buf []byte, offset int64) (n int, err error)

	// Truncate our the content to the given number of bytes, extending if n is
	// greater than the current size.
	Truncate(ctx context.Context, n int64) (err error)
}

type StatResult struct {
	// The current size in bytes of the content.
	Size int64

	// It is guaranteed that all bytes in the range [0, DirtyThreshold) are
	// unmodified from the original content with which the mutable content object
	// was created.
	DirtyThreshold int64

	// The time at which the content was last updated, or nil if we've never
	// changed it.
	Mtime *time.Time
}

// Create a mutable content object whose initial contents are given by the
// supplied file. The caller must ensure that the file is not used further by
// anybody else.
func NewContent(
	initialContent *os.File,
	clock timeutil.Clock) (mc Content) {
	mc = &mutableContent{
		clock:          clock,
		initialContent: initialContent,
		dirtyThreshold: initialContent.Size(),
	}

	return
}

type mutableContent struct {
	/////////////////////////
	// Dependencies
	/////////////////////////

	clock timeutil.Clock

	/////////////////////////
	// Mutable state
	/////////////////////////

	destroyed bool

	// Have we been dirtied since we were created?
	dirty bool

	// A file containing our current contents.
	contents *os.File

	// The lowest byte index that has been modified from the initial contents.
	//
	// INVARIANT: !dirty() => Stat().DirtyThreshold == Stat().Size
	dirtyThreshold int64

	// The time at which a method that modifies our contents was last called, or
	// nil if never.
	//
	// INVARIANT: dirty() => mtime != nil
	mtime *time.Time
}

////////////////////////////////////////////////////////////////////////
// Public interface
////////////////////////////////////////////////////////////////////////

func (mc *mutableContent) CheckInvariants() {
	if mc.destroyed {
		panic("Use of destroyed mutableContent object.")
	}

	// INVARIANT: When non-nil, initialContent.CheckInvariants() does not panic.
	if mc.initialContent != nil {
		mc.initialContent.CheckInvariants()
	}

	// INVARIANT: (initialContent == nil) != (readWriteLease == nil)
	if mc.initialContent == nil && mc.readWriteLease == nil {
		panic("Both initialContent and readWriteLease are nil")
	}

	if mc.initialContent != nil && mc.readWriteLease != nil {
		panic("Both initialContent and readWriteLease are non-nil")
	}

	// INVARIANT: If dirty(), then mtime != nil
	if mc.dirty() && mc.mtime == nil {
		panic("Expected non-nil mtime.")
	}

	// INVARIANT: initialContent != nil => dirtyThreshold == initialContent.Size()
	if mc.initialContent != nil {
		if mc.dirtyThreshold != mc.initialContent.Size() {
			panic(fmt.Sprintf(
				"Dirty threshold mismatch: %d vs. %d",
				mc.dirtyThreshold,
				mc.initialContent.Size()))
		}
	}
}

func (mc *mutableContent) Destroy() {
	mc.destroyed = true

	if mc.initialContent != nil {
		mc.initialContent.Destroy()
		mc.initialContent = nil
	}

	if mc.readWriteLease != nil {
		mc.readWriteLease.Downgrade().Revoke()
		mc.readWriteLease = nil
	}
}

func (mc *mutableContent) Release() (rwl lease.ReadWriteLease) {
	if !mc.dirty() {
		return
	}

	rwl = mc.readWriteLease
	mc.readWriteLease = nil
	mc.Destroy()

	return
}

func (mc *mutableContent) ReadAt(
	ctx context.Context,
	buf []byte,
	offset int64) (n int, err error) {
	// Serve from the appropriate place.
	if mc.dirty() {
		n, err = mc.readWriteLease.ReadAt(buf, offset)
	} else {
		n, err = mc.initialContent.ReadAt(ctx, buf, offset)
	}

	return
}

func (mc *mutableContent) Stat(
	ctx context.Context) (sr StatResult, err error) {
	sr.DirtyThreshold = mc.dirtyThreshold
	sr.Mtime = mc.mtime

	// Get the size from the appropriate place.
	if mc.dirty() {
		sr.Size, err = mc.readWriteLease.Size()
		if err != nil {
			return
		}
	} else {
		sr.Size = mc.initialContent.Size()
	}

	return
}

func (mc *mutableContent) WriteAt(
	ctx context.Context,
	buf []byte,
	offset int64) (n int, err error) {
	// Make sure we have a read/write lease.
	if err = mc.ensureReadWriteLease(ctx); err != nil {
		err = fmt.Errorf("ensureReadWriteLease: %v", err)
		return
	}

	// Update our state regarding being dirty.
	mc.dirtyThreshold = minInt64(mc.dirtyThreshold, offset)

	newMtime := mc.clock.Now()
	mc.mtime = &newMtime

	// Call through.
	n, err = mc.readWriteLease.WriteAt(buf, offset)

	return
}

func (mc *mutableContent) Truncate(
	ctx context.Context,
	n int64) (err error) {
	// Make sure we have a read/write lease.
	if err = mc.ensureReadWriteLease(ctx); err != nil {
		err = fmt.Errorf("ensureReadWriteLease: %v", err)
		return
	}

	// Convert to signed, which is what lease.ReadWriteLease wants.
	if n > math.MaxInt64 {
		err = fmt.Errorf("Illegal offset: %v", n)
		return
	}

	// Update our state regarding being dirty.
	mc.dirtyThreshold = minInt64(mc.dirtyThreshold, n)

	newMtime := mc.clock.Now()
	mc.mtime = &newMtime

	// Call through.
	err = mc.readWriteLease.Truncate(int64(n))

	return
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func minInt64(a int64, b int64) int64 {
	if a < b {
		return a
	}

	return b
}

func (mc *mutableContent) dirty() bool {
	return mc.readWriteLease != nil
}

// Ensure that mc.readWriteLease is non-nil with an authoritative view of mc's
// contents.
func (mc *mutableContent) ensureReadWriteLease(
	ctx context.Context) (err error) {
	// Is there anything to do?
	if mc.readWriteLease != nil {
		return
	}

	// Set up the read/write lease.
	rwl, err := mc.initialContent.Upgrade(ctx)
	if err != nil {
		err = fmt.Errorf("initialContent.Upgrade: %v", err)
		return
	}

	mc.readWriteLease = rwl
	mc.initialContent = nil

	return
}
<commit_msg>mutableContent.CheckInvariants<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package mutable

import (
	"fmt"
	"math"
	"os"
	"time"

	"github.com/jacobsa/timeutil"
	"golang.org/x/net/context"
)

// A mutable view on some content. Created with an initial read-only view,
// which then can be modified by the user and read back. Keeps track of which
// portion of the content has been dirtied.
//
// External synchronization is required.
type Content interface {
	// Panic if any internal invariants are violated.
	CheckInvariants()

	// Destroy any state used by the object, putting it into an indeterminate
	// state. The object must not be used again.
	Destroy()

	// Read part of the content, with semantics equivalent to io.ReaderAt aside
	// from context support.
	ReadAt(ctx context.Context, buf []byte, offset int64) (n int, err error)

	// Return information about the current state of the content.
	Stat(ctx context.Context) (sr StatResult, err error)

	// Write into the content, with semantics equivalent to io.WriterAt aside from
	// context support.
	WriteAt(ctx context.Context, buf []byte, offset int64) (n int, err error)

	// Truncate our the content to the given number of bytes, extending if n is
	// greater than the current size.
	Truncate(ctx context.Context, n int64) (err error)
}

type StatResult struct {
	// The current size in bytes of the content.
	Size int64

	// It is guaranteed that all bytes in the range [0, DirtyThreshold) are
	// unmodified from the original content with which the mutable content object
	// was created.
	DirtyThreshold int64

	// The time at which the content was last updated, or nil if we've never
	// changed it.
	Mtime *time.Time
}

// Create a mutable content object whose initial contents are given by the
// supplied file. The caller must ensure that the file is not used further by
// anybody else.
func NewContent(
	initialContent *os.File,
	clock timeutil.Clock) (mc Content, err error) {
	// Find the file's size.
	size, err := initialContent.Seek(0, 2)
	if err != nil {
		err = fmt.Errorf("Seek: %v", err)
		return
	}

	// Create the Content.
	mc = &mutableContent{
		clock:          clock,
		contents:       initialContent,
		dirtyThreshold: size,
	}

	return
}

type mutableContent struct {
	/////////////////////////
	// Dependencies
	/////////////////////////

	clock timeutil.Clock

	/////////////////////////
	// Mutable state
	/////////////////////////

	destroyed bool

	// Have we been dirtied since we were created?
	dirty bool

	// A file containing our current contents.
	contents *os.File

	// The lowest byte index that has been modified from the initial contents.
	//
	// INVARIANT: !dirty => Stat().DirtyThreshold == Stat().Size
	dirtyThreshold int64

	// The time at which a method that modifies our contents was last called, or
	// nil if never.
	//
	// INVARIANT: dirty => mtime != nil
	mtime *time.Time
}

////////////////////////////////////////////////////////////////////////
// Public interface
////////////////////////////////////////////////////////////////////////

func (mc *mutableContent) CheckInvariants() {
	if mc.destroyed {
		panic("Use of destroyed mutableContent object.")
	}

	// INVARIANT: !dirty => Stat().DirtyThreshold == Stat().Size
	if !mc.dirty {
		sr, err := mc.Stat(context.Background())
		if err != nil {
			panic(fmt.Sprintf("Stat: %v", err))
		}

		if sr.DirtyThreshold != sr.Size {
			panic(fmt.Sprintf("Mismatch: %d vs. %d", sr.DirtyThreshold, sr.Size))
		}
	}

	// INVARIANT: dirty => mtime != nil
	if mc.dirty && mc.mtime == nil {
		panic("Expected a non-nil mtime")
	}
}

func (mc *mutableContent) Destroy() {
	mc.destroyed = true

	if mc.initialContent != nil {
		mc.initialContent.Destroy()
		mc.initialContent = nil
	}

	if mc.readWriteLease != nil {
		mc.readWriteLease.Downgrade().Revoke()
		mc.readWriteLease = nil
	}
}

func (mc *mutableContent) Release() (rwl lease.ReadWriteLease) {
	if !mc.dirty() {
		return
	}

	rwl = mc.readWriteLease
	mc.readWriteLease = nil
	mc.Destroy()

	return
}

func (mc *mutableContent) ReadAt(
	ctx context.Context,
	buf []byte,
	offset int64) (n int, err error) {
	// Serve from the appropriate place.
	if mc.dirty() {
		n, err = mc.readWriteLease.ReadAt(buf, offset)
	} else {
		n, err = mc.initialContent.ReadAt(ctx, buf, offset)
	}

	return
}

func (mc *mutableContent) Stat(
	ctx context.Context) (sr StatResult, err error) {
	sr.DirtyThreshold = mc.dirtyThreshold
	sr.Mtime = mc.mtime

	// Get the size from the appropriate place.
	if mc.dirty() {
		sr.Size, err = mc.readWriteLease.Size()
		if err != nil {
			return
		}
	} else {
		sr.Size = mc.initialContent.Size()
	}

	return
}

func (mc *mutableContent) WriteAt(
	ctx context.Context,
	buf []byte,
	offset int64) (n int, err error) {
	// Make sure we have a read/write lease.
	if err = mc.ensureReadWriteLease(ctx); err != nil {
		err = fmt.Errorf("ensureReadWriteLease: %v", err)
		return
	}

	// Update our state regarding being dirty.
	mc.dirtyThreshold = minInt64(mc.dirtyThreshold, offset)

	newMtime := mc.clock.Now()
	mc.mtime = &newMtime

	// Call through.
	n, err = mc.readWriteLease.WriteAt(buf, offset)

	return
}

func (mc *mutableContent) Truncate(
	ctx context.Context,
	n int64) (err error) {
	// Make sure we have a read/write lease.
	if err = mc.ensureReadWriteLease(ctx); err != nil {
		err = fmt.Errorf("ensureReadWriteLease: %v", err)
		return
	}

	// Convert to signed, which is what lease.ReadWriteLease wants.
	if n > math.MaxInt64 {
		err = fmt.Errorf("Illegal offset: %v", n)
		return
	}

	// Update our state regarding being dirty.
	mc.dirtyThreshold = minInt64(mc.dirtyThreshold, n)

	newMtime := mc.clock.Now()
	mc.mtime = &newMtime

	// Call through.
	err = mc.readWriteLease.Truncate(int64(n))

	return
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func minInt64(a int64, b int64) int64 {
	if a < b {
		return a
	}

	return b
}

// Ensure that mc.readWriteLease is non-nil with an authoritative view of mc's
// contents.
func (mc *mutableContent) ensureReadWriteLease(
	ctx context.Context) (err error) {
	// Is there anything to do?
	if mc.readWriteLease != nil {
		return
	}

	// Set up the read/write lease.
	rwl, err := mc.initialContent.Upgrade(ctx)
	if err != nil {
		err = fmt.Errorf("initialContent.Upgrade: %v", err)
		return
	}

	mc.readWriteLease = rwl
	mc.initialContent = nil

	return
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package handler

import (
	"fmt"
	"net/http"
	"net/http/httptest"
	"testing"

	network "knative.dev/networking/pkg"
	"knative.dev/serving/pkg/activator"
	"knative.dev/serving/pkg/queue"
)

func TestProbeHandler(t *testing.T) {
	examples := []struct {
		label          string
		headers        http.Header
		passed         bool
		expectedStatus int
		method         string
	}{{
		label:          "forward a normal POST request",
		headers:        http.Header{},
		passed:         true,
		expectedStatus: http.StatusOK,
		method:         http.MethodPost,
	}, {
		label:          "filter a POST request containing probe header, even if probe is for a different target",
		headers:        mapToHeader(map[string]string{network.ProbeHeaderName: queue.Name}),
		passed:         false,
		expectedStatus: http.StatusBadRequest,
		method:         http.MethodPost,
	}, {
		label:          "filter a POST request containing probe header",
		headers:        mapToHeader(map[string]string{network.ProbeHeaderName: activator.Name}),
		passed:         false,
		expectedStatus: http.StatusOK,
		method:         http.MethodPost,
	}, {
		label:          "forward a normal GET request",
		headers:        http.Header{},
		passed:         true,
		expectedStatus: http.StatusOK,
		method:         http.MethodGet,
	}, {
		label:          "filter a GET request containing probe header, with wrong target system",
		headers:        mapToHeader(map[string]string{network.ProbeHeaderName: "not-empty"}),
		passed:         false,
		expectedStatus: http.StatusBadRequest,
		method:         http.MethodGet,
	}, {
		label:          "filter a GET request containing probe header",
		headers:        mapToHeader(map[string]string{network.ProbeHeaderName: activator.Name}),
		passed:         false,
		expectedStatus: http.StatusOK,
		method:         http.MethodGet,
	}, {
		label:          "forward a request containing empty retry header",
		headers:        mapToHeader(map[string]string{network.ProbeHeaderName: ""}),
		passed:         true,
		expectedStatus: http.StatusOK,
		method:         http.MethodPost,
	}}

	for _, e := range examples {
		t.Run(e.label, func(t *testing.T) {
			wasPassed := false
			baseHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				wasPassed = true
				w.WriteHeader(http.StatusOK)
			})
			handler := ProbeHandler{NextHandler: baseHandler}

			resp := httptest.NewRecorder()
			req := httptest.NewRequest(e.method, "http://example.com", nil)
			req.Header = e.headers

			handler.ServeHTTP(resp, req)

			if wasPassed != e.passed {
				if !e.passed {
					t.Error("Request got passed to the next handler unexpectedly")
				} else {
					t.Error("Request was not passed to the next handler as expected")
				}
			}

			if resp.Code != e.expectedStatus {
				t.Errorf("Unexpected response status. Want %d, got %d", e.expectedStatus, resp.Code)
			}
		})
	}
}

func mapToHeader(m map[string]string) http.Header {
	h := http.Header{}
	for k, v := range m {
		h.Add(k, v)
	}
	return h
}

func BenchmarkProbeHandler(b *testing.B) {
	tests := []struct {
		label   string
		headers http.Header
	}{{
		label:   "valid header name",
		headers: mapToHeader(map[string]string{network.ProbeHeaderName: activator.Name}),
	}, {
		label:   "invalid header name",
		headers: mapToHeader(map[string]string{network.ProbeHeaderName: "not-empty"}),
	}, {
		label:   "empty header name",
		headers: http.Header{},
	}}

	baseHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})
	handler := ProbeHandler{NextHandler: baseHandler}
	req := httptest.NewRequest(http.MethodPost, "http://example.com", nil)

	for _, test := range tests {
		req.Header = test.headers
		b.Run(fmt.Sprintf("%s-sequential", test.label), func(b *testing.B) {
			resp := httptest.NewRecorder()
			for j := 0; j < b.N; j++ {
				handler.ServeHTTP(resp, req)
			}
		})

		b.Run(fmt.Sprintf("%s-parallel", test.label), func(b *testing.B) {
			b.RunParallel(func(pb *testing.PB) {
				resp := httptest.NewRecorder()
				for pb.Next() {
					handler.ServeHTTP(resp, req)
				}
			})
		})
	}
}
<commit_msg>Remove mapToHeader helper. (#10818)<commit_after>/*
Copyright 2019 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package handler

import (
	"fmt"
	"net/http"
	"net/http/httptest"
	"testing"

	network "knative.dev/networking/pkg"
	"knative.dev/serving/pkg/activator"
	"knative.dev/serving/pkg/queue"
)

func TestProbeHandler(t *testing.T) {
	examples := []struct {
		label          string
		headers        http.Header
		passed         bool
		expectedStatus int
		method         string
	}{{
		label:          "forward a normal POST request",
		headers:        http.Header{},
		passed:         true,
		expectedStatus: http.StatusOK,
		method:         http.MethodPost,
	}, {
		label:          "filter a POST request containing probe header, even if probe is for a different target",
		headers:        http.Header{network.ProbeHeaderName: []string{queue.Name}},
		expectedStatus: http.StatusBadRequest,
		method:         http.MethodPost,
	}, {
		label:          "filter a POST request containing probe header",
		headers:        http.Header{network.ProbeHeaderName: []string{activator.Name}},
		expectedStatus: http.StatusOK,
		method:         http.MethodPost,
	}, {
		label:          "forward a normal GET request",
		headers:        http.Header{},
		passed:         true,
		expectedStatus: http.StatusOK,
		method:         http.MethodGet,
	}, {
		label:          "filter a GET request containing probe header, with wrong target system",
		headers:        http.Header{network.ProbeHeaderName: []string{"not-empty"}},
		expectedStatus: http.StatusBadRequest,
		method:         http.MethodGet,
	}, {
		label:          "filter a GET request containing probe header",
		headers:        http.Header{network.ProbeHeaderName: []string{activator.Name}},
		passed:         false,
		expectedStatus: http.StatusOK,
		method:         http.MethodGet,
	}, {
		label:          "forward a request containing empty retry header",
		headers:        http.Header{network.ProbeHeaderName: []string{""}},
		passed:         true,
		expectedStatus: http.StatusOK,
		method:         http.MethodPost,
	}}

	for _, e := range examples {
		t.Run(e.label, func(t *testing.T) {
			wasPassed := false
			baseHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				wasPassed = true
				w.WriteHeader(http.StatusOK)
			})
			handler := ProbeHandler{NextHandler: baseHandler}

			resp := httptest.NewRecorder()
			req := httptest.NewRequest(e.method, "http://example.com", nil)
			req.Header = e.headers

			handler.ServeHTTP(resp, req)

			if wasPassed != e.passed {
				if !e.passed {
					t.Error("Request got passed to the next handler unexpectedly")
				} else {
					t.Error("Request was not passed to the next handler as expected")
				}
			}

			if resp.Code != e.expectedStatus {
				t.Errorf("Unexpected response status. Want %d, got %d", e.expectedStatus, resp.Code)
			}
		})
	}
}

func BenchmarkProbeHandler(b *testing.B) {
	tests := []struct {
		label   string
		headers http.Header
	}{{
		label:   "valid header name",
		headers: http.Header{network.ProbeHeaderName: []string{activator.Name}},
	}, {
		label:   "invalid header name",
		headers: http.Header{network.ProbeHeaderName: []string{"some-other-cool-value"}},
	}, {
		label:   "empty header name",
		headers: http.Header{},
	}}

	baseHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})
	handler := ProbeHandler{NextHandler: baseHandler}
	req := httptest.NewRequest(http.MethodPost, "http://example.com", nil)

	for _, test := range tests {
		req.Header = test.headers
		b.Run(fmt.Sprintf("%s-sequential", test.label), func(b *testing.B) {
			resp := httptest.NewRecorder()
			for j := 0; j < b.N; j++ {
				handler.ServeHTTP(resp, req)
			}
		})

		b.Run(fmt.Sprintf("%s-parallel", test.label), func(b *testing.B) {
			b.RunParallel(func(pb *testing.PB) {
				resp := httptest.NewRecorder()
				for pb.Next() {
					handler.ServeHTTP(resp, req)
				}
			})
		})
	}
}
<|endoftext|>


<commit_before>// +build providerless

/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package nodeipam

import (
	"net"

	coreinformers "k8s.io/client-go/informers/core/v1"
	clientset "k8s.io/client-go/kubernetes"
	cloudprovider "k8s.io/cloud-provider"
	"k8s.io/klog"
)

func startLegacyIPAM(
	ic *Controller,
	nodeInformer coreinformers.NodeInformer,
	cloud cloudprovider.Interface,
	kubeClient clientset.Interface,
	clusterCIDRs []*net.IPNet,
	serviceCIDR *net.IPNet,
	nodeCIDRMaskSize int,
) {
	klog.Fatal("Error trying to Init(): legacy cloud provider support disabled at build time")
}
<commit_msg>fix multiple node cidr masks in providerless build<commit_after>// +build providerless

/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package nodeipam

import (
	"net"

	coreinformers "k8s.io/client-go/informers/core/v1"
	clientset "k8s.io/client-go/kubernetes"
	cloudprovider "k8s.io/cloud-provider"
	"k8s.io/klog"
)

func startLegacyIPAM(
	ic *Controller,
	nodeInformer coreinformers.NodeInformer,
	cloud cloudprovider.Interface,
	kubeClient clientset.Interface,
	clusterCIDRs []*net.IPNet,
	serviceCIDR *net.IPNet,
	nodeCIDRMaskSizes []int,
) {
	klog.Fatal("Error trying to Init(): legacy cloud provider support disabled at build time")
}
<|endoftext|>


<commit_before>package net

import (
	"bytes"
	"context"
	"fmt"
	"io/ioutil"
	"os"
	"testing"
	"time"

	"github.com/sahib/brig/backend"
	"github.com/sahib/brig/catfs"
	ie "github.com/sahib/brig/catfs/errors"
	"github.com/sahib/brig/net/peer"
	"github.com/sahib/brig/repo"
	"github.com/stretchr/testify/require"
)

type testUnit struct {
	ctl *Client
	srv *Server
	fs  *catfs.FS
	rp  *repo.Repository
	bk  backend.Backend
}

func withNetServer(t *testing.T, name string, backendPort int, basePath string, fn func(u testUnit)) {
	if basePath == "" {
		var err error
		basePath, err = ioutil.TempDir("", "brig-ctl-test")
		require.Nil(t, err)

		defer func() {
			require.Nil(t, os.RemoveAll(basePath))
		}()
	}

	fullPath := fmt.Sprintf("%s/user=%s-port=%d", basePath, name, backendPort)
	require.Nil(t, os.MkdirAll(fullPath, 0700))

	bk, err := backend.FromName("mock", fullPath, nil)
	require.Nil(t, err)

	err = repo.Init(fullPath, name, "password", "mock")
	require.Nil(t, err)

	rp, err := repo.Open(fullPath, "password")
	require.Nil(t, err)

	srv, err := NewServer(rp, bk)
	require.Nil(t, err)

	fs, err := rp.FS(name, bk)
	require.Nil(t, err)

	waitForDeath := make(chan bool)
	go func() {
		defer func() {
			waitForDeath <- true
		}()
		require.Nil(t, srv.Serve())
		require.Nil(t, srv.Close())
	}()

	// Allow a short time for the server go routine to fully boot up.
	time.Sleep(50 * time.Millisecond)

	// Run the actual test function:
	fn(testUnit{
		rp:  rp,
		bk:  bk,
		fs:  fs,
		srv: srv,
	})

	// wait until serve was done.
	srv.Quit()
	<-waitForDeath
}

func buildFingerprint(t *testing.T, u testUnit) peer.Fingerprint {
	ownPubKey, err := u.rp.Keyring().OwnPubKey()
	require.Nil(t, err)

	self, err := u.srv.Identity()
	require.Nil(t, err)

	return peer.BuildFingerprint(self.Addr, ownPubKey)
}

func withNetPair(t *testing.T, fn func(a, b testUnit)) {
	basePath, err := ioutil.TempDir("", "brig-net-test")
	require.Nil(t, err)

	defer func() {
		require.Nil(t, os.RemoveAll(basePath))
	}()

	withNetServer(t, "alice", 9998, basePath, func(a testUnit) {
		withNetServer(t, "bob", 9999, basePath, func(b testUnit) {
			// Add each other's fingerprints:
			require.Nil(t, a.rp.Remotes.AddRemote(repo.Remote{
				Name:        "bob",
				Fingerprint: buildFingerprint(t, b),
			}))
			require.Nil(t, b.rp.Remotes.AddRemote(repo.Remote{
				Name:        "alice",
				Fingerprint: buildFingerprint(t, a),
			}))

			aliCtl, err := Dial("alice", b.rp, b.bk, context.Background())
			require.Nil(t, err)

			bobCtl, err := Dial("bob", a.rp, a.bk, context.Background())
			require.Nil(t, err)

			a.ctl = bobCtl
			b.ctl = aliCtl

			fn(a, b)
		})
	})
}

func TestClientPing(t *testing.T) {
	withNetPair(t, func(a, b testUnit) {
		for i := 0; i < 100; i++ {
			if err := a.ctl.Ping(); err != nil {
				t.Fatalf("ping to bob failed: %v", err)
			}

			if err := b.ctl.Ping(); err != nil {
				t.Fatalf("ping to alice failed: %v", err)
			}
		}
	})
}

func TestClientFetchStore(t *testing.T) {
	withNetPair(t, func(a, b testUnit) {
		filePath := "/a/new/name/has/been/born"
		fileData := []byte{1, 2, 3}
		fileSrc := bytes.NewReader(fileData)

		if err := a.fs.Stage(filePath, fileSrc); err != nil {
			t.Fatalf("failed to stage simple file: %v", err)
		}

		data, err := b.ctl.FetchStore()
		if err != nil {
			t.Fatalf("failed to read store: %v", err)
		}

		aliceFs, err := b.rp.FS("alice", b.bk)
		if err != nil {
			t.Fatalf("Failed to get empty bob fs: %v", err)
		}

		_, err = aliceFs.Stat(filePath)
		if !ie.IsNoSuchFileError(err) {
			t.Fatalf("File has existed in bob's empty store (wtf?): %v", err)
		}

		if err := aliceFs.Import(data); err != nil {
			t.Fatalf("Failed to import data: %v", err)
		}

		info, err := aliceFs.Stat(filePath)
		if err != nil {
			t.Fatalf("Failed to read file exported from alice: %v", err)
		}

		// Check superficially that store was imported right:
		require.Equal(t, info.Path, filePath)
		require.Equal(t, info.User, "alice")
		require.Equal(t, info.Size, uint64(3))
		require.Equal(t, info.IsDir, false)
	})
}

func TestClientFetchPatch(t *testing.T) {
	withNetPair(t, func(a, b testUnit) {
		// Create a new file in alice's fs.
		require.Nil(t, a.fs.Stage("/new_file", bytes.NewReader([]byte{1, 2, 3})))

		// Get a patch from alice:
		patchData, err := b.ctl.FetchPatch(0)
		require.Nil(t, err)
		require.NotNil(t, patchData)

		// Create a new empty FS for alice.
		aliceFs, err := b.rp.FS("alice", b.bk)
		if err != nil {
			t.Fatalf("Failed to get empty bob fs: %v", err)
		}

		// It should have the initial patch version of 0.
		lastPatchIdx, err := aliceFs.LastPatchIndex()
		require.Nil(t, err)
		require.Equal(t, int64(0), lastPatchIdx)

		// After applying the patch, we should have bob's data.
		require.Nil(t, aliceFs.ApplyPatch(patchData))
		newFileInfo, err := aliceFs.Stat("/new_file")
		require.Nil(t, err)
		require.Equal(t, "/new_file", newFileInfo.Path)
		require.Equal(t, uint64(3), newFileInfo.Size)

		// Check that the new patch version is 2 (i.e. bob has two commits)
		lastPatchIdx, err = aliceFs.LastPatchIndex()
		require.Nil(t, err)
		require.Equal(t, int64(2), lastPatchIdx)

		// If we fetch the same patch again, it will be empty.
		// (data will be not len=0, but no real contents)
		patchData, err = b.ctl.FetchPatch(2)
		require.Nil(t, err)
		require.NotNil(t, patchData)
		require.Nil(t, aliceFs.ApplyPatch(patchData))

		// Last patch was empty, so should not bump the version.
		lastPatchIdx, err = aliceFs.LastPatchIndex()
		require.Nil(t, err)
		require.Equal(t, int64(2), lastPatchIdx)

		// Bob's patch index should not have changed.
		// For bob, the patch index does not make really sense,
		// since he's the owner of the fs and always has the latest version.
		lastBobPatchIdx, err := b.fs.LastPatchIndex()
		require.Nil(t, err)
		require.Equal(t, int64(0), lastBobPatchIdx)
	})
}

func TestClientCompleteFetchAllowed(t *testing.T) {
	withNetPair(t, func(a, b testUnit) {
		isAllowed, err := b.ctl.IsCompleteFetchAllowed()
		require.Nil(t, err)
		require.True(t, isAllowed)

		// Make the remote have only access to a specific sub folder:
		rmt, err := a.rp.Remotes.Remote("bob")
		require.Nil(t, err)

		err = a.rp.Remotes.SetRemote("bob", repo.Remote{
			Fingerprint: rmt.Fingerprint,
			Name:        rmt.Name,
			Folders: []repo.Folder{
				{
					Folder: "/photos",
				},
			},
		})
		require.Nil(t, err)

		isAllowed, err = b.ctl.IsCompleteFetchAllowed()
		require.Nil(t, err)
		require.False(t, isAllowed)

		// Try again with the root folder enabled:
		err = a.rp.Remotes.SetRemote("bob", repo.Remote{
			Fingerprint: rmt.Fingerprint,
			Name:        rmt.Name,
			Folders: []repo.Folder{
				{
					Folder: "/",
				},
				{
					Folder: "/photos",
				},
			},
		})
		require.Nil(t, err)

		// This also tests that we are able to change remote config
		// without needing to reconnect.
		isAllowed, err = b.ctl.IsCompleteFetchAllowed()
		require.Nil(t, err)
		require.True(t, isAllowed)
	})
}
<commit_msg>test: net: fix setup<commit_after>package net

import (
	"bytes"
	"context"
	"fmt"
	"io/ioutil"
	"os"
	"testing"
	"time"

	"github.com/sahib/brig/backend"
	"github.com/sahib/brig/catfs"
	ie "github.com/sahib/brig/catfs/errors"
	"github.com/sahib/brig/net/peer"
	"github.com/sahib/brig/repo"
	"github.com/stretchr/testify/require"
)

type testUnit struct {
	ctl *Client
	srv *Server
	fs  *catfs.FS
	rp  *repo.Repository
	bk  backend.Backend
}

func withNetServer(t *testing.T, name string, backendPort int, basePath string, fn func(u testUnit)) {
	basePath, err := ioutil.TempDir("", "brig-ctl-test")
	require.Nil(t, err)

	netDbPath, err := ioutil.TempDir("", "brig-net-db")
	require.Nil(t, err)

	defer func() {
		require.Nil(t, os.RemoveAll(basePath))
		require.Nil(t, os.RemoveAll(netDbPath))
	}()

	require.Nil(t, os.Setenv("BRIG_MOCK_PORT", fmt.Sprintf("%d", backendPort)))
	require.Nil(t, os.Setenv("BRIG_MOCK_USER", name))
	require.Nil(t, os.Setenv("BRIG_MOCK_NET_DB_PATH", netDbPath))
	bk, err := backend.FromName("mock", basePath, nil)
	require.Nil(t, err)

	err = repo.Init(basePath, name, "password", "mock")
	require.Nil(t, err)

	rp, err := repo.Open(basePath, "password")
	require.Nil(t, err)

	srv, err := NewServer(rp, bk)
	require.Nil(t, err)

	fs, err := rp.FS(name, bk)
	require.Nil(t, err)

	waitForDeath := make(chan bool)
	go func() {
		defer func() {
			waitForDeath <- true
		}()
		require.Nil(t, srv.Serve())
		require.Nil(t, srv.Close())
	}()

	// Allow a short time for the server go routine to fully boot up.
	time.Sleep(50 * time.Millisecond)

	// Run the actual test function:
	fn(testUnit{
		rp:  rp,
		bk:  bk,
		fs:  fs,
		srv: srv,
	})

	// wait until serve was done.
	srv.Quit()
	<-waitForDeath
}

func buildFingerprint(t *testing.T, u testUnit) peer.Fingerprint {
	ownPubKey, err := u.rp.Keyring().OwnPubKey()
	require.Nil(t, err)

	self, err := u.srv.Identity()
	require.Nil(t, err)

	return peer.BuildFingerprint(self.Addr, ownPubKey)
}

func withNetPair(t *testing.T, fn func(a, b testUnit)) {
	basePath, err := ioutil.TempDir("", "brig-net-test")
	require.Nil(t, err)

	defer func() {
		require.Nil(t, os.RemoveAll(basePath))
	}()

	withNetServer(t, "alice", 9998, basePath, func(a testUnit) {
		withNetServer(t, "bob", 9999, basePath, func(b testUnit) {
			// Add each other's fingerprints:
			require.Nil(t, a.rp.Remotes.AddRemote(repo.Remote{
				Name:        "bob",
				Fingerprint: buildFingerprint(t, b),
			}))
			require.Nil(t, b.rp.Remotes.AddRemote(repo.Remote{
				Name:        "alice",
				Fingerprint: buildFingerprint(t, a),
			}))

			aliCtl, err := Dial("alice", b.rp, b.bk, context.Background())
			require.Nil(t, err)

			bobCtl, err := Dial("bob", a.rp, a.bk, context.Background())
			require.Nil(t, err)

			a.ctl = bobCtl
			b.ctl = aliCtl

			fn(a, b)
		})
	})
}

func TestClientPing(t *testing.T) {
	withNetPair(t, func(a, b testUnit) {
		for i := 0; i < 100; i++ {
			if err := a.ctl.Ping(); err != nil {
				t.Fatalf("ping to bob failed: %v", err)
			}

			if err := b.ctl.Ping(); err != nil {
				t.Fatalf("ping to alice failed: %v", err)
			}
		}
	})
}

func TestClientFetchStore(t *testing.T) {
	withNetPair(t, func(a, b testUnit) {
		filePath := "/a/new/name/has/been/born"
		fileData := []byte{1, 2, 3}
		fileSrc := bytes.NewReader(fileData)

		if err := a.fs.Stage(filePath, fileSrc); err != nil {
			t.Fatalf("failed to stage simple file: %v", err)
		}

		data, err := b.ctl.FetchStore()
		if err != nil {
			t.Fatalf("failed to read store: %v", err)
		}

		aliceFs, err := b.rp.FS("alice", b.bk)
		if err != nil {
			t.Fatalf("Failed to get empty bob fs: %v", err)
		}

		_, err = aliceFs.Stat(filePath)
		if !ie.IsNoSuchFileError(err) {
			t.Fatalf("File has existed in bob's empty store (wtf?): %v", err)
		}

		if err := aliceFs.Import(data); err != nil {
			t.Fatalf("Failed to import data: %v", err)
		}

		info, err := aliceFs.Stat(filePath)
		if err != nil {
			t.Fatalf("Failed to read file exported from alice: %v", err)
		}

		// Check superficially that store was imported right:
		require.Equal(t, info.Path, filePath)
		require.Equal(t, info.User, "alice")
		require.Equal(t, info.Size, uint64(3))
		require.Equal(t, info.IsDir, false)
	})
}

func TestClientFetchPatch(t *testing.T) {
	withNetPair(t, func(a, b testUnit) {
		// Create a new file in alice's fs.
		require.Nil(t, a.fs.Stage("/new_file", bytes.NewReader([]byte{1, 2, 3})))

		// Get a patch from alice:
		patchData, err := b.ctl.FetchPatch(0)
		require.Nil(t, err)
		require.NotNil(t, patchData)

		// Create a new empty FS for alice.
		aliceFs, err := b.rp.FS("alice", b.bk)
		if err != nil {
			t.Fatalf("Failed to get empty bob fs: %v", err)
		}

		// It should have the initial patch version of 0.
		lastPatchIdx, err := aliceFs.LastPatchIndex()
		require.Nil(t, err)
		require.Equal(t, int64(0), lastPatchIdx)

		// After applying the patch, we should have bob's data.
		require.Nil(t, aliceFs.ApplyPatch(patchData))
		newFileInfo, err := aliceFs.Stat("/new_file")
		require.Nil(t, err)
		require.Equal(t, "/new_file", newFileInfo.Path)
		require.Equal(t, uint64(3), newFileInfo.Size)

		// Check that the new patch version is 2 (i.e. bob has two commits)
		lastPatchIdx, err = aliceFs.LastPatchIndex()
		require.Nil(t, err)
		require.Equal(t, int64(2), lastPatchIdx)

		// If we fetch the same patch again, it will be empty.
		// (data will be not len=0, but no real contents)
		patchData, err = b.ctl.FetchPatch(2)
		require.Nil(t, err)
		require.NotNil(t, patchData)
		require.Nil(t, aliceFs.ApplyPatch(patchData))

		// Last patch was empty, so should not bump the version.
		lastPatchIdx, err = aliceFs.LastPatchIndex()
		require.Nil(t, err)
		require.Equal(t, int64(2), lastPatchIdx)

		// Bob's patch index should not have changed.
		// For bob, the patch index does not make really sense,
		// since he's the owner of the fs and always has the latest version.
		lastBobPatchIdx, err := b.fs.LastPatchIndex()
		require.Nil(t, err)
		require.Equal(t, int64(0), lastBobPatchIdx)
	})
}

func TestClientCompleteFetchAllowed(t *testing.T) {
	withNetPair(t, func(a, b testUnit) {
		isAllowed, err := b.ctl.IsCompleteFetchAllowed()
		require.Nil(t, err)
		require.True(t, isAllowed)

		// Make the remote have only access to a specific sub folder:
		rmt, err := a.rp.Remotes.Remote("bob")
		require.Nil(t, err)

		err = a.rp.Remotes.SetRemote("bob", repo.Remote{
			Fingerprint: rmt.Fingerprint,
			Name:        rmt.Name,
			Folders: []repo.Folder{
				{
					Folder: "/photos",
				},
			},
		})
		require.Nil(t, err)

		isAllowed, err = b.ctl.IsCompleteFetchAllowed()
		require.Nil(t, err)
		require.False(t, isAllowed)

		// Try again with the root folder enabled:
		err = a.rp.Remotes.SetRemote("bob", repo.Remote{
			Fingerprint: rmt.Fingerprint,
			Name:        rmt.Name,
			Folders: []repo.Folder{
				{
					Folder: "/",
				},
				{
					Folder: "/photos",
				},
			},
		})
		require.Nil(t, err)

		// This also tests that we are able to change remote config
		// without needing to reconnect.
		isAllowed, err = b.ctl.IsCompleteFetchAllowed()
		require.Nil(t, err)
		require.True(t, isAllowed)
	})
}
<|endoftext|>