<commit_before>/**
* Copyright 2021 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*      http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
 */

// Package modulewriter writes modules to a deployment directory
package modulewriter

import (
	"embed"
	"fmt"
	"hpc-toolkit/pkg/config"
	"hpc-toolkit/pkg/deploymentio"
	"hpc-toolkit/pkg/sourcereader"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
)

const (
	hiddenGhpcDirName          = ".ghpc"
	prevDeploymentGroupDirName = "previous_resource_groups"
	gitignoreTemplate          = "deployment.gitignore.tmpl"
)

// ModuleWriter interface for writing modules to a deployment
type ModuleWriter interface {
	getNumModules() int
	addNumModules(int)
	writeDeploymentGroup(
		depGroup config.DeploymentGroup,
		globalVars map[string]interface{},
		deployDir string,
	) error
	restoreState(deploymentDir string) error
}

var kinds = map[string]ModuleWriter{
	"terraform": new(TFWriter),
	"packer":    new(PackerWriter),
}

//go:embed *.tmpl
var templatesFS embed.FS

func factory(kind string) ModuleWriter {
	writer, exists := kinds[kind]
	if !exists {
		log.Fatalf(
			"modulewriter: Module kind (%s) is not valid. "+
				"kind must be in (terraform, packer).", kind)
	}
	return writer
}

// WriteDeployment writes a deployment directory using modules defined the
// environment blueprint.
func WriteDeployment(blueprint *config.Blueprint, outputDir string, overwriteFlag bool) error {
	deploymentName, err := blueprint.DeploymentName()
	if err != nil {
		return err
	}
	deploymentDir := filepath.Join(outputDir, deploymentName)

	overwrite := isOverwriteAllowed(deploymentDir, blueprint, overwriteFlag)
	if err := prepDepDir(deploymentDir, overwrite); err != nil {
		return err
	}

	copySource(deploymentDir, &blueprint.DeploymentGroups)

	for _, grp := range blueprint.DeploymentGroups {

		deploymentName, err := blueprint.DeploymentName()
		if err != nil {
			return err
		}

		deploymentPath := filepath.Join(outputDir, deploymentName)
		writer, ok := kinds[grp.Kind]
		if !ok {
			return fmt.Errorf(
				"Invalid kind in deployment group %s, got '%s'", grp.Name, grp.Kind)
		}

		if err := writer.writeDeploymentGroup(
			grp, blueprint.Vars, deploymentPath,
		); err != nil {
			return fmt.Errorf("error writing deployment group %s: %w", grp.Name, err)
		}
	}

	for _, writer := range kinds {
		if writer.getNumModules() > 0 {
			if err := writer.restoreState(deploymentDir); err != nil {
				return fmt.Errorf("error trying to restore terraform state: %w", err)
			}
		}
	}
	return nil
}

func copySource(deploymentPath string, deploymentGroups *[]config.DeploymentGroup) {
	for iGrp, grp := range *deploymentGroups {
		for iMod, module := range grp.Modules {
			if sourcereader.IsGitHubPath(module.Source) {
				continue
			}

			/* Copy source files */
			moduleName := filepath.Base(module.Source)
			(*deploymentGroups)[iGrp].Modules[iMod].ModuleName = moduleName
			basePath := filepath.Join(deploymentPath, grp.Name)
			var destPath string
			switch module.Kind {
			case "terraform":
				destPath = filepath.Join(basePath, "modules", moduleName)
			case "packer":
				destPath = filepath.Join(basePath, module.ID)
			}
			_, err := os.Stat(destPath)
			if err == nil {
				continue
			}

			reader := sourcereader.Factory(module.Source)
			if err := reader.GetModule(module.Source, destPath); err != nil {
				log.Fatalf("failed to get module from %s to %s: %v", module.Source, destPath, err)
			}

			/* Create module level files */
			writer := factory(module.Kind)
			writer.addNumModules(1)
		}
	}
}

func printInstructionsPreamble(kind string, path string) {
	fmt.Printf("%s group was successfully created in directory %s\n", kind, path)
	fmt.Println("To deploy, run the following commands:")
}

// Determines if overwrite is allowed
func isOverwriteAllowed(depDir string, overwritingConfig *config.Blueprint, overwriteFlag bool) bool {
	if !overwriteFlag {
		return false
	}

	files, err := ioutil.ReadDir(depDir)
	if err != nil {
		return false
	}

	// build list of previous and current resource groups
	var prevGroups []string
	for _, f := range files {
		if f.IsDir() && f.Name() != hiddenGhpcDirName {
			prevGroups = append(prevGroups, f.Name())
		}
	}

	var curGroups []string
	for _, group := range overwritingConfig.DeploymentGroups {
		curGroups = append(curGroups, group.Name)
	}

	return isSubset(prevGroups, curGroups)
}

func isSubset(sub, super []string) bool {
	// build set (map keys) from slice
	superM := make(map[string]bool)
	for _, item := range super {
		superM[item] = true
	}

	for _, item := range sub {
		if _, found := superM[item]; !found {
			return false
		}
	}
	return true
}

// OverwriteDeniedError signifies when a deployment overwrite was denied.
type OverwriteDeniedError struct {
	cause error
}

func (err *OverwriteDeniedError) Error() string {
	return fmt.Sprintf("Failed to overwrite existing deployment.\n\n"+
		"Use the -w command line argument to enable overwrite.\n"+
		"If overwrite is already enabled then this may be because "+
		"you are attempting to remove a deployment group, which is not supported.\n"+
		"original error: %v",
		err.cause)
}

// Prepares a deployment directory to be written to.
func prepDepDir(depDir string, overwrite bool) error {
	deploymentio := deploymentio.GetDeploymentioLocal()
	ghpcDir := filepath.Join(depDir, hiddenGhpcDirName)
	gitignoreFile := filepath.Join(depDir, ".gitignore")

	// create deployment directory
	if err := deploymentio.CreateDirectory(depDir); err != nil {
		if !overwrite {
			return &OverwriteDeniedError{err}
		}

		// Confirm we have a previously written deployment dir before overwritting.
		if _, err := os.Stat(ghpcDir); os.IsNotExist(err) {
			return fmt.Errorf(
				"While trying to update the deployment directory at %s, the '.ghpc/' dir could not be found", depDir)
		}
	} else {
		if err := deploymentio.CreateDirectory(ghpcDir); err != nil {
			return fmt.Errorf("Failed to create directory at %s: err=%w", ghpcDir, err)
		}

		if err := deploymentio.CopyFromFS(templatesFS, gitignoreTemplate, gitignoreFile); err != nil {
			return fmt.Errorf("Failed to copy template.gitignore file to %s: err=%w", gitignoreFile, err)
		}
	}

	// clean up old dirs
	prevGroupDir := filepath.Join(ghpcDir, prevDeploymentGroupDirName)
	os.RemoveAll(prevGroupDir)
	if err := os.MkdirAll(prevGroupDir, 0755); err != nil {
		return fmt.Errorf("Failed to create directory to save previous deployment groups at %s: %w", prevGroupDir, err)
	}

	// move resource groups
	files, err := ioutil.ReadDir(depDir)
	if err != nil {
		return fmt.Errorf("Error trying to read directories in %s, %w", depDir, err)
	}
	for _, f := range files {
		if !f.IsDir() || f.Name() == hiddenGhpcDirName {
			continue
		}
		src := filepath.Join(depDir, f.Name())
		dest := filepath.Join(prevGroupDir, f.Name())
		if err := os.Rename(src, dest); err != nil {
			return fmt.Errorf("Error while moving previous deployment groups: failed on %s: %w", f.Name(), err)
		}
	}
	return nil
}
<commit_msg>Update name of previous resource groups folder to match new schema<commit_after>/**
* Copyright 2021 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*      http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
 */

// Package modulewriter writes modules to a deployment directory
package modulewriter

import (
	"embed"
	"fmt"
	"hpc-toolkit/pkg/config"
	"hpc-toolkit/pkg/deploymentio"
	"hpc-toolkit/pkg/sourcereader"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
)

const (
	hiddenGhpcDirName          = ".ghpc"
	prevDeploymentGroupDirName = "previous_deployment_groups"
	gitignoreTemplate          = "deployment.gitignore.tmpl"
)

// ModuleWriter interface for writing modules to a deployment
type ModuleWriter interface {
	getNumModules() int
	addNumModules(int)
	writeDeploymentGroup(
		depGroup config.DeploymentGroup,
		globalVars map[string]interface{},
		deployDir string,
	) error
	restoreState(deploymentDir string) error
}

var kinds = map[string]ModuleWriter{
	"terraform": new(TFWriter),
	"packer":    new(PackerWriter),
}

//go:embed *.tmpl
var templatesFS embed.FS

func factory(kind string) ModuleWriter {
	writer, exists := kinds[kind]
	if !exists {
		log.Fatalf(
			"modulewriter: Module kind (%s) is not valid. "+
				"kind must be in (terraform, packer).", kind)
	}
	return writer
}

// WriteDeployment writes a deployment directory using modules defined the
// environment blueprint.
func WriteDeployment(blueprint *config.Blueprint, outputDir string, overwriteFlag bool) error {
	deploymentName, err := blueprint.DeploymentName()
	if err != nil {
		return err
	}
	deploymentDir := filepath.Join(outputDir, deploymentName)

	overwrite := isOverwriteAllowed(deploymentDir, blueprint, overwriteFlag)
	if err := prepDepDir(deploymentDir, overwrite); err != nil {
		return err
	}

	copySource(deploymentDir, &blueprint.DeploymentGroups)

	for _, grp := range blueprint.DeploymentGroups {

		deploymentName, err := blueprint.DeploymentName()
		if err != nil {
			return err
		}

		deploymentPath := filepath.Join(outputDir, deploymentName)
		writer, ok := kinds[grp.Kind]
		if !ok {
			return fmt.Errorf(
				"Invalid kind in deployment group %s, got '%s'", grp.Name, grp.Kind)
		}

		if err := writer.writeDeploymentGroup(
			grp, blueprint.Vars, deploymentPath,
		); err != nil {
			return fmt.Errorf("error writing deployment group %s: %w", grp.Name, err)
		}
	}

	for _, writer := range kinds {
		if writer.getNumModules() > 0 {
			if err := writer.restoreState(deploymentDir); err != nil {
				return fmt.Errorf("error trying to restore terraform state: %w", err)
			}
		}
	}
	return nil
}

func copySource(deploymentPath string, deploymentGroups *[]config.DeploymentGroup) {
	for iGrp, grp := range *deploymentGroups {
		for iMod, module := range grp.Modules {
			if sourcereader.IsGitHubPath(module.Source) {
				continue
			}

			/* Copy source files */
			moduleName := filepath.Base(module.Source)
			(*deploymentGroups)[iGrp].Modules[iMod].ModuleName = moduleName
			basePath := filepath.Join(deploymentPath, grp.Name)
			var destPath string
			switch module.Kind {
			case "terraform":
				destPath = filepath.Join(basePath, "modules", moduleName)
			case "packer":
				destPath = filepath.Join(basePath, module.ID)
			}
			_, err := os.Stat(destPath)
			if err == nil {
				continue
			}

			reader := sourcereader.Factory(module.Source)
			if err := reader.GetModule(module.Source, destPath); err != nil {
				log.Fatalf("failed to get module from %s to %s: %v", module.Source, destPath, err)
			}

			/* Create module level files */
			writer := factory(module.Kind)
			writer.addNumModules(1)
		}
	}
}

func printInstructionsPreamble(kind string, path string) {
	fmt.Printf("%s group was successfully created in directory %s\n", kind, path)
	fmt.Println("To deploy, run the following commands:")
}

// Determines if overwrite is allowed
func isOverwriteAllowed(depDir string, overwritingConfig *config.Blueprint, overwriteFlag bool) bool {
	if !overwriteFlag {
		return false
	}

	files, err := ioutil.ReadDir(depDir)
	if err != nil {
		return false
	}

	// build list of previous and current deployment groups
	var prevGroups []string
	for _, f := range files {
		if f.IsDir() && f.Name() != hiddenGhpcDirName {
			prevGroups = append(prevGroups, f.Name())
		}
	}

	var curGroups []string
	for _, group := range overwritingConfig.DeploymentGroups {
		curGroups = append(curGroups, group.Name)
	}

	return isSubset(prevGroups, curGroups)
}

func isSubset(sub, super []string) bool {
	// build set (map keys) from slice
	superM := make(map[string]bool)
	for _, item := range super {
		superM[item] = true
	}

	for _, item := range sub {
		if _, found := superM[item]; !found {
			return false
		}
	}
	return true
}

// OverwriteDeniedError signifies when a deployment overwrite was denied.
type OverwriteDeniedError struct {
	cause error
}

func (err *OverwriteDeniedError) Error() string {
	return fmt.Sprintf("Failed to overwrite existing deployment.\n\n"+
		"Use the -w command line argument to enable overwrite.\n"+
		"If overwrite is already enabled then this may be because "+
		"you are attempting to remove a deployment group, which is not supported.\n"+
		"original error: %v",
		err.cause)
}

// Prepares a deployment directory to be written to.
func prepDepDir(depDir string, overwrite bool) error {
	deploymentio := deploymentio.GetDeploymentioLocal()
	ghpcDir := filepath.Join(depDir, hiddenGhpcDirName)
	gitignoreFile := filepath.Join(depDir, ".gitignore")

	// create deployment directory
	if err := deploymentio.CreateDirectory(depDir); err != nil {
		if !overwrite {
			return &OverwriteDeniedError{err}
		}

		// Confirm we have a previously written deployment dir before overwritting.
		if _, err := os.Stat(ghpcDir); os.IsNotExist(err) {
			return fmt.Errorf(
				"While trying to update the deployment directory at %s, the '.ghpc/' dir could not be found", depDir)
		}
	} else {
		if err := deploymentio.CreateDirectory(ghpcDir); err != nil {
			return fmt.Errorf("Failed to create directory at %s: err=%w", ghpcDir, err)
		}

		if err := deploymentio.CopyFromFS(templatesFS, gitignoreTemplate, gitignoreFile); err != nil {
			return fmt.Errorf("Failed to copy template.gitignore file to %s: err=%w", gitignoreFile, err)
		}
	}

	// clean up old dirs
	prevGroupDir := filepath.Join(ghpcDir, prevDeploymentGroupDirName)
	os.RemoveAll(prevGroupDir)
	if err := os.MkdirAll(prevGroupDir, 0755); err != nil {
		return fmt.Errorf("Failed to create directory to save previous deployment groups at %s: %w", prevGroupDir, err)
	}

	// move deployment groups
	files, err := ioutil.ReadDir(depDir)
	if err != nil {
		return fmt.Errorf("Error trying to read directories in %s, %w", depDir, err)
	}
	for _, f := range files {
		if !f.IsDir() || f.Name() == hiddenGhpcDirName {
			continue
		}
		src := filepath.Join(depDir, f.Name())
		dest := filepath.Join(prevGroupDir, f.Name())
		if err := os.Rename(src, dest); err != nil {
			return fmt.Errorf("Error while moving previous deployment groups: failed on %s: %w", f.Name(), err)
		}
	}
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package serverconfig

import (
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"net/http"
	"os"
	"strconv"
	"strings"

	"camlistore.org/pkg/auth"
	"camlistore.org/pkg/blobserver"
	"camlistore.org/pkg/blobserver/handlers"
	"camlistore.org/pkg/httputil"
	"camlistore.org/pkg/jsonconfig"
)

const camliPrefix = "/camli/"

var ErrCamliPath = errors.New("Invalid Camlistore request path")

type handlerConfig struct {
	prefix string         // "/foo/"
	htype  string         // "localdisk", etc
	conf   jsonconfig.Obj // never nil

	settingUp, setupDone bool
}

type handlerLoader struct {
	installer HandlerInstaller
	baseURL   string
	config    map[string]*handlerConfig // prefix -> config
	handler   map[string]interface{}    // prefix -> http.Handler / func / blobserver.Storage

	// optional context (for App Engine, the first request that
	// started up the process).  we may need this if setting up
	// handlers involves doing datastore/memcache/blobstore
	// lookups.
	context *http.Request
}

type HandlerInstaller interface {
	Handle(path string, handler http.Handler)
}

type storageAndConfig struct {
	blobserver.Storage
	config *blobserver.Config
}

var _ blobserver.ContextWrapper = (*storageAndConfig)(nil)

func (sc *storageAndConfig) WrapContext(req *http.Request) blobserver.Storage {
	if w, ok := sc.Storage.(blobserver.ContextWrapper); ok {
		return &storageAndConfig{w.WrapContext(req), sc.config}
	}
	return sc
}

func parseCamliPath(path string) (action string, err error) {
	camIdx := strings.Index(path, camliPrefix)
	if camIdx == -1 {
		return "", ErrCamliPath
	}
	action = path[camIdx+len(camliPrefix):]
	return
}

func unsupportedHandler(conn http.ResponseWriter, req *http.Request) {
	httputil.BadRequestError(conn, "Unsupported camlistore path or method.")
}

func (s *storageAndConfig) Config() *blobserver.Config {
	return s.config
}

func handleCamliUsingStorage(conn http.ResponseWriter, req *http.Request, action string, storage blobserver.StorageConfiger) {
	handler := unsupportedHandler
	switch req.Method {
	case "GET":
		switch action {
		case "enumerate-blobs":
			handler = auth.RequireAuth(handlers.CreateEnumerateHandler(storage))
		case "stat":
			handler = auth.RequireAuth(handlers.CreateStatHandler(storage))
		default:
			handler = handlers.CreateGetHandler(storage)
		}
	case "POST":
		switch action {
		case "stat":
			handler = auth.RequireAuth(handlers.CreateStatHandler(storage))
		case "upload":
			handler = auth.RequireAuth(handlers.CreateUploadHandler(storage))
		case "remove":
			handler = auth.RequireAuth(handlers.CreateRemoveHandler(storage))
		}
	case "PUT": // no longer part of spec
		handler = auth.RequireAuth(handlers.CreateNonStandardPutHandler(storage))
	}
	handler(conn, req)
}

// where prefix is like "/" or "/s3/" for e.g. "/camli/" or "/s3/camli/*"
func makeCamliHandler(prefix, baseURL string, storage blobserver.Storage) http.Handler {
	if !strings.HasSuffix(prefix, "/") {
		panic("expected prefix to end in slash")
	}
	baseURL = strings.TrimRight(baseURL, "/")

	canLongPoll := true
	// TODO(bradfitz): set to false if this is App Engine, or provide some way to disable

	storageConfig := &storageAndConfig{
		storage,
		&blobserver.Config{
			Writable:    true,
			Readable:    true,
			IsQueue:     false,
			URLBase:     baseURL + prefix[:len(prefix)-1],
			CanLongPoll: canLongPoll,
		},
	}
	return http.HandlerFunc(func(conn http.ResponseWriter, req *http.Request) {
		action, err := parseCamliPath(req.URL.Path[len(prefix)-1:])
		if err != nil {
			log.Printf("Invalid request for method %q, path %q",
				req.Method, req.URL.Path)
			unsupportedHandler(conn, req)
			return
		}
		handleCamliUsingStorage(conn, req, action, storageConfig)
	})
}

func (hl *handlerLoader) GetRequestContext() (req *http.Request, ok bool) {
	return hl.context, hl.context != nil
}

func (hl *handlerLoader) FindHandlerByTypeIfLoaded(htype string) (prefix string, handler interface{}, err error) {
	for prefix, config := range hl.config {
		if config.htype == htype {
			return prefix, hl.handler[prefix], nil
		}
	}
	return "", nil, os.ErrNotExist
}

func (hl *handlerLoader) setupAll() {
	for prefix := range hl.config {
		hl.setupHandler(prefix)
	}
}

func (hl *handlerLoader) configType(prefix string) string {
	if h, ok := hl.config[prefix]; ok {
		return h.htype
	}
	return ""
}

func (hl *handlerLoader) getOrSetup(prefix string) interface{} {
	hl.setupHandler(prefix)
	return hl.handler[prefix]
}

func (hl *handlerLoader) GetStorage(prefix string) (blobserver.Storage, error) {
	hl.setupHandler(prefix)
	if s, ok := hl.handler[prefix].(blobserver.Storage); ok {
		return s, nil
	}
	return nil, fmt.Errorf("bogus storage handler referenced as %q", prefix)
}

func (hl *handlerLoader) GetHandler(prefix string) (interface{}, error) {
	hl.setupHandler(prefix)
	if s, ok := hl.handler[prefix].(blobserver.Storage); ok {
		return s, nil
	}
	if h, ok := hl.handler[prefix].(http.Handler); ok {
		return h, nil
	}
	return nil, fmt.Errorf("bogus http or storage handler referenced as %q", prefix)
}

func (hl *handlerLoader) GetHandlerType(prefix string) string {
	hl.setupHandler(prefix)
	return hl.configType(prefix)
}

func exitFailure(pattern string, args ...interface{}) {
	if !strings.HasSuffix(pattern, "\n") {
		pattern = pattern + "\n"
	}
	panic(fmt.Sprintf(pattern, args...))
}

func (hl *handlerLoader) setupHandler(prefix string) {
	h, ok := hl.config[prefix]
	if !ok {
		exitFailure("invalid reference to undefined handler %q", prefix)
	}
	if h.setupDone {
		// Already setup by something else reference it and forcing it to be
		// setup before the bottom loop got to it.
		return
	}
	if h.settingUp {
		exitFailure("loop in configuration graph; %q tried to load itself indirectly", prefix)
	}
	h.settingUp = true
	defer func() {
		h.setupDone = true
		r := recover()
		if r == nil {
			if hl.handler[prefix] == nil {
				panic(fmt.Sprintf("setupHandler for %q didn't install a handler", prefix))
			}
		} else {
			panic(r)
		}
	}()

	if strings.HasPrefix(h.htype, "storage-") {
		stype := h.htype[len("storage-"):]
		// Assume a storage interface
		pstorage, err := blobserver.CreateStorage(stype, hl, h.conf)
		if err != nil {
			exitFailure("error instantiating storage for prefix %q, type %q: %v",
				h.prefix, stype, err)
		}
		hl.handler[h.prefix] = pstorage
		hl.installer.Handle(prefix+"camli/", makeCamliHandler(prefix, hl.baseURL, pstorage))
		return
	}

	hh, err := blobserver.CreateHandler(h.htype, hl, h.conf)
	if err != nil {
		exitFailure("error instantiating handler for prefix %q, type %q: %v",
			h.prefix, h.htype, err)
	}
	hl.handler[prefix] = hh
	var wrappedHandler http.Handler = &httputil.PrefixHandler{prefix, hh}
	if handerTypeWantsAuth(h.htype) {
		wrappedHandler = auth.Handler{wrappedHandler}
	}
	hl.installer.Handle(prefix, wrappedHandler)
}

func handerTypeWantsAuth(handlerType string) bool {
	// TODO(bradfitz): ask the handler instead? This is a bit of a
	// weird spot for this policy maybe?
	switch handlerType {
	case "ui", "search", "jsonsign", "sync":
		return true
	}
	return false
}

type Config struct {
	jsonconfig.Obj
	UIPath     string // Not valid until after InstallHandlers
	configPath string // Filesystem path
}

// Load returns a low-level "handler config" from the provided filename.
// If the config file doesn't contain a top-level JSON key of "handlerConfig"
// with boolean value true, the configuration is assumed to be a high-level
// "user config" file, and transformed into a low-level config.
func Load(filename string) (*Config, error) {
	obj, err := jsonconfig.ReadFile(filename)
	if err != nil {
		return nil, err
	}
	conf := &Config{
		Obj:        obj,
		configPath: filename,
	}

	if lowLevel := obj.OptionalBool("handlerConfig", false); !lowLevel {
		conf, err = GenLowLevelConfig(conf)
		if err != nil {
			return nil, fmt.Errorf(
				"Failed to transform user config file %q into internal handler configuration: %v",
				filename, err)
		}
		if v, _ := strconv.ParseBool(os.Getenv("CAMLI_DEBUG_CONFIG")); v {
			jsconf, _ := json.MarshalIndent(conf.Obj, "", "  ")
			log.Printf("From high-level config, generated low-level config: %s", jsconf)
		}
	}

	return conf, nil
}

func (config *Config) initAuth() error {
	authConfig := config.OptionalString("auth", "")
	_, err := auth.FromConfig(authConfig)
	return err
}

// context may be nil
func (config *Config) InstallHandlers(hi HandlerInstaller, baseURL string, context *http.Request) (outerr error) {
	defer func() {
		err := recover()
		if err == nil {
			return
		}
		outerr = fmt.Errorf("%v", err)
	}()

	err := config.initAuth()
	if err != nil {
		return fmt.Errorf("error while configuring auth: %v", err)
	}
	prefixes := config.RequiredObject("prefixes")
	if err := config.Validate(); err != nil {
		return fmt.Errorf("configuration error in root object's keys: %v", err)
	}

	hl := &handlerLoader{
		installer: hi,
		baseURL:   baseURL,
		config:    make(map[string]*handlerConfig),
		handler:   make(map[string]interface{}),
		context:   context,
	}

	for prefix, vei := range prefixes {
		if !strings.HasPrefix(prefix, "/") {
			exitFailure("prefix %q doesn't start with /", prefix)
		}
		if !strings.HasSuffix(prefix, "/") {
			exitFailure("prefix %q doesn't end with /", prefix)
		}
		pmap, ok := vei.(map[string]interface{})
		if !ok {
			exitFailure("prefix %q value is a %T, not an object", prefix, vei)
		}
		pconf := jsonconfig.Obj(pmap)
		enabled := pconf.OptionalBool("enabled", true)
		if !enabled {
			continue
		}
		handlerType := pconf.RequiredString("handler")
		handlerArgs := pconf.OptionalObject("handlerArgs")
		if err := pconf.Validate(); err != nil {
			exitFailure("configuration error in prefix %s: %v", prefix, err)
		}
		h := &handlerConfig{
			prefix: prefix,
			htype:  handlerType,
			conf:   handlerArgs,
		}
		hl.config[prefix] = h

		if handlerType == "ui" {
			config.UIPath = prefix
		}
	}
	hl.setupAll()
	return nil
}
<commit_msg>rename a function<commit_after>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package serverconfig

import (
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"net/http"
	"os"
	"strconv"
	"strings"

	"camlistore.org/pkg/auth"
	"camlistore.org/pkg/blobserver"
	"camlistore.org/pkg/blobserver/handlers"
	"camlistore.org/pkg/httputil"
	"camlistore.org/pkg/jsonconfig"
)

const camliPrefix = "/camli/"

var ErrCamliPath = errors.New("Invalid Camlistore request path")

type handlerConfig struct {
	prefix string         // "/foo/"
	htype  string         // "localdisk", etc
	conf   jsonconfig.Obj // never nil

	settingUp, setupDone bool
}

type handlerLoader struct {
	installer HandlerInstaller
	baseURL   string
	config    map[string]*handlerConfig // prefix -> config
	handler   map[string]interface{}    // prefix -> http.Handler / func / blobserver.Storage

	// optional context (for App Engine, the first request that
	// started up the process).  we may need this if setting up
	// handlers involves doing datastore/memcache/blobstore
	// lookups.
	context *http.Request
}

type HandlerInstaller interface {
	Handle(path string, handler http.Handler)
}

type storageAndConfig struct {
	blobserver.Storage
	config *blobserver.Config
}

var _ blobserver.ContextWrapper = (*storageAndConfig)(nil)

func (sc *storageAndConfig) WrapContext(req *http.Request) blobserver.Storage {
	if w, ok := sc.Storage.(blobserver.ContextWrapper); ok {
		return &storageAndConfig{w.WrapContext(req), sc.config}
	}
	return sc
}

func parseCamliPath(path string) (action string, err error) {
	camIdx := strings.Index(path, camliPrefix)
	if camIdx == -1 {
		return "", ErrCamliPath
	}
	action = path[camIdx+len(camliPrefix):]
	return
}

func unsupportedHandler(conn http.ResponseWriter, req *http.Request) {
	httputil.BadRequestError(conn, "Unsupported camlistore path or method.")
}

func (s *storageAndConfig) Config() *blobserver.Config {
	return s.config
}

func handleCamliUsingStorage(conn http.ResponseWriter, req *http.Request, action string, storage blobserver.StorageConfiger) {
	handler := unsupportedHandler
	switch req.Method {
	case "GET":
		switch action {
		case "enumerate-blobs":
			handler = auth.RequireAuth(handlers.CreateEnumerateHandler(storage))
		case "stat":
			handler = auth.RequireAuth(handlers.CreateStatHandler(storage))
		default:
			handler = handlers.CreateGetHandler(storage)
		}
	case "POST":
		switch action {
		case "stat":
			handler = auth.RequireAuth(handlers.CreateStatHandler(storage))
		case "upload":
			handler = auth.RequireAuth(handlers.CreateUploadHandler(storage))
		case "remove":
			handler = auth.RequireAuth(handlers.CreateRemoveHandler(storage))
		}
	case "PUT": // no longer part of spec
		handler = auth.RequireAuth(handlers.CreateNonStandardPutHandler(storage))
	}
	handler(conn, req)
}

// where prefix is like "/" or "/s3/" for e.g. "/camli/" or "/s3/camli/*"
func makeCamliHandler(prefix, baseURL string, storage blobserver.Storage) http.Handler {
	if !strings.HasSuffix(prefix, "/") {
		panic("expected prefix to end in slash")
	}
	baseURL = strings.TrimRight(baseURL, "/")

	canLongPoll := true
	// TODO(bradfitz): set to false if this is App Engine, or provide some way to disable

	storageConfig := &storageAndConfig{
		storage,
		&blobserver.Config{
			Writable:    true,
			Readable:    true,
			IsQueue:     false,
			URLBase:     baseURL + prefix[:len(prefix)-1],
			CanLongPoll: canLongPoll,
		},
	}
	return http.HandlerFunc(func(conn http.ResponseWriter, req *http.Request) {
		action, err := parseCamliPath(req.URL.Path[len(prefix)-1:])
		if err != nil {
			log.Printf("Invalid request for method %q, path %q",
				req.Method, req.URL.Path)
			unsupportedHandler(conn, req)
			return
		}
		handleCamliUsingStorage(conn, req, action, storageConfig)
	})
}

func (hl *handlerLoader) GetRequestContext() (req *http.Request, ok bool) {
	return hl.context, hl.context != nil
}

func (hl *handlerLoader) FindHandlerByTypeIfLoaded(htype string) (prefix string, handler interface{}, err error) {
	for prefix, config := range hl.config {
		if config.htype == htype {
			return prefix, hl.handler[prefix], nil
		}
	}
	return "", nil, os.ErrNotExist
}

func (hl *handlerLoader) setupAll() {
	for prefix := range hl.config {
		hl.setupHandler(prefix)
	}
}

func (hl *handlerLoader) configType(prefix string) string {
	if h, ok := hl.config[prefix]; ok {
		return h.htype
	}
	return ""
}

func (hl *handlerLoader) getOrSetup(prefix string) interface{} {
	hl.setupHandler(prefix)
	return hl.handler[prefix]
}

func (hl *handlerLoader) GetStorage(prefix string) (blobserver.Storage, error) {
	hl.setupHandler(prefix)
	if s, ok := hl.handler[prefix].(blobserver.Storage); ok {
		return s, nil
	}
	return nil, fmt.Errorf("bogus storage handler referenced as %q", prefix)
}

func (hl *handlerLoader) GetHandler(prefix string) (interface{}, error) {
	hl.setupHandler(prefix)
	if s, ok := hl.handler[prefix].(blobserver.Storage); ok {
		return s, nil
	}
	if h, ok := hl.handler[prefix].(http.Handler); ok {
		return h, nil
	}
	return nil, fmt.Errorf("bogus http or storage handler referenced as %q", prefix)
}

func (hl *handlerLoader) GetHandlerType(prefix string) string {
	hl.setupHandler(prefix)
	return hl.configType(prefix)
}

func exitFailure(pattern string, args ...interface{}) {
	if !strings.HasSuffix(pattern, "\n") {
		pattern = pattern + "\n"
	}
	panic(fmt.Sprintf(pattern, args...))
}

func (hl *handlerLoader) setupHandler(prefix string) {
	h, ok := hl.config[prefix]
	if !ok {
		exitFailure("invalid reference to undefined handler %q", prefix)
	}
	if h.setupDone {
		// Already setup by something else reference it and forcing it to be
		// setup before the bottom loop got to it.
		return
	}
	if h.settingUp {
		exitFailure("loop in configuration graph; %q tried to load itself indirectly", prefix)
	}
	h.settingUp = true
	defer func() {
		h.setupDone = true
		r := recover()
		if r == nil {
			if hl.handler[prefix] == nil {
				panic(fmt.Sprintf("setupHandler for %q didn't install a handler", prefix))
			}
		} else {
			panic(r)
		}
	}()

	if strings.HasPrefix(h.htype, "storage-") {
		stype := h.htype[len("storage-"):]
		// Assume a storage interface
		pstorage, err := blobserver.CreateStorage(stype, hl, h.conf)
		if err != nil {
			exitFailure("error instantiating storage for prefix %q, type %q: %v",
				h.prefix, stype, err)
		}
		hl.handler[h.prefix] = pstorage
		hl.installer.Handle(prefix+"camli/", makeCamliHandler(prefix, hl.baseURL, pstorage))
		return
	}

	hh, err := blobserver.CreateHandler(h.htype, hl, h.conf)
	if err != nil {
		exitFailure("error instantiating handler for prefix %q, type %q: %v",
			h.prefix, h.htype, err)
	}
	hl.handler[prefix] = hh
	var wrappedHandler http.Handler = &httputil.PrefixHandler{prefix, hh}
	if handerTypeWantsAuth(h.htype) {
		wrappedHandler = auth.Handler{wrappedHandler}
	}
	hl.installer.Handle(prefix, wrappedHandler)
}

func handerTypeWantsAuth(handlerType string) bool {
	// TODO(bradfitz): ask the handler instead? This is a bit of a
	// weird spot for this policy maybe?
	switch handlerType {
	case "ui", "search", "jsonsign", "sync":
		return true
	}
	return false
}

type Config struct {
	jsonconfig.Obj
	UIPath     string // Not valid until after InstallHandlers
	configPath string // Filesystem path
}

// Load returns a low-level "handler config" from the provided filename.
// If the config file doesn't contain a top-level JSON key of "handlerConfig"
// with boolean value true, the configuration is assumed to be a high-level
// "user config" file, and transformed into a low-level config.
func Load(filename string) (*Config, error) {
	obj, err := jsonconfig.ReadFile(filename)
	if err != nil {
		return nil, err
	}
	conf := &Config{
		Obj:        obj,
		configPath: filename,
	}

	if lowLevel := obj.OptionalBool("handlerConfig", false); !lowLevel {
		conf, err = GenLowLevelConfig(conf)
		if err != nil {
			return nil, fmt.Errorf(
				"Failed to transform user config file %q into internal handler configuration: %v",
				filename, err)
		}
		if v, _ := strconv.ParseBool(os.Getenv("CAMLI_DEBUG_CONFIG")); v {
			jsconf, _ := json.MarshalIndent(conf.Obj, "", "  ")
			log.Printf("From high-level config, generated low-level config: %s", jsconf)
		}
	}

	return conf, nil
}

func (config *Config) checkValidAuth() error {
	authConfig := config.OptionalString("auth", "")
	_, err := auth.FromConfig(authConfig)
	return err
}

// context may be nil
func (config *Config) InstallHandlers(hi HandlerInstaller, baseURL string, context *http.Request) (outerr error) {
	defer func() {
		err := recover()
		if err == nil {
			return
		}
		outerr = fmt.Errorf("%v", err)
	}()

	if err := config.checkValidAuth(); err != nil {
		return fmt.Errorf("error while configuring auth: %v", err)
	}
	prefixes := config.RequiredObject("prefixes")
	if err := config.Validate(); err != nil {
		return fmt.Errorf("configuration error in root object's keys: %v", err)
	}

	hl := &handlerLoader{
		installer: hi,
		baseURL:   baseURL,
		config:    make(map[string]*handlerConfig),
		handler:   make(map[string]interface{}),
		context:   context,
	}

	for prefix, vei := range prefixes {
		if !strings.HasPrefix(prefix, "/") {
			exitFailure("prefix %q doesn't start with /", prefix)
		}
		if !strings.HasSuffix(prefix, "/") {
			exitFailure("prefix %q doesn't end with /", prefix)
		}
		pmap, ok := vei.(map[string]interface{})
		if !ok {
			exitFailure("prefix %q value is a %T, not an object", prefix, vei)
		}
		pconf := jsonconfig.Obj(pmap)
		enabled := pconf.OptionalBool("enabled", true)
		if !enabled {
			continue
		}
		handlerType := pconf.RequiredString("handler")
		handlerArgs := pconf.OptionalObject("handlerArgs")
		if err := pconf.Validate(); err != nil {
			exitFailure("configuration error in prefix %s: %v", prefix, err)
		}
		h := &handlerConfig{
			prefix: prefix,
			htype:  handlerType,
			conf:   handlerArgs,
		}
		hl.config[prefix] = h

		if handlerType == "ui" {
			config.UIPath = prefix
		}
	}
	hl.setupAll()
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cinder

import (
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"testing"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/resource"
	"k8s.io/kubernetes/pkg/types"
	"k8s.io/kubernetes/pkg/util/mount"
	"k8s.io/kubernetes/pkg/volume"
)

func TestCanSupport(t *testing.T) {
	tmpDir, err := ioutil.TempDir(os.TempDir(), "cinderTest")
	if err != nil {
		t.Fatalf("can't make a temp dir: %v", err)
	}
	defer os.RemoveAll(tmpDir)
	plugMgr := volume.VolumePluginMgr{}
	plugMgr.InitPlugins(ProbeVolumePlugins(), volume.NewFakeVolumeHost(tmpDir, nil, nil))

	plug, err := plugMgr.FindPluginByName("kubernetes.io/cinder")
	if err != nil {
		t.Errorf("Can't find the plugin by name")
	}
	if plug.Name() != "kubernetes.io/cinder" {
		t.Errorf("Wrong name: %s", plug.Name())
	}
	if !plug.CanSupport(&volume.Spec{Volume: &api.Volume{VolumeSource: api.VolumeSource{Cinder: &api.CinderVolumeSource{}}}}) {
		t.Errorf("Expected true")
	}

	if !plug.CanSupport(&volume.Spec{PersistentVolume: &api.PersistentVolume{Spec: api.PersistentVolumeSpec{PersistentVolumeSource: api.PersistentVolumeSource{Cinder: &api.CinderVolumeSource{}}}}}) {
		t.Errorf("Expected true")
	}
}

type fakePDManager struct{}

func (fake *fakePDManager) AttachDisk(b *cinderVolumeBuilder, globalPDPath string) error {
	globalPath := makeGlobalPDName(b.plugin.host, b.pdName)
	err := os.MkdirAll(globalPath, 0750)
	if err != nil {
		return err
	}
	return nil
}

func (fake *fakePDManager) DetachDisk(c *cinderVolumeCleaner) error {
	globalPath := makeGlobalPDName(c.plugin.host, c.pdName)
	err := os.RemoveAll(globalPath)
	if err != nil {
		return err
	}
	return nil
}

func (fake *fakePDManager) CreateVolume(c *cinderVolumeProvisioner) (volumeID string, volumeSizeGB int, err error) {
	return "test-volume-name", 1, nil
}

func (fake *fakePDManager) DeleteVolume(cd *cinderVolumeDeleter) error {
	if cd.pdName != "test-volume-name" {
		return fmt.Errorf("Deleter got unexpected volume name: %s", cd.pdName)
	}
	return nil
}

func TestPlugin(t *testing.T) {
	tmpDir, err := ioutil.TempDir(os.TempDir(), "cinderTest")
	if err != nil {
		t.Fatalf("can't make a temp dir: %v", err)
	}
	defer os.RemoveAll(tmpDir)
	plugMgr := volume.VolumePluginMgr{}
	plugMgr.InitPlugins(ProbeVolumePlugins(), volume.NewFakeVolumeHost(tmpDir, nil, nil))

	plug, err := plugMgr.FindPluginByName("kubernetes.io/cinder")
	if err != nil {
		t.Errorf("Can't find the plugin by name")
	}
	spec := &api.Volume{
		Name: "vol1",
		VolumeSource: api.VolumeSource{
			Cinder: &api.CinderVolumeSource{
				VolumeID: "pd",
				FSType:   "ext4",
			},
		},
	}
	builder, err := plug.(*cinderPlugin).newBuilderInternal(volume.NewSpecFromVolume(spec), types.UID("poduid"), &fakePDManager{}, &mount.FakeMounter{})
	if err != nil {
		t.Errorf("Failed to make a new Builder: %v", err)
	}
	if builder == nil {
		t.Errorf("Got a nil Builder: %v")
	}
	volPath := path.Join(tmpDir, "pods/poduid/volumes/kubernetes.io~cinder/vol1")
	path := builder.GetPath()
	if path != volPath {
		t.Errorf("Got unexpected path: %s", path)
	}

	if err := builder.SetUp(nil); err != nil {
		t.Errorf("Expected success, got: %v", err)
	}
	if _, err := os.Stat(path); err != nil {
		if os.IsNotExist(err) {
			t.Errorf("SetUp() failed, volume path not created: %s", path)
		} else {
			t.Errorf("SetUp() failed: %v", err)
		}
	}
	if _, err := os.Stat(path); err != nil {
		if os.IsNotExist(err) {
			t.Errorf("SetUp() failed, volume path not created: %s", path)
		} else {
			t.Errorf("SetUp() failed: %v", err)
		}
	}

	cleaner, err := plug.(*cinderPlugin).newCleanerInternal("vol1", types.UID("poduid"), &fakePDManager{}, &mount.FakeMounter{})
	if err != nil {
		t.Errorf("Failed to make a new Cleaner: %v", err)
	}
	if cleaner == nil {
		t.Errorf("Got a nil Cleaner: %v")
	}

	if err := cleaner.TearDown(); err != nil {
		t.Errorf("Expected success, got: %v", err)
	}
	if _, err := os.Stat(path); err == nil {
		t.Errorf("TearDown() failed, volume path still exists: %s", path)
	} else if !os.IsNotExist(err) {
		t.Errorf("SetUp() failed: %v", err)
	}

	// Test Provisioner
	cap := resource.MustParse("100Mi")
	options := volume.VolumeOptions{
		Capacity: cap,
		AccessModes: []api.PersistentVolumeAccessMode{
			api.ReadWriteOnce,
		},
		PersistentVolumeReclaimPolicy: api.PersistentVolumeReclaimDelete,
	}
	provisioner, err := plug.(*cinderPlugin).newProvisionerInternal(options, &fakePDManager{})
	persistentSpec, err := provisioner.NewPersistentVolumeTemplate()
	if err != nil {
		t.Errorf("NewPersistentVolumeTemplate() failed: %v", err)
	}

	// get 2nd Provisioner - persistent volume controller will do the same
	provisioner, err = plug.(*cinderPlugin).newProvisionerInternal(options, &fakePDManager{})
	err = provisioner.Provision(persistentSpec)
	if err != nil {
		t.Errorf("Provision() failed: %v", err)
	}

	if persistentSpec.Spec.PersistentVolumeSource.Cinder.VolumeID != "test-volume-name" {
		t.Errorf("Provision() returned unexpected volume ID: %s", persistentSpec.Spec.PersistentVolumeSource.Cinder.VolumeID)
	}
	cap = persistentSpec.Spec.Capacity[api.ResourceStorage]
	size := cap.Value()
	if size != 1024*1024*1024 {
		t.Errorf("Provision() returned unexpected volume size: %v", size)
	}

	// Test Deleter
	volSpec := &volume.Spec{
		PersistentVolume: persistentSpec,
	}
	deleter, err := plug.(*cinderPlugin).newDeleterInternal(volSpec, &fakePDManager{})
	err = deleter.Delete()
	if err != nil {
		t.Errorf("Deleter() failed: %v", err)
	}
}
<commit_msg>Remove all instances of "/tmp" from unit tests and replace with a common tmp directory creator. Exception is documented.<commit_after>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cinder

import (
	"fmt"
	"os"
	"path"
	"testing"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/resource"
	"k8s.io/kubernetes/pkg/types"
	"k8s.io/kubernetes/pkg/util/mount"
	utiltesting "k8s.io/kubernetes/pkg/util/testing"
	"k8s.io/kubernetes/pkg/volume"
)

func TestCanSupport(t *testing.T) {
	tmpDir, err := utiltesting.MkTmpdir("cinderTest")
	if err != nil {
		t.Fatalf("can't make a temp dir: %v", err)
	}
	defer os.RemoveAll(tmpDir)
	plugMgr := volume.VolumePluginMgr{}
	plugMgr.InitPlugins(ProbeVolumePlugins(), volume.NewFakeVolumeHost(tmpDir, nil, nil))

	plug, err := plugMgr.FindPluginByName("kubernetes.io/cinder")
	if err != nil {
		t.Errorf("Can't find the plugin by name")
	}
	if plug.Name() != "kubernetes.io/cinder" {
		t.Errorf("Wrong name: %s", plug.Name())
	}
	if !plug.CanSupport(&volume.Spec{Volume: &api.Volume{VolumeSource: api.VolumeSource{Cinder: &api.CinderVolumeSource{}}}}) {
		t.Errorf("Expected true")
	}

	if !plug.CanSupport(&volume.Spec{PersistentVolume: &api.PersistentVolume{Spec: api.PersistentVolumeSpec{PersistentVolumeSource: api.PersistentVolumeSource{Cinder: &api.CinderVolumeSource{}}}}}) {
		t.Errorf("Expected true")
	}
}

type fakePDManager struct{}

func (fake *fakePDManager) AttachDisk(b *cinderVolumeBuilder, globalPDPath string) error {
	globalPath := makeGlobalPDName(b.plugin.host, b.pdName)
	err := os.MkdirAll(globalPath, 0750)
	if err != nil {
		return err
	}
	return nil
}

func (fake *fakePDManager) DetachDisk(c *cinderVolumeCleaner) error {
	globalPath := makeGlobalPDName(c.plugin.host, c.pdName)
	err := os.RemoveAll(globalPath)
	if err != nil {
		return err
	}
	return nil
}

func (fake *fakePDManager) CreateVolume(c *cinderVolumeProvisioner) (volumeID string, volumeSizeGB int, err error) {
	return "test-volume-name", 1, nil
}

func (fake *fakePDManager) DeleteVolume(cd *cinderVolumeDeleter) error {
	if cd.pdName != "test-volume-name" {
		return fmt.Errorf("Deleter got unexpected volume name: %s", cd.pdName)
	}
	return nil
}

func TestPlugin(t *testing.T) {
	tmpDir, err := utiltesting.MkTmpdir("cinderTest")
	if err != nil {
		t.Fatalf("can't make a temp dir: %v", err)
	}
	defer os.RemoveAll(tmpDir)
	plugMgr := volume.VolumePluginMgr{}
	plugMgr.InitPlugins(ProbeVolumePlugins(), volume.NewFakeVolumeHost(tmpDir, nil, nil))

	plug, err := plugMgr.FindPluginByName("kubernetes.io/cinder")
	if err != nil {
		t.Errorf("Can't find the plugin by name")
	}
	spec := &api.Volume{
		Name: "vol1",
		VolumeSource: api.VolumeSource{
			Cinder: &api.CinderVolumeSource{
				VolumeID: "pd",
				FSType:   "ext4",
			},
		},
	}
	builder, err := plug.(*cinderPlugin).newBuilderInternal(volume.NewSpecFromVolume(spec), types.UID("poduid"), &fakePDManager{}, &mount.FakeMounter{})
	if err != nil {
		t.Errorf("Failed to make a new Builder: %v", err)
	}
	if builder == nil {
		t.Errorf("Got a nil Builder: %v")
	}
	volPath := path.Join(tmpDir, "pods/poduid/volumes/kubernetes.io~cinder/vol1")
	path := builder.GetPath()
	if path != volPath {
		t.Errorf("Got unexpected path: %s", path)
	}

	if err := builder.SetUp(nil); err != nil {
		t.Errorf("Expected success, got: %v", err)
	}
	if _, err := os.Stat(path); err != nil {
		if os.IsNotExist(err) {
			t.Errorf("SetUp() failed, volume path not created: %s", path)
		} else {
			t.Errorf("SetUp() failed: %v", err)
		}
	}
	if _, err := os.Stat(path); err != nil {
		if os.IsNotExist(err) {
			t.Errorf("SetUp() failed, volume path not created: %s", path)
		} else {
			t.Errorf("SetUp() failed: %v", err)
		}
	}

	cleaner, err := plug.(*cinderPlugin).newCleanerInternal("vol1", types.UID("poduid"), &fakePDManager{}, &mount.FakeMounter{})
	if err != nil {
		t.Errorf("Failed to make a new Cleaner: %v", err)
	}
	if cleaner == nil {
		t.Errorf("Got a nil Cleaner: %v")
	}

	if err := cleaner.TearDown(); err != nil {
		t.Errorf("Expected success, got: %v", err)
	}
	if _, err := os.Stat(path); err == nil {
		t.Errorf("TearDown() failed, volume path still exists: %s", path)
	} else if !os.IsNotExist(err) {
		t.Errorf("SetUp() failed: %v", err)
	}

	// Test Provisioner
	cap := resource.MustParse("100Mi")
	options := volume.VolumeOptions{
		Capacity: cap,
		AccessModes: []api.PersistentVolumeAccessMode{
			api.ReadWriteOnce,
		},
		PersistentVolumeReclaimPolicy: api.PersistentVolumeReclaimDelete,
	}
	provisioner, err := plug.(*cinderPlugin).newProvisionerInternal(options, &fakePDManager{})
	persistentSpec, err := provisioner.NewPersistentVolumeTemplate()
	if err != nil {
		t.Errorf("NewPersistentVolumeTemplate() failed: %v", err)
	}

	// get 2nd Provisioner - persistent volume controller will do the same
	provisioner, err = plug.(*cinderPlugin).newProvisionerInternal(options, &fakePDManager{})
	err = provisioner.Provision(persistentSpec)
	if err != nil {
		t.Errorf("Provision() failed: %v", err)
	}

	if persistentSpec.Spec.PersistentVolumeSource.Cinder.VolumeID != "test-volume-name" {
		t.Errorf("Provision() returned unexpected volume ID: %s", persistentSpec.Spec.PersistentVolumeSource.Cinder.VolumeID)
	}
	cap = persistentSpec.Spec.Capacity[api.ResourceStorage]
	size := cap.Value()
	if size != 1024*1024*1024 {
		t.Errorf("Provision() returned unexpected volume size: %v", size)
	}

	// Test Deleter
	volSpec := &volume.Spec{
		PersistentVolume: persistentSpec,
	}
	deleter, err := plug.(*cinderPlugin).newDeleterInternal(volSpec, &fakePDManager{})
	err = deleter.Delete()
	if err != nil {
		t.Errorf("Deleter() failed: %v", err)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package flexvolume

import (
	"strconv"

	"k8s.io/kubernetes/pkg/util/mount"
	"k8s.io/kubernetes/pkg/volume"
	"k8s.io/utils/exec"
)

// FlexVolumeMounter is the disk that will be exposed by this plugin.
type flexVolumeMounter struct {
	*flexVolume
	// Runner used to setup the volume.
	runner exec.Interface
	// blockDeviceMounter provides the interface to create filesystem if the
	// filesystem doesn't exist.
	blockDeviceMounter mount.Interface
	// the considered volume spec
	spec     *volume.Spec
	readOnly bool
	volume.MetricsNil
}

var _ volume.Mounter = &flexVolumeMounter{}

// Mounter interface

// SetUp creates new directory.
func (f *flexVolumeMounter) SetUp(fsGroup *int64) error {
	return f.SetUpAt(f.GetPath(), fsGroup)
}

// SetUpAt creates new directory.
func (f *flexVolumeMounter) SetUpAt(dir string, fsGroup *int64) error {
	// Mount only once.
	alreadyMounted, err := prepareForMount(f.mounter, dir)
	if err != nil {
		return err
	}
	if alreadyMounted {
		return nil
	}

	call := f.plugin.NewDriverCall(mountCmd)

	// Interface parameters
	call.Append(dir)

	extraOptions := make(map[string]string)

	// pod metadata
	extraOptions[optionKeyPodName] = f.podName
	extraOptions[optionKeyPodNamespace] = f.podNamespace
	extraOptions[optionKeyPodUID] = string(f.podUID)
	// service account metadata
	extraOptions[optionKeyServiceAccountName] = f.podServiceAccountName

	// Extract secret and pass it as options.
	if err := addSecretsToOptions(extraOptions, f.spec, f.podNamespace, f.driverName, f.plugin.host); err != nil {
		return err
	}

	// Implicit parameters
	if fsGroup != nil {
		extraOptions[optionFSGroup] = strconv.FormatInt(int64(*fsGroup), 10)
	}

	call.AppendSpec(f.spec, f.plugin.host, extraOptions)

	_, err = call.Run()
	if isCmdNotSupportedErr(err) {
		err = (*mounterDefaults)(f).SetUpAt(dir, fsGroup)
	}

	if err != nil {
		return err
	}

	if !f.readOnly {
		volume.SetVolumeOwnership(f, fsGroup)
	}

	return nil
}

// GetAttributes get the flex volume attributes. The attributes will be queried
// using plugin callout after we finalize the callout syntax.
func (f *flexVolumeMounter) GetAttributes() volume.Attributes {
	return (*mounterDefaults)(f).GetAttributes()
}

func (f *flexVolumeMounter) CanMount() error {
	return nil
}
<commit_msg>flexvol: remove a mount directory in a error case<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package flexvolume

import (
	"os"
	"strconv"

	"k8s.io/kubernetes/pkg/util/mount"
	"k8s.io/kubernetes/pkg/volume"
	"k8s.io/utils/exec"
)

// FlexVolumeMounter is the disk that will be exposed by this plugin.
type flexVolumeMounter struct {
	*flexVolume
	// Runner used to setup the volume.
	runner exec.Interface
	// blockDeviceMounter provides the interface to create filesystem if the
	// filesystem doesn't exist.
	blockDeviceMounter mount.Interface
	// the considered volume spec
	spec     *volume.Spec
	readOnly bool
	volume.MetricsNil
}

var _ volume.Mounter = &flexVolumeMounter{}

// Mounter interface

// SetUp creates new directory.
func (f *flexVolumeMounter) SetUp(fsGroup *int64) error {
	return f.SetUpAt(f.GetPath(), fsGroup)
}

// SetUpAt creates new directory.
func (f *flexVolumeMounter) SetUpAt(dir string, fsGroup *int64) error {
	// Mount only once.
	alreadyMounted, err := prepareForMount(f.mounter, dir)
	if err != nil {
		return err
	}
	if alreadyMounted {
		return nil
	}

	call := f.plugin.NewDriverCall(mountCmd)

	// Interface parameters
	call.Append(dir)

	extraOptions := make(map[string]string)

	// pod metadata
	extraOptions[optionKeyPodName] = f.podName
	extraOptions[optionKeyPodNamespace] = f.podNamespace
	extraOptions[optionKeyPodUID] = string(f.podUID)
	// service account metadata
	extraOptions[optionKeyServiceAccountName] = f.podServiceAccountName

	// Extract secret and pass it as options.
	if err := addSecretsToOptions(extraOptions, f.spec, f.podNamespace, f.driverName, f.plugin.host); err != nil {
		os.Remove(dir)
		return err
	}

	// Implicit parameters
	if fsGroup != nil {
		extraOptions[optionFSGroup] = strconv.FormatInt(int64(*fsGroup), 10)
	}

	call.AppendSpec(f.spec, f.plugin.host, extraOptions)

	_, err = call.Run()
	if isCmdNotSupportedErr(err) {
		err = (*mounterDefaults)(f).SetUpAt(dir, fsGroup)
	}

	if err != nil {
		os.Remove(dir)
		return err
	}

	if !f.readOnly {
		volume.SetVolumeOwnership(f, fsGroup)
	}

	return nil
}

// GetAttributes get the flex volume attributes. The attributes will be queried
// using plugin callout after we finalize the callout syntax.
func (f *flexVolumeMounter) GetAttributes() volume.Attributes {
	return (*mounterDefaults)(f).GetAttributes()
}

func (f *flexVolumeMounter) CanMount() error {
	return nil
}
<|endoftext|>


<commit_before>package checkers

import (
	"errors"
	"github.com/jkomoros/boardgame"
	"github.com/jkomoros/boardgame/enum"
	"github.com/jkomoros/boardgame/enum/graph"
)

//+autoreader
const (
	PhaseSetup = iota
	PhasePlaying
)

//+autoreader
const (
	ColorBlack = iota
	ColorRed
)

//+autoreader reader
type token struct {
	boardgame.BaseComponentValues
	Color enum.Val
}

//+autoreader
type tokenDynamic struct {
	boardgame.BaseSubState
	Crowned bool
}

const numTokens = 12

//note: the struct tag for Spaces in gameState implicitly depends on this
//value.
const boardWidth = 8

var SpacesEnum = Enums.MustAddRange("Spaces", boardWidth, boardWidth)

var GraphDownward = graph.MustNewGridConnectedness(SpacesEnum, graph.DirectionDown, graph.DirectionDiagonal)
var GraphUpward = graph.MustNewGridConnectedness(SpacesEnum, graph.DirectionUp, graph.DirectionDiagonal)

const tokenDeckName = "Tokens"

//The first space in the upper left is black, and it alternates from there.
//The red tokens start at the top, and the black tokens are arrayed from the
//bottom.
func spaceIsBlack(spaceIndex int) bool {
	return spaceIndex%2 == 0
}

func newTokenDeck() *boardgame.Deck {

	deck := boardgame.NewDeck()

	deck.AddComponentMulti(&token{
		Color: ColorEnum.MustNewVal(ColorBlack),
	}, numTokens)

	deck.AddComponentMulti(&token{
		Color: ColorEnum.MustNewVal(ColorRed),
	}, numTokens)

	return deck
}

func (t *token) Legal(state boardgame.State, legalType int, componentIndex int) error {
	//Red starts at top, moves towards bottom
	targetRow := boardWidth - 1

	if t.Color.Value() == ColorBlack {
		//Black starts at top, moves towards bottom
		targetRow = 0
	}

	indexes := SpacesEnum.ValueToRange(componentIndex)

	if indexes[0] != targetRow {
		//Not in the target row
		return errors.New("Not in the target row")
	}

	d := t.ContainingComponent().DynamicValues(state).(*tokenDynamic)

	if d.Crowned {
		//Already crowned
		return errors.New("Already crowned")
	}

	return nil
}

//FreeNextSpaces is like AllNextSpaces, but spaces taht are occupied won't be returned.
func (t *token) FreeNextSpaces(state boardgame.State, componentIndex int) []int {

	spaces := state.GameState().(*gameState).Spaces

	var result []int
	for _, space := range t.FreeNextSpaces(state, componentIndex) {
		if spaces.ComponentAt(space) == nil {
			result = append(result, space)
		}
	}

	return result
}

//AllNextSpaces returns all the spaces that t could move to, if the rest of
//the board were empty.
func (t *token) AllNextSpaces(state boardgame.State, componentIndex int) []int {

	//Red starts from top
	fromBottom := false

	if t.Color.Value() == ColorBlack {
		fromBottom = true
	}

	var nextSpaces []int

	dyn := t.ContainingComponent().DynamicValues(state).(*tokenDynamic)

	crowned := dyn.Crowned

	g := GraphUpward
	oppositeG := GraphDownward

	if fromBottom {
		g = GraphDownward
		oppositeG = GraphUpward
	}

	for _, val := range g.Neighbors(componentIndex) {
		nextSpaces = append(nextSpaces, val)
	}

	if crowned {
		for _, val := range oppositeG.Neighbors(componentIndex) {
			nextSpaces = append(nextSpaces, val)
		}
	}

	return nextSpaces
}

//LegalCaptureSpaces returns cells that are legal for this cell to capture from there.
func (t *token) LegalCaptureSpaces(state boardgame.State, componentIndex int) []int {

	spaces := state.GameState().(*gameState).Spaces

	nextSpaces := t.AllNextSpaces(state, componentIndex)

	var result []int

	for _, space := range nextSpaces {
		c := spaces.ComponentAt(space)
		if c == nil {
			continue
		}
		if c.Values == nil {
			continue
		}
		v := c.Values.(*token)
		if v.Color.Equals(t.Color) {
			//One of our own.
			continue
		}
		//The item at space is a legal capture. What's the spot one beyond it,
		//and is it taken?

		startIndexes := SpacesEnum.ValueToRange(componentIndex)
		endIndexes := SpacesEnum.ValueToRange(space)

		diff := []int{
			endIndexes[0] - startIndexes[0],
			endIndexes[1] - startIndexes[1],
		}

		finalIndexes := []int{
			endIndexes[0] + diff[0],
			endIndexes[1] + diff[1],
		}

		finalSpace := SpacesEnum.RangeToValue(finalIndexes...)

		if finalSpace == enum.IllegalValue {
			//A space beyond the bounds
			continue
		}

		if spaces.ComponentAt(finalSpace) == nil {
			//An empty, real space!
			result = append(result, finalSpace)
		}

	}

	return result
}
<commit_msg>Added t.Dynamic() convenience getter. Part of #486.<commit_after>package checkers

import (
	"errors"
	"github.com/jkomoros/boardgame"
	"github.com/jkomoros/boardgame/enum"
	"github.com/jkomoros/boardgame/enum/graph"
)

//+autoreader
const (
	PhaseSetup = iota
	PhasePlaying
)

//+autoreader
const (
	ColorBlack = iota
	ColorRed
)

//+autoreader reader
type token struct {
	boardgame.BaseComponentValues
	Color enum.Val
}

//+autoreader
type tokenDynamic struct {
	boardgame.BaseSubState
	Crowned bool
}

const numTokens = 12

//note: the struct tag for Spaces in gameState implicitly depends on this
//value.
const boardWidth = 8

var SpacesEnum = Enums.MustAddRange("Spaces", boardWidth, boardWidth)

var GraphDownward = graph.MustNewGridConnectedness(SpacesEnum, graph.DirectionDown, graph.DirectionDiagonal)
var GraphUpward = graph.MustNewGridConnectedness(SpacesEnum, graph.DirectionUp, graph.DirectionDiagonal)

const tokenDeckName = "Tokens"

//The first space in the upper left is black, and it alternates from there.
//The red tokens start at the top, and the black tokens are arrayed from the
//bottom.
func spaceIsBlack(spaceIndex int) bool {
	return spaceIndex%2 == 0
}

func newTokenDeck() *boardgame.Deck {

	deck := boardgame.NewDeck()

	deck.AddComponentMulti(&token{
		Color: ColorEnum.MustNewVal(ColorBlack),
	}, numTokens)

	deck.AddComponentMulti(&token{
		Color: ColorEnum.MustNewVal(ColorRed),
	}, numTokens)

	return deck
}

func (t *token) Dynamic(state boardgame.State) *tokenDynamic {
	return t.ContainingComponent().DynamicValues(state).(*tokenDynamic)
}

func (t *token) Legal(state boardgame.State, legalType int, componentIndex int) error {
	//Red starts at top, moves towards bottom
	targetRow := boardWidth - 1

	if t.Color.Value() == ColorBlack {
		//Black starts at top, moves towards bottom
		targetRow = 0
	}

	indexes := SpacesEnum.ValueToRange(componentIndex)

	if indexes[0] != targetRow {
		//Not in the target row
		return errors.New("Not in the target row")
	}

	d := t.Dynamic(state)

	if d.Crowned {
		//Already crowned
		return errors.New("Already crowned")
	}

	return nil
}

//FreeNextSpaces is like AllNextSpaces, but spaces taht are occupied won't be returned.
func (t *token) FreeNextSpaces(state boardgame.State, componentIndex int) []int {

	spaces := state.GameState().(*gameState).Spaces

	var result []int
	for _, space := range t.FreeNextSpaces(state, componentIndex) {
		if spaces.ComponentAt(space) == nil {
			result = append(result, space)
		}
	}

	return result
}

//AllNextSpaces returns all the spaces that t could move to, if the rest of
//the board were empty.
func (t *token) AllNextSpaces(state boardgame.State, componentIndex int) []int {

	//Red starts from top
	fromBottom := false

	if t.Color.Value() == ColorBlack {
		fromBottom = true
	}

	var nextSpaces []int

	dyn := t.Dynamic(state)

	crowned := dyn.Crowned

	g := GraphUpward
	oppositeG := GraphDownward

	if fromBottom {
		g = GraphDownward
		oppositeG = GraphUpward
	}

	for _, val := range g.Neighbors(componentIndex) {
		nextSpaces = append(nextSpaces, val)
	}

	if crowned {
		for _, val := range oppositeG.Neighbors(componentIndex) {
			nextSpaces = append(nextSpaces, val)
		}
	}

	return nextSpaces
}

//LegalCaptureSpaces returns cells that are legal for this cell to capture from there.
func (t *token) LegalCaptureSpaces(state boardgame.State, componentIndex int) []int {

	spaces := state.GameState().(*gameState).Spaces

	nextSpaces := t.AllNextSpaces(state, componentIndex)

	var result []int

	for _, space := range nextSpaces {
		c := spaces.ComponentAt(space)
		if c == nil {
			continue
		}
		if c.Values == nil {
			continue
		}
		v := c.Values.(*token)
		if v.Color.Equals(t.Color) {
			//One of our own.
			continue
		}
		//The item at space is a legal capture. What's the spot one beyond it,
		//and is it taken?

		startIndexes := SpacesEnum.ValueToRange(componentIndex)
		endIndexes := SpacesEnum.ValueToRange(space)

		diff := []int{
			endIndexes[0] - startIndexes[0],
			endIndexes[1] - startIndexes[1],
		}

		finalIndexes := []int{
			endIndexes[0] + diff[0],
			endIndexes[1] + diff[1],
		}

		finalSpace := SpacesEnum.RangeToValue(finalIndexes...)

		if finalSpace == enum.IllegalValue {
			//A space beyond the bounds
			continue
		}

		if spaces.ComponentAt(finalSpace) == nil {
			//An empty, real space!
			result = append(result, finalSpace)
		}

	}

	return result
}
<|endoftext|>


<commit_before>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// OWNER = sig/network

package network

import (
	"fmt"
	"math"
	"net/http"
	"strings"
	"sync"
	"time"

	"k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/intstr"
	"k8s.io/apimachinery/pkg/util/net"
	clientset "k8s.io/client-go/kubernetes"
	"k8s.io/kubernetes/pkg/api/testapi"
	utilversion "k8s.io/kubernetes/pkg/util/version"
	"k8s.io/kubernetes/test/e2e/framework"
	testutils "k8s.io/kubernetes/test/utils"
	imageutils "k8s.io/kubernetes/test/utils/image"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

const (
	// Try all the proxy tests this many times (to catch even rare flakes).
	proxyAttempts = 20
	// Only print this many characters of the response (to keep the logs
	// legible).
	maxDisplayBodyLen = 100

	// We have seen one of these calls take just over 15 seconds, so putting this at 30.
	proxyHTTPCallTimeout = 30 * time.Second
)

var deprecatedCAdvisorPortRemovedVersion = utilversion.MustParseSemantic("v1.11.0-alpha.0")

var _ = SIGDescribe("Proxy", func() {
	version := testapi.Groups[v1.GroupName].GroupVersion().Version

	Context("version "+version, func() {
		options := framework.FrameworkOptions{
			ClientQPS: -1.0,
		}
		f := framework.NewFramework("proxy", options, nil)
		prefix := "/api/" + version

		skipCAdvisorProxyTests := false
		BeforeEach(func() {
			var err error
			skipCAdvisorProxyTests, err = framework.ServerVersionGTE(deprecatedCAdvisorPortRemovedVersion, f.ClientSet.Discovery())
			Expect(err).NotTo(HaveOccurred())
		})

		/*
			    Testname: proxy-subresource-node-logs-port
			    Description: Ensure that proxy on node logs works with node proxy
				subresource and explicit kubelet port.
		*/
		framework.ConformanceIt("should proxy logs on node with explicit kubelet port using proxy subresource ", func() { nodeProxyTest(f, prefix+"/nodes/", ":10250/proxy/logs/") })

		/*
			    Testname: proxy-subresource-node-logs
			    Description: Ensure that proxy on node logs works with node proxy
				subresource.
		*/
		framework.ConformanceIt("should proxy logs on node using proxy subresource ", func() { nodeProxyTest(f, prefix+"/nodes/", "/proxy/logs/") })
		if !skipCAdvisorProxyTests {
			It("should proxy to cadvisor using proxy subresource", func() { nodeProxyTest(f, prefix+"/nodes/", ":4194/proxy/containers/") })
		}

		// using the porter image to serve content, access the content
		// (of multiple pods?) from multiple (endpoints/services?)

		/*
			    Testname: proxy-service-pod
			    Description: Ensure that proxy through a service and a pod works with
				both generic top level prefix proxy and proxy subresource.
		*/
		framework.ConformanceIt("should proxy through a service and a pod ", func() {
			start := time.Now()
			labels := map[string]string{"proxy-service-target": "true"}
			service, err := f.ClientSet.CoreV1().Services(f.Namespace.Name).Create(&v1.Service{
				ObjectMeta: metav1.ObjectMeta{
					GenerateName: "proxy-service-",
				},
				Spec: v1.ServiceSpec{
					Selector: labels,
					Ports: []v1.ServicePort{
						{
							Name:       "portname1",
							Port:       80,
							TargetPort: intstr.FromString("dest1"),
						},
						{
							Name:       "portname2",
							Port:       81,
							TargetPort: intstr.FromInt(162),
						},
						{
							Name:       "tlsportname1",
							Port:       443,
							TargetPort: intstr.FromString("tlsdest1"),
						},
						{
							Name:       "tlsportname2",
							Port:       444,
							TargetPort: intstr.FromInt(462),
						},
					},
				},
			})
			Expect(err).NotTo(HaveOccurred())

			// Make an RC with a single pod. The 'porter' image is
			// a simple server which serves the values of the
			// environmental variables below.
			By("starting an echo server on multiple ports")
			pods := []*v1.Pod{}
			cfg := testutils.RCConfig{
				Client:         f.ClientSet,
				InternalClient: f.InternalClientset,
				Image:          imageutils.GetE2EImage(imageutils.Porter),
				Name:           service.Name,
				Namespace:      f.Namespace.Name,
				Replicas:       1,
				PollInterval:   time.Second,
				Env: map[string]string{
					"SERVE_PORT_80":   `<a href="/rewriteme">test</a>`,
					"SERVE_PORT_1080": `<a href="/rewriteme">test</a>`,
					"SERVE_PORT_160":  "foo",
					"SERVE_PORT_162":  "bar",

					"SERVE_TLS_PORT_443": `<a href="/tlsrewriteme">test</a>`,
					"SERVE_TLS_PORT_460": `tls baz`,
					"SERVE_TLS_PORT_462": `tls qux`,
				},
				Ports: map[string]int{
					"dest1": 160,
					"dest2": 162,

					"tlsdest1": 460,
					"tlsdest2": 462,
				},
				ReadinessProbe: &v1.Probe{
					Handler: v1.Handler{
						HTTPGet: &v1.HTTPGetAction{
							Port: intstr.FromInt(80),
						},
					},
					InitialDelaySeconds: 1,
					TimeoutSeconds:      5,
					PeriodSeconds:       10,
				},
				Labels:      labels,
				CreatedPods: &pods,
			}
			Expect(framework.RunRC(cfg)).NotTo(HaveOccurred())
			defer framework.DeleteRCAndPods(f.ClientSet, f.InternalClientset, f.Namespace.Name, cfg.Name)

			Expect(framework.WaitForEndpoint(f.ClientSet, f.Namespace.Name, service.Name)).NotTo(HaveOccurred())

			// table constructors
			// Try proxying through the service and directly to through the pod.
			subresourceServiceProxyURL := func(scheme, port string) string {
				return prefix + "/namespaces/" + f.Namespace.Name + "/services/" + net.JoinSchemeNamePort(scheme, service.Name, port) + "/proxy"
			}
			subresourcePodProxyURL := func(scheme, port string) string {
				return prefix + "/namespaces/" + f.Namespace.Name + "/pods/" + net.JoinSchemeNamePort(scheme, pods[0].Name, port) + "/proxy"
			}

			// construct the table
			expectations := map[string]string{
				subresourceServiceProxyURL("", "portname1") + "/":         "foo",
				subresourceServiceProxyURL("http", "portname1") + "/":     "foo",
				subresourceServiceProxyURL("", "portname2") + "/":         "bar",
				subresourceServiceProxyURL("http", "portname2") + "/":     "bar",
				subresourceServiceProxyURL("https", "tlsportname1") + "/": "tls baz",
				subresourceServiceProxyURL("https", "tlsportname2") + "/": "tls qux",

				subresourcePodProxyURL("", "") + "/":         `<a href="` + subresourcePodProxyURL("", "") + `/rewriteme">test</a>`,
				subresourcePodProxyURL("", "1080") + "/":     `<a href="` + subresourcePodProxyURL("", "1080") + `/rewriteme">test</a>`,
				subresourcePodProxyURL("http", "1080") + "/": `<a href="` + subresourcePodProxyURL("http", "1080") + `/rewriteme">test</a>`,
				subresourcePodProxyURL("", "160") + "/":      "foo",
				subresourcePodProxyURL("http", "160") + "/":  "foo",
				subresourcePodProxyURL("", "162") + "/":      "bar",
				subresourcePodProxyURL("http", "162") + "/":  "bar",

				subresourcePodProxyURL("https", "443") + "/": `<a href="` + subresourcePodProxyURL("https", "443") + `/tlsrewriteme">test</a>`,
				subresourcePodProxyURL("https", "460") + "/": "tls baz",
				subresourcePodProxyURL("https", "462") + "/": "tls qux",

				// TODO: below entries don't work, but I believe we should make them work.
				// podPrefix + ":dest1": "foo",
				// podPrefix + ":dest2": "bar",
			}

			wg := sync.WaitGroup{}
			errs := []string{}
			errLock := sync.Mutex{}
			recordError := func(s string) {
				errLock.Lock()
				defer errLock.Unlock()
				errs = append(errs, s)
			}
			d := time.Since(start)
			framework.Logf("setup took %v, starting test cases", d)
			numberTestCases := len(expectations)
			totalAttempts := numberTestCases * proxyAttempts
			By(fmt.Sprintf("running %v cases, %v attempts per case, %v total attempts", numberTestCases, proxyAttempts, totalAttempts))

			for i := 0; i < proxyAttempts; i++ {
				wg.Add(numberTestCases)
				for path, val := range expectations {
					go func(i int, path, val string) {
						defer wg.Done()
						// this runs the test case
						body, status, d, err := doProxy(f, path, i)

						if err != nil {
							if serr, ok := err.(*errors.StatusError); ok {
								recordError(fmt.Sprintf("%v (%v; %v): path %v gave status error: %+v",
									i, status, d, path, serr.Status()))
							} else {
								recordError(fmt.Sprintf("%v: path %v gave error: %v", i, path, err))
							}
							return
						}
						if status != http.StatusOK {
							recordError(fmt.Sprintf("%v: path %v gave status: %v", i, path, status))
						}
						if e, a := val, string(body); e != a {
							recordError(fmt.Sprintf("%v: path %v: wanted %v, got %v", i, path, e, a))
						}
						if d > proxyHTTPCallTimeout {
							recordError(fmt.Sprintf("%v: path %v took %v > %v", i, path, d, proxyHTTPCallTimeout))
						}
					}(i, path, val)
				}
				wg.Wait()
			}

			if len(errs) != 0 {
				body, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).GetLogs(pods[0].Name, &v1.PodLogOptions{}).Do().Raw()
				if err != nil {
					framework.Logf("Error getting logs for pod %s: %v", pods[0].Name, err)
				} else {
					framework.Logf("Pod %s has the following error logs: %s", pods[0].Name, body)
				}

				framework.Failf(strings.Join(errs, "\n"))
			}
		})
	})
})

func doProxy(f *framework.Framework, path string, i int) (body []byte, statusCode int, d time.Duration, err error) {
	// About all of the proxy accesses in this file:
	// * AbsPath is used because it preserves the trailing '/'.
	// * Do().Raw() is used (instead of DoRaw()) because it will turn an
	//   error from apiserver proxy into an actual error, and there is no
	//   chance of the things we are talking to being confused for an error
	//   that apiserver would have emitted.
	start := time.Now()
	body, err = f.ClientSet.CoreV1().RESTClient().Get().AbsPath(path).Do().StatusCode(&statusCode).Raw()
	d = time.Since(start)
	if len(body) > 0 {
		framework.Logf("(%v) %v: %s (%v; %v)", i, path, truncate(body, maxDisplayBodyLen), statusCode, d)
	} else {
		framework.Logf("%v: %s (%v; %v)", path, "no body", statusCode, d)
	}
	return
}

func truncate(b []byte, maxLen int) []byte {
	if len(b) <= maxLen-3 {
		return b
	}
	b2 := append([]byte(nil), b[:maxLen-3]...)
	b2 = append(b2, '.', '.', '.')
	return b2
}

func pickNode(cs clientset.Interface) (string, error) {
	// TODO: investigate why it doesn't work on master Node.
	nodes := framework.GetReadySchedulableNodesOrDie(cs)
	if len(nodes.Items) == 0 {
		return "", fmt.Errorf("no nodes exist, can't test node proxy")
	}
	return nodes.Items[0].Name, nil
}

func nodeProxyTest(f *framework.Framework, prefix, nodeDest string) {
	node, err := pickNode(f.ClientSet)
	Expect(err).NotTo(HaveOccurred())
	// TODO: Change it to test whether all requests succeeded when requests
	// not reaching Kubelet issue is debugged.
	serviceUnavailableErrors := 0
	for i := 0; i < proxyAttempts; i++ {
		_, status, d, err := doProxy(f, prefix+node+nodeDest, i)
		if status == http.StatusServiceUnavailable {
			framework.Logf("Failed proxying node logs due to service unavailable: %v", err)
			time.Sleep(time.Second)
			serviceUnavailableErrors++
		} else {
			Expect(err).NotTo(HaveOccurred())
			Expect(status).To(Equal(http.StatusOK))
			Expect(d).To(BeNumerically("<", proxyHTTPCallTimeout))
		}
	}
	if serviceUnavailableErrors > 0 {
		framework.Logf("error: %d requests to proxy node logs failed", serviceUnavailableErrors)
	}
	maxFailures := int(math.Floor(0.1 * float64(proxyAttempts)))
	Expect(serviceUnavailableErrors).To(BeNumerically("<", maxFailures))
}
<commit_msg>properly skip cadvisor proxy test<commit_after>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// OWNER = sig/network

package network

import (
	"fmt"
	"math"
	"net/http"
	"strings"
	"sync"
	"time"

	"k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/intstr"
	"k8s.io/apimachinery/pkg/util/net"
	clientset "k8s.io/client-go/kubernetes"
	"k8s.io/kubernetes/pkg/api/testapi"
	utilversion "k8s.io/kubernetes/pkg/util/version"
	"k8s.io/kubernetes/test/e2e/framework"
	testutils "k8s.io/kubernetes/test/utils"
	imageutils "k8s.io/kubernetes/test/utils/image"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

const (
	// Try all the proxy tests this many times (to catch even rare flakes).
	proxyAttempts = 20
	// Only print this many characters of the response (to keep the logs
	// legible).
	maxDisplayBodyLen = 100

	// We have seen one of these calls take just over 15 seconds, so putting this at 30.
	proxyHTTPCallTimeout = 30 * time.Second
)

var deprecatedCAdvisorPortRemovedVersion = utilversion.MustParseSemantic("v1.11.0-alpha.0")

var _ = SIGDescribe("Proxy", func() {
	version := testapi.Groups[v1.GroupName].GroupVersion().Version

	Context("version "+version, func() {
		options := framework.FrameworkOptions{
			ClientQPS: -1.0,
		}
		f := framework.NewFramework("proxy", options, nil)
		prefix := "/api/" + version

		skipCAdvisorProxyTests := false
		BeforeEach(func() {
			var err error
			skipCAdvisorProxyTests, err = framework.ServerVersionGTE(deprecatedCAdvisorPortRemovedVersion, f.ClientSet.Discovery())
			Expect(err).NotTo(HaveOccurred())
		})

		/*
			    Testname: proxy-subresource-node-logs-port
			    Description: Ensure that proxy on node logs works with node proxy
				subresource and explicit kubelet port.
		*/
		framework.ConformanceIt("should proxy logs on node with explicit kubelet port using proxy subresource ", func() { nodeProxyTest(f, prefix+"/nodes/", ":10250/proxy/logs/") })

		/*
			    Testname: proxy-subresource-node-logs
			    Description: Ensure that proxy on node logs works with node proxy
				subresource.
		*/
		framework.ConformanceIt("should proxy logs on node using proxy subresource ", func() { nodeProxyTest(f, prefix+"/nodes/", "/proxy/logs/") })

		It("should proxy to cadvisor using proxy subresource", func() {
			if skipCAdvisorProxyTests {
				framework.Skipf("cadvisor proxy test removed on newer server version")
			}
			nodeProxyTest(f, prefix+"/nodes/", ":4194/proxy/containers/")
		})

		// using the porter image to serve content, access the content
		// (of multiple pods?) from multiple (endpoints/services?)

		/*
			    Testname: proxy-service-pod
			    Description: Ensure that proxy through a service and a pod works with
				both generic top level prefix proxy and proxy subresource.
		*/
		framework.ConformanceIt("should proxy through a service and a pod ", func() {
			start := time.Now()
			labels := map[string]string{"proxy-service-target": "true"}
			service, err := f.ClientSet.CoreV1().Services(f.Namespace.Name).Create(&v1.Service{
				ObjectMeta: metav1.ObjectMeta{
					GenerateName: "proxy-service-",
				},
				Spec: v1.ServiceSpec{
					Selector: labels,
					Ports: []v1.ServicePort{
						{
							Name:       "portname1",
							Port:       80,
							TargetPort: intstr.FromString("dest1"),
						},
						{
							Name:       "portname2",
							Port:       81,
							TargetPort: intstr.FromInt(162),
						},
						{
							Name:       "tlsportname1",
							Port:       443,
							TargetPort: intstr.FromString("tlsdest1"),
						},
						{
							Name:       "tlsportname2",
							Port:       444,
							TargetPort: intstr.FromInt(462),
						},
					},
				},
			})
			Expect(err).NotTo(HaveOccurred())

			// Make an RC with a single pod. The 'porter' image is
			// a simple server which serves the values of the
			// environmental variables below.
			By("starting an echo server on multiple ports")
			pods := []*v1.Pod{}
			cfg := testutils.RCConfig{
				Client:         f.ClientSet,
				InternalClient: f.InternalClientset,
				Image:          imageutils.GetE2EImage(imageutils.Porter),
				Name:           service.Name,
				Namespace:      f.Namespace.Name,
				Replicas:       1,
				PollInterval:   time.Second,
				Env: map[string]string{
					"SERVE_PORT_80":   `<a href="/rewriteme">test</a>`,
					"SERVE_PORT_1080": `<a href="/rewriteme">test</a>`,
					"SERVE_PORT_160":  "foo",
					"SERVE_PORT_162":  "bar",

					"SERVE_TLS_PORT_443": `<a href="/tlsrewriteme">test</a>`,
					"SERVE_TLS_PORT_460": `tls baz`,
					"SERVE_TLS_PORT_462": `tls qux`,
				},
				Ports: map[string]int{
					"dest1": 160,
					"dest2": 162,

					"tlsdest1": 460,
					"tlsdest2": 462,
				},
				ReadinessProbe: &v1.Probe{
					Handler: v1.Handler{
						HTTPGet: &v1.HTTPGetAction{
							Port: intstr.FromInt(80),
						},
					},
					InitialDelaySeconds: 1,
					TimeoutSeconds:      5,
					PeriodSeconds:       10,
				},
				Labels:      labels,
				CreatedPods: &pods,
			}
			Expect(framework.RunRC(cfg)).NotTo(HaveOccurred())
			defer framework.DeleteRCAndPods(f.ClientSet, f.InternalClientset, f.Namespace.Name, cfg.Name)

			Expect(framework.WaitForEndpoint(f.ClientSet, f.Namespace.Name, service.Name)).NotTo(HaveOccurred())

			// table constructors
			// Try proxying through the service and directly to through the pod.
			subresourceServiceProxyURL := func(scheme, port string) string {
				return prefix + "/namespaces/" + f.Namespace.Name + "/services/" + net.JoinSchemeNamePort(scheme, service.Name, port) + "/proxy"
			}
			subresourcePodProxyURL := func(scheme, port string) string {
				return prefix + "/namespaces/" + f.Namespace.Name + "/pods/" + net.JoinSchemeNamePort(scheme, pods[0].Name, port) + "/proxy"
			}

			// construct the table
			expectations := map[string]string{
				subresourceServiceProxyURL("", "portname1") + "/":         "foo",
				subresourceServiceProxyURL("http", "portname1") + "/":     "foo",
				subresourceServiceProxyURL("", "portname2") + "/":         "bar",
				subresourceServiceProxyURL("http", "portname2") + "/":     "bar",
				subresourceServiceProxyURL("https", "tlsportname1") + "/": "tls baz",
				subresourceServiceProxyURL("https", "tlsportname2") + "/": "tls qux",

				subresourcePodProxyURL("", "") + "/":         `<a href="` + subresourcePodProxyURL("", "") + `/rewriteme">test</a>`,
				subresourcePodProxyURL("", "1080") + "/":     `<a href="` + subresourcePodProxyURL("", "1080") + `/rewriteme">test</a>`,
				subresourcePodProxyURL("http", "1080") + "/": `<a href="` + subresourcePodProxyURL("http", "1080") + `/rewriteme">test</a>`,
				subresourcePodProxyURL("", "160") + "/":      "foo",
				subresourcePodProxyURL("http", "160") + "/":  "foo",
				subresourcePodProxyURL("", "162") + "/":      "bar",
				subresourcePodProxyURL("http", "162") + "/":  "bar",

				subresourcePodProxyURL("https", "443") + "/": `<a href="` + subresourcePodProxyURL("https", "443") + `/tlsrewriteme">test</a>`,
				subresourcePodProxyURL("https", "460") + "/": "tls baz",
				subresourcePodProxyURL("https", "462") + "/": "tls qux",

				// TODO: below entries don't work, but I believe we should make them work.
				// podPrefix + ":dest1": "foo",
				// podPrefix + ":dest2": "bar",
			}

			wg := sync.WaitGroup{}
			errs := []string{}
			errLock := sync.Mutex{}
			recordError := func(s string) {
				errLock.Lock()
				defer errLock.Unlock()
				errs = append(errs, s)
			}
			d := time.Since(start)
			framework.Logf("setup took %v, starting test cases", d)
			numberTestCases := len(expectations)
			totalAttempts := numberTestCases * proxyAttempts
			By(fmt.Sprintf("running %v cases, %v attempts per case, %v total attempts", numberTestCases, proxyAttempts, totalAttempts))

			for i := 0; i < proxyAttempts; i++ {
				wg.Add(numberTestCases)
				for path, val := range expectations {
					go func(i int, path, val string) {
						defer wg.Done()
						// this runs the test case
						body, status, d, err := doProxy(f, path, i)

						if err != nil {
							if serr, ok := err.(*errors.StatusError); ok {
								recordError(fmt.Sprintf("%v (%v; %v): path %v gave status error: %+v",
									i, status, d, path, serr.Status()))
							} else {
								recordError(fmt.Sprintf("%v: path %v gave error: %v", i, path, err))
							}
							return
						}
						if status != http.StatusOK {
							recordError(fmt.Sprintf("%v: path %v gave status: %v", i, path, status))
						}
						if e, a := val, string(body); e != a {
							recordError(fmt.Sprintf("%v: path %v: wanted %v, got %v", i, path, e, a))
						}
						if d > proxyHTTPCallTimeout {
							recordError(fmt.Sprintf("%v: path %v took %v > %v", i, path, d, proxyHTTPCallTimeout))
						}
					}(i, path, val)
				}
				wg.Wait()
			}

			if len(errs) != 0 {
				body, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).GetLogs(pods[0].Name, &v1.PodLogOptions{}).Do().Raw()
				if err != nil {
					framework.Logf("Error getting logs for pod %s: %v", pods[0].Name, err)
				} else {
					framework.Logf("Pod %s has the following error logs: %s", pods[0].Name, body)
				}

				framework.Failf(strings.Join(errs, "\n"))
			}
		})
	})
})

func doProxy(f *framework.Framework, path string, i int) (body []byte, statusCode int, d time.Duration, err error) {
	// About all of the proxy accesses in this file:
	// * AbsPath is used because it preserves the trailing '/'.
	// * Do().Raw() is used (instead of DoRaw()) because it will turn an
	//   error from apiserver proxy into an actual error, and there is no
	//   chance of the things we are talking to being confused for an error
	//   that apiserver would have emitted.
	start := time.Now()
	body, err = f.ClientSet.CoreV1().RESTClient().Get().AbsPath(path).Do().StatusCode(&statusCode).Raw()
	d = time.Since(start)
	if len(body) > 0 {
		framework.Logf("(%v) %v: %s (%v; %v)", i, path, truncate(body, maxDisplayBodyLen), statusCode, d)
	} else {
		framework.Logf("%v: %s (%v; %v)", path, "no body", statusCode, d)
	}
	return
}

func truncate(b []byte, maxLen int) []byte {
	if len(b) <= maxLen-3 {
		return b
	}
	b2 := append([]byte(nil), b[:maxLen-3]...)
	b2 = append(b2, '.', '.', '.')
	return b2
}

func pickNode(cs clientset.Interface) (string, error) {
	// TODO: investigate why it doesn't work on master Node.
	nodes := framework.GetReadySchedulableNodesOrDie(cs)
	if len(nodes.Items) == 0 {
		return "", fmt.Errorf("no nodes exist, can't test node proxy")
	}
	return nodes.Items[0].Name, nil
}

func nodeProxyTest(f *framework.Framework, prefix, nodeDest string) {
	node, err := pickNode(f.ClientSet)
	Expect(err).NotTo(HaveOccurred())
	// TODO: Change it to test whether all requests succeeded when requests
	// not reaching Kubelet issue is debugged.
	serviceUnavailableErrors := 0
	for i := 0; i < proxyAttempts; i++ {
		_, status, d, err := doProxy(f, prefix+node+nodeDest, i)
		if status == http.StatusServiceUnavailable {
			framework.Logf("Failed proxying node logs due to service unavailable: %v", err)
			time.Sleep(time.Second)
			serviceUnavailableErrors++
		} else {
			Expect(err).NotTo(HaveOccurred())
			Expect(status).To(Equal(http.StatusOK))
			Expect(d).To(BeNumerically("<", proxyHTTPCallTimeout))
		}
	}
	if serviceUnavailableErrors > 0 {
		framework.Logf("error: %d requests to proxy node logs failed", serviceUnavailableErrors)
	}
	maxFailures := int(math.Floor(0.1 * float64(proxyAttempts)))
	Expect(serviceUnavailableErrors).To(BeNumerically("<", maxFailures))
}
<|endoftext|>


<commit_before>package models

import (
	"database/sql"
	"fmt"
)

// SearchOptions are options that can be passed to SearchSets for filtering
// sets.
type SearchOptions struct {
	// If len is 0, then it should be treated as if all statuses are good.
	Status []int
	Query  string
	// Gamemodes to which limit the results. If len is 0, it means all modes
	// are ok.
	Mode []int

	// Pagination options.
	Offset int
	Amount int
}

func (o SearchOptions) setModes() (total uint8) {
	for _, m := range o.Mode {
		if m < 0 || m >= 4 {
			continue
		}
		total |= 1 << uint8(m)
	}
	return
}

// SearchSets retrieves sets, filtering them using SearchOptions.
func SearchSets(db *sql.DB, opts SearchOptions) ([]Set, error) {
	setsQuery := "SELECT " + setFields +
		", MATCH(artist, title, creator, source, tags) AGAINST (? IN NATURAL LANGUAGE MODE) AS relevance FROM sets WHERE 1 "
	args := []interface{}{opts.Query}

	// add filters to query
	if len(opts.Status) != 0 {
		setsQuery += "AND ranked_status IN (" + inClause(len(opts.Status)) + ") "
		args = append(args, sIntToSInterface(opts.Status)...)
	}
	if len(opts.Mode) != 0 {
		setsQuery += "AND (set_modes & ?) = ? "
		sm := opts.setModes()
		args = append(args, sm, sm)
	}

	// set order by
	if opts.Query == "" {
		setsQuery += "ORDER BY id ASC "
	} else {
		setsQuery += "AND MATCH(artist, title, creator, source, tags) AGAINST (? IN NATURAL LANGUAGE MODE) ORDER BY relevance DESC "
		args = append(args, opts.Query)
	}

	// set limit
	setsQuery += fmt.Sprintf("LIMIT %d, %d", opts.Offset, opts.Amount)

	// fetch rows
	rows, err := db.Query(setsQuery, args...)
	if err != nil {
		return nil, err
	}

	sets := make([]Set, 0, opts.Amount)
	// setIDs is used to make the IN statement later on. setMap is used for
	// finding the beatmap to which append the child.
	setIDs := make([]int, 0, opts.Amount)
	setMap := make(map[int]*Set, opts.Amount)

	// find all beatmaps, but leave children aside for the moment.
	for rows.Next() {
		var s Set
		var rel float64
		err = rows.Scan(
			&s.ID, &s.RankedStatus, &s.ApprovedDate, &s.LastUpdate, &s.LastChecked,
			&s.Artist, &s.Title, &s.Creator, &s.Source, &s.Tags, &s.HasVideo, &s.Genre,
			&s.Language, &s.Favourites, &rel,
		)
		if err != nil {
			return nil, err
		}
		sets = append(sets, s)
		setIDs = append(setIDs, s.ID)
		setMap[s.ID] = &sets[len(sets)-1]
	}

	if len(sets) == 0 {
		return []Set{}, nil
	}

	rows, err = db.Query(
		"SELECT "+beatmapFields+" FROM beatmaps WHERE parent_set_id IN ("+
			inClause(len(setIDs))+")",
		sIntToSInterface(setIDs)...,
	)
	if err != nil {
		return nil, err
	}

	for rows.Next() {
		var b Beatmap
		err = rows.Scan(
			&b.ID, &b.ParentSetID, &b.DiffName, &b.FileMD5, &b.Mode, &b.BPM,
			&b.AR, &b.OD, &b.CS, &b.HP, &b.TotalLength, &b.HitLength,
			&b.Playcount, &b.Passcount, &b.MaxCombo, &b.DifficultyRating,
		)
		if err != nil {
			return nil, err
		}
		parentSet := setMap[b.ParentSetID]
		if parentSet == nil {
			continue
		}
		parentSet.ChildrenBeatmaps = append(parentSet.ChildrenBeatmaps, b)
	}

	return sets, nil
}
<commit_msg>make default sorting (in case of no query) id DESC<commit_after>package models

import (
	"database/sql"
	"fmt"
)

// SearchOptions are options that can be passed to SearchSets for filtering
// sets.
type SearchOptions struct {
	// If len is 0, then it should be treated as if all statuses are good.
	Status []int
	Query  string
	// Gamemodes to which limit the results. If len is 0, it means all modes
	// are ok.
	Mode []int

	// Pagination options.
	Offset int
	Amount int
}

func (o SearchOptions) setModes() (total uint8) {
	for _, m := range o.Mode {
		if m < 0 || m >= 4 {
			continue
		}
		total |= 1 << uint8(m)
	}
	return
}

// SearchSets retrieves sets, filtering them using SearchOptions.
func SearchSets(db *sql.DB, opts SearchOptions) ([]Set, error) {
	setsQuery := "SELECT " + setFields +
		", MATCH(artist, title, creator, source, tags) AGAINST (? IN NATURAL LANGUAGE MODE) AS relevance FROM sets WHERE 1 "
	args := []interface{}{opts.Query}

	// add filters to query
	if len(opts.Status) != 0 {
		setsQuery += "AND ranked_status IN (" + inClause(len(opts.Status)) + ") "
		args = append(args, sIntToSInterface(opts.Status)...)
	}
	if len(opts.Mode) != 0 {
		setsQuery += "AND (set_modes & ?) = ? "
		sm := opts.setModes()
		args = append(args, sm, sm)
	}

	// set order by
	if opts.Query == "" {
		setsQuery += "ORDER BY id DESC "
	} else {
		setsQuery += "AND MATCH(artist, title, creator, source, tags) AGAINST (? IN NATURAL LANGUAGE MODE) ORDER BY relevance DESC "
		args = append(args, opts.Query)
	}

	// set limit
	setsQuery += fmt.Sprintf("LIMIT %d, %d", opts.Offset, opts.Amount)

	// fetch rows
	rows, err := db.Query(setsQuery, args...)
	if err != nil {
		return nil, err
	}

	sets := make([]Set, 0, opts.Amount)
	// setIDs is used to make the IN statement later on. setMap is used for
	// finding the beatmap to which append the child.
	setIDs := make([]int, 0, opts.Amount)
	setMap := make(map[int]*Set, opts.Amount)

	// find all beatmaps, but leave children aside for the moment.
	for rows.Next() {
		var s Set
		var rel float64
		err = rows.Scan(
			&s.ID, &s.RankedStatus, &s.ApprovedDate, &s.LastUpdate, &s.LastChecked,
			&s.Artist, &s.Title, &s.Creator, &s.Source, &s.Tags, &s.HasVideo, &s.Genre,
			&s.Language, &s.Favourites, &rel,
		)
		if err != nil {
			return nil, err
		}
		sets = append(sets, s)
		setIDs = append(setIDs, s.ID)
		setMap[s.ID] = &sets[len(sets)-1]
	}

	if len(sets) == 0 {
		return []Set{}, nil
	}

	rows, err = db.Query(
		"SELECT "+beatmapFields+" FROM beatmaps WHERE parent_set_id IN ("+
			inClause(len(setIDs))+")",
		sIntToSInterface(setIDs)...,
	)
	if err != nil {
		return nil, err
	}

	for rows.Next() {
		var b Beatmap
		err = rows.Scan(
			&b.ID, &b.ParentSetID, &b.DiffName, &b.FileMD5, &b.Mode, &b.BPM,
			&b.AR, &b.OD, &b.CS, &b.HP, &b.TotalLength, &b.HitLength,
			&b.Playcount, &b.Passcount, &b.MaxCombo, &b.DifficultyRating,
		)
		if err != nil {
			return nil, err
		}
		parentSet := setMap[b.ParentSetID]
		if parentSet == nil {
			continue
		}
		parentSet.ChildrenBeatmaps = append(parentSet.ChildrenBeatmaps, b)
	}

	return sets, nil
}
<|endoftext|>