<commit_before>package storage

import (
	"fmt"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/BurntSushi/toml"
	"github.com/containers/storage/pkg/idtools"
	"github.com/containers/storage/pkg/system"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

// ParseIDMapping takes idmappings and subuid and subgid maps and returns a storage mapping
func ParseIDMapping(UIDMapSlice, GIDMapSlice []string, subUIDMap, subGIDMap string) (*IDMappingOptions, error) {
	options := IDMappingOptions{
		HostUIDMapping: true,
		HostGIDMapping: true,
	}
	if subGIDMap == "" && subUIDMap != "" {
		subGIDMap = subUIDMap
	}
	if subUIDMap == "" && subGIDMap != "" {
		subUIDMap = subGIDMap
	}
	if len(GIDMapSlice) == 0 && len(UIDMapSlice) != 0 {
		GIDMapSlice = UIDMapSlice
	}
	if len(UIDMapSlice) == 0 && len(GIDMapSlice) != 0 {
		UIDMapSlice = GIDMapSlice
	}
	if len(UIDMapSlice) == 0 && subUIDMap == "" && os.Getuid() != 0 {
		UIDMapSlice = []string{fmt.Sprintf("0:%d:1", os.Getuid())}
	}
	if len(GIDMapSlice) == 0 && subGIDMap == "" && os.Getuid() != 0 {
		GIDMapSlice = []string{fmt.Sprintf("0:%d:1", os.Getgid())}
	}

	if subUIDMap != "" && subGIDMap != "" {
		mappings, err := idtools.NewIDMappings(subUIDMap, subGIDMap)
		if err != nil {
			return nil, errors.Wrapf(err, "failed to create NewIDMappings for uidmap=%s gidmap=%s", subUIDMap, subGIDMap)
		}
		options.UIDMap = mappings.UIDs()
		options.GIDMap = mappings.GIDs()
	}
	parsedUIDMap, err := idtools.ParseIDMap(UIDMapSlice, "UID")
	if err != nil {
		return nil, errors.Wrapf(err, "failed to create ParseUIDMap UID=%s", UIDMapSlice)
	}
	parsedGIDMap, err := idtools.ParseIDMap(GIDMapSlice, "GID")
	if err != nil {
		return nil, errors.Wrapf(err, "failed to create ParseGIDMap GID=%s", UIDMapSlice)
	}
	options.UIDMap = append(options.UIDMap, parsedUIDMap...)
	options.GIDMap = append(options.GIDMap, parsedGIDMap...)
	if len(options.UIDMap) > 0 {
		options.HostUIDMapping = false
	}
	if len(options.GIDMap) > 0 {
		options.HostGIDMapping = false
	}
	return &options, nil
}

// GetRootlessRuntimeDir returns the runtime directory when running as non root
func GetRootlessRuntimeDir(rootlessUid int) (string, error) {
	runtimeDir := os.Getenv("XDG_RUNTIME_DIR")
	if runtimeDir == "" {
		tmpDir := fmt.Sprintf("/run/user/%d", rootlessUid)
		st, err := system.Stat(tmpDir)
		if err == nil && int(st.UID()) == os.Getuid() && st.Mode() == 0700 {
			return tmpDir, nil
		}
	}
	tmpDir := fmt.Sprintf("%s/%d", os.TempDir(), rootlessUid)
	if err := os.MkdirAll(tmpDir, 0700); err != nil {
		logrus.Errorf("failed to create %s: %v", tmpDir, err)
	} else {
		return tmpDir, nil
	}
	home, err := homeDir()
	if err != nil {
		return "", errors.Wrapf(err, "neither XDG_RUNTIME_DIR nor HOME was set non-empty")
	}
	resolvedHome, err := filepath.EvalSymlinks(home)
	if err != nil {
		return "", errors.Wrapf(err, "cannot resolve %s", home)
	}
	return filepath.Join(resolvedHome, "rundir"), nil
}

// getRootlessDirInfo returns the parent path of where the storage for containers and
// volumes will be in rootless mode
func getRootlessDirInfo(rootlessUid int) (string, string, error) {
	rootlessRuntime, err := GetRootlessRuntimeDir(rootlessUid)
	if err != nil {
		return "", "", err
	}

	dataDir := os.Getenv("XDG_DATA_HOME")
	if dataDir == "" {
		home, err := homeDir()
		if err != nil {
			return "", "", errors.Wrapf(err, "neither XDG_DATA_HOME nor HOME was set non-empty")
		}
		// runc doesn't like symlinks in the rootfs path, and at least
		// on CoreOS /home is a symlink to /var/home, so resolve any symlink.
		resolvedHome, err := filepath.EvalSymlinks(home)
		if err != nil {
			return "", "", errors.Wrapf(err, "cannot resolve %s", home)
		}
		dataDir = filepath.Join(resolvedHome, ".local", "share")
	}
	return dataDir, rootlessRuntime, nil
}

// getRootlessStorageOpts returns the storage opts for containers running as non root
func getRootlessStorageOpts(rootlessUid int) (StoreOptions, error) {
	var opts StoreOptions

	dataDir, rootlessRuntime, err := getRootlessDirInfo(rootlessUid)
	if err != nil {
		return opts, err
	}
	opts.RunRoot = rootlessRuntime
	opts.GraphRoot = filepath.Join(dataDir, "containers", "storage")
	if path, err := exec.LookPath("fuse-overlayfs"); err == nil {
		opts.GraphDriverName = "overlay"
		opts.GraphDriverOptions = []string{fmt.Sprintf("overlay.mount_program=%s", path)}
	} else {
		opts.GraphDriverName = "vfs"
	}
	return opts, nil
}

type tomlOptionsConfig struct {
	MountProgram string `toml:"mount_program"`
}

func getTomlStorage(storeOptions *StoreOptions) *tomlConfig {
	config := new(tomlConfig)

	config.Storage.Driver = storeOptions.GraphDriverName
	config.Storage.RunRoot = storeOptions.RunRoot
	config.Storage.GraphRoot = storeOptions.GraphRoot
	for _, i := range storeOptions.GraphDriverOptions {
		s := strings.Split(i, "=")
		if s[0] == "overlay.mount_program" {
			config.Storage.Options.MountProgram = s[1]
		}
	}

	return config
}

func getRootlessUID() int {
	uidEnv := os.Getenv("_CONTAINERS_ROOTLESS_UID")
	if uidEnv != "" {
		u, _ := strconv.Atoi(uidEnv)
		return u
	}
	return os.Geteuid()
}

// DefaultStoreOptionsAutoDetectUID returns the default storage ops for containers
func DefaultStoreOptionsAutoDetectUID() (StoreOptions, error) {
	uid := getRootlessUID()
	return DefaultStoreOptions(uid != 0, uid)
}

// DefaultStoreOptions returns the default storage ops for containers
func DefaultStoreOptions(rootless bool, rootlessUid int) (StoreOptions, error) {
	var (
		defaultRootlessRunRoot   string
		defaultRootlessGraphRoot string
		err                      error
	)
	storageOpts := defaultStoreOptions
	if rootless {
		storageOpts, err = getRootlessStorageOpts(rootlessUid)
		if err != nil {
			return storageOpts, err
		}
	}

	storageConf, err := DefaultConfigFile(rootless)
	if err != nil {
		return storageOpts, err
	}
	if _, err = os.Stat(storageConf); err == nil {
		defaultRootlessRunRoot = storageOpts.RunRoot
		defaultRootlessGraphRoot = storageOpts.GraphRoot
		storageOpts = StoreOptions{}
		ReloadConfigurationFile(storageConf, &storageOpts)
	}

	if !os.IsNotExist(err) {
		return storageOpts, errors.Wrapf(err, "cannot stat %s", storageConf)
	}

	if rootless {
		if err == nil {
			// If the file did not specify a graphroot or runroot,
			// set sane defaults so we don't try and use root-owned
			// directories
			if storageOpts.RunRoot == "" {
				storageOpts.RunRoot = defaultRootlessRunRoot
			}
			if storageOpts.GraphRoot == "" {
				storageOpts.GraphRoot = defaultRootlessGraphRoot
			}
		} else {
			if err := os.MkdirAll(filepath.Dir(storageConf), 0755); err != nil {
				return storageOpts, errors.Wrapf(err, "cannot make directory %s", filepath.Dir(storageConf))
			}
			file, err := os.OpenFile(storageConf, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0666)
			if err != nil {
				return storageOpts, errors.Wrapf(err, "cannot open %s", storageConf)
			}

			tomlConfiguration := getTomlStorage(&storageOpts)
			defer file.Close()
			enc := toml.NewEncoder(file)
			if err := enc.Encode(tomlConfiguration); err != nil {
				os.Remove(storageConf)

				return storageOpts, errors.Wrapf(err, "failed to encode %s", storageConf)
			}
		}
	}
	return storageOpts, nil
}

func homeDir() (string, error) {
	home := os.Getenv("HOME")
	if home == "" {
		usr, err := user.Current()
		if err != nil {
			return "", errors.Wrapf(err, "neither XDG_RUNTIME_DIR nor HOME was set non-empty")
		}
		home = usr.HomeDir
	}
	return home, nil
}
<commit_msg>Correctly check permissions on rootless directory<commit_after>package storage

import (
	"fmt"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/BurntSushi/toml"
	"github.com/containers/storage/pkg/idtools"
	"github.com/containers/storage/pkg/system"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

// ParseIDMapping takes idmappings and subuid and subgid maps and returns a storage mapping
func ParseIDMapping(UIDMapSlice, GIDMapSlice []string, subUIDMap, subGIDMap string) (*IDMappingOptions, error) {
	options := IDMappingOptions{
		HostUIDMapping: true,
		HostGIDMapping: true,
	}
	if subGIDMap == "" && subUIDMap != "" {
		subGIDMap = subUIDMap
	}
	if subUIDMap == "" && subGIDMap != "" {
		subUIDMap = subGIDMap
	}
	if len(GIDMapSlice) == 0 && len(UIDMapSlice) != 0 {
		GIDMapSlice = UIDMapSlice
	}
	if len(UIDMapSlice) == 0 && len(GIDMapSlice) != 0 {
		UIDMapSlice = GIDMapSlice
	}
	if len(UIDMapSlice) == 0 && subUIDMap == "" && os.Getuid() != 0 {
		UIDMapSlice = []string{fmt.Sprintf("0:%d:1", os.Getuid())}
	}
	if len(GIDMapSlice) == 0 && subGIDMap == "" && os.Getuid() != 0 {
		GIDMapSlice = []string{fmt.Sprintf("0:%d:1", os.Getgid())}
	}

	if subUIDMap != "" && subGIDMap != "" {
		mappings, err := idtools.NewIDMappings(subUIDMap, subGIDMap)
		if err != nil {
			return nil, errors.Wrapf(err, "failed to create NewIDMappings for uidmap=%s gidmap=%s", subUIDMap, subGIDMap)
		}
		options.UIDMap = mappings.UIDs()
		options.GIDMap = mappings.GIDs()
	}
	parsedUIDMap, err := idtools.ParseIDMap(UIDMapSlice, "UID")
	if err != nil {
		return nil, errors.Wrapf(err, "failed to create ParseUIDMap UID=%s", UIDMapSlice)
	}
	parsedGIDMap, err := idtools.ParseIDMap(GIDMapSlice, "GID")
	if err != nil {
		return nil, errors.Wrapf(err, "failed to create ParseGIDMap GID=%s", UIDMapSlice)
	}
	options.UIDMap = append(options.UIDMap, parsedUIDMap...)
	options.GIDMap = append(options.GIDMap, parsedGIDMap...)
	if len(options.UIDMap) > 0 {
		options.HostUIDMapping = false
	}
	if len(options.GIDMap) > 0 {
		options.HostGIDMapping = false
	}
	return &options, nil
}

// GetRootlessRuntimeDir returns the runtime directory when running as non root
func GetRootlessRuntimeDir(rootlessUid int) (string, error) {
	runtimeDir := os.Getenv("XDG_RUNTIME_DIR")
	if runtimeDir == "" {
		tmpDir := fmt.Sprintf("/run/user/%d", rootlessUid)
		st, err := system.Stat(tmpDir)
		if err == nil && int(st.UID()) == os.Getuid() && st.Mode()&0700 == 0700 && st.Mode()&0066 == 0000 {
			return tmpDir, nil
		}
	}
	tmpDir := fmt.Sprintf("%s/%d", os.TempDir(), rootlessUid)
	if err := os.MkdirAll(tmpDir, 0700); err != nil {
		logrus.Errorf("failed to create %s: %v", tmpDir, err)
	} else {
		return tmpDir, nil
	}
	home, err := homeDir()
	if err != nil {
		return "", errors.Wrapf(err, "neither XDG_RUNTIME_DIR nor HOME was set non-empty")
	}
	resolvedHome, err := filepath.EvalSymlinks(home)
	if err != nil {
		return "", errors.Wrapf(err, "cannot resolve %s", home)
	}
	return filepath.Join(resolvedHome, "rundir"), nil
}

// getRootlessDirInfo returns the parent path of where the storage for containers and
// volumes will be in rootless mode
func getRootlessDirInfo(rootlessUid int) (string, string, error) {
	rootlessRuntime, err := GetRootlessRuntimeDir(rootlessUid)
	if err != nil {
		return "", "", err
	}

	dataDir := os.Getenv("XDG_DATA_HOME")
	if dataDir == "" {
		home, err := homeDir()
		if err != nil {
			return "", "", errors.Wrapf(err, "neither XDG_DATA_HOME nor HOME was set non-empty")
		}
		// runc doesn't like symlinks in the rootfs path, and at least
		// on CoreOS /home is a symlink to /var/home, so resolve any symlink.
		resolvedHome, err := filepath.EvalSymlinks(home)
		if err != nil {
			return "", "", errors.Wrapf(err, "cannot resolve %s", home)
		}
		dataDir = filepath.Join(resolvedHome, ".local", "share")
	}
	return dataDir, rootlessRuntime, nil
}

// getRootlessStorageOpts returns the storage opts for containers running as non root
func getRootlessStorageOpts(rootlessUid int) (StoreOptions, error) {
	var opts StoreOptions

	dataDir, rootlessRuntime, err := getRootlessDirInfo(rootlessUid)
	if err != nil {
		return opts, err
	}
	opts.RunRoot = rootlessRuntime
	opts.GraphRoot = filepath.Join(dataDir, "containers", "storage")
	if path, err := exec.LookPath("fuse-overlayfs"); err == nil {
		opts.GraphDriverName = "overlay"
		opts.GraphDriverOptions = []string{fmt.Sprintf("overlay.mount_program=%s", path)}
	} else {
		opts.GraphDriverName = "vfs"
	}
	return opts, nil
}

type tomlOptionsConfig struct {
	MountProgram string `toml:"mount_program"`
}

func getTomlStorage(storeOptions *StoreOptions) *tomlConfig {
	config := new(tomlConfig)

	config.Storage.Driver = storeOptions.GraphDriverName
	config.Storage.RunRoot = storeOptions.RunRoot
	config.Storage.GraphRoot = storeOptions.GraphRoot
	for _, i := range storeOptions.GraphDriverOptions {
		s := strings.Split(i, "=")
		if s[0] == "overlay.mount_program" {
			config.Storage.Options.MountProgram = s[1]
		}
	}

	return config
}

func getRootlessUID() int {
	uidEnv := os.Getenv("_CONTAINERS_ROOTLESS_UID")
	if uidEnv != "" {
		u, _ := strconv.Atoi(uidEnv)
		return u
	}
	return os.Geteuid()
}

// DefaultStoreOptionsAutoDetectUID returns the default storage ops for containers
func DefaultStoreOptionsAutoDetectUID() (StoreOptions, error) {
	uid := getRootlessUID()
	return DefaultStoreOptions(uid != 0, uid)
}

// DefaultStoreOptions returns the default storage ops for containers
func DefaultStoreOptions(rootless bool, rootlessUid int) (StoreOptions, error) {
	var (
		defaultRootlessRunRoot   string
		defaultRootlessGraphRoot string
		err                      error
	)
	storageOpts := defaultStoreOptions
	if rootless {
		storageOpts, err = getRootlessStorageOpts(rootlessUid)
		if err != nil {
			return storageOpts, err
		}
	}

	storageConf, err := DefaultConfigFile(rootless)
	if err != nil {
		return storageOpts, err
	}
	if _, err = os.Stat(storageConf); err == nil {
		defaultRootlessRunRoot = storageOpts.RunRoot
		defaultRootlessGraphRoot = storageOpts.GraphRoot
		storageOpts = StoreOptions{}
		ReloadConfigurationFile(storageConf, &storageOpts)
	}

	if !os.IsNotExist(err) {
		return storageOpts, errors.Wrapf(err, "cannot stat %s", storageConf)
	}

	if rootless {
		if err == nil {
			// If the file did not specify a graphroot or runroot,
			// set sane defaults so we don't try and use root-owned
			// directories
			if storageOpts.RunRoot == "" {
				storageOpts.RunRoot = defaultRootlessRunRoot
			}
			if storageOpts.GraphRoot == "" {
				storageOpts.GraphRoot = defaultRootlessGraphRoot
			}
		} else {
			if err := os.MkdirAll(filepath.Dir(storageConf), 0755); err != nil {
				return storageOpts, errors.Wrapf(err, "cannot make directory %s", filepath.Dir(storageConf))
			}
			file, err := os.OpenFile(storageConf, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0666)
			if err != nil {
				return storageOpts, errors.Wrapf(err, "cannot open %s", storageConf)
			}

			tomlConfiguration := getTomlStorage(&storageOpts)
			defer file.Close()
			enc := toml.NewEncoder(file)
			if err := enc.Encode(tomlConfiguration); err != nil {
				os.Remove(storageConf)

				return storageOpts, errors.Wrapf(err, "failed to encode %s", storageConf)
			}
		}
	}
	return storageOpts, nil
}

func homeDir() (string, error) {
	home := os.Getenv("HOME")
	if home == "" {
		usr, err := user.Current()
		if err != nil {
			return "", errors.Wrapf(err, "neither XDG_RUNTIME_DIR nor HOME was set non-empty")
		}
		home = usr.HomeDir
	}
	return home, nil
}
<|endoftext|>


<commit_before>// Copyright 2014 Manu Martinez-Almeida.  All rights reserved.
// Use of this source code is governed by a MIT style
// license that can be found in the LICENSE file.

package gin

import (
	"encoding/xml"
	"net/http"
	"os"
	"path"
	"reflect"
	"runtime"
	"strings"
)

// BindKey indicates a default bind key.
const BindKey = "_gin-gonic/gin/bindkey"

// Bind is a helper function for given interface object and returns a Gin middleware.
func Bind(val interface{}) HandlerFunc {
	value := reflect.ValueOf(val)
	if value.Kind() == reflect.Ptr {
		panic(`Bind struct can not be a pointer. Example:
	Use: gin.Bind(Struct{}) instead of gin.Bind(&Struct{})
`)
	}
	typ := value.Type()

	return func(c *Context) {
		obj := reflect.New(typ).Interface()
		if c.Bind(obj) == nil {
			c.Set(BindKey, obj)
		}
	}
}

// WrapF is a helper function for wrapping http.HandlerFunc and returns a Gin middleware.
func WrapF(f http.HandlerFunc) HandlerFunc {
	return func(c *Context) {
		f(c.Writer, c.Request)
	}
}

// WrapH is a helper function for wrapping http.Handler and returns a Gin middleware.
func WrapH(h http.Handler) HandlerFunc {
	return func(c *Context) {
		h.ServeHTTP(c.Writer, c.Request)
	}
}

// H is a shortcut for map[string]interface{}
type H map[string]interface{}

// MarshalXML allows type H to be used with xml.Marshal.
func (h H) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
	start.Name = xml.Name{
		Space: "",
		Local: "map",
	}
	if err := e.EncodeToken(start); err != nil {
		return err
	}
	for key, value := range h {
		elem := xml.StartElement{
			Name: xml.Name{Space: "", Local: key},
			Attr: []xml.Attr{},
		}
		if err := e.EncodeElement(value, elem); err != nil {
			return err
		}
	}

	return e.EncodeToken(xml.EndElement{Name: start.Name})
}

func assert1(guard bool, text string) {
	if !guard {
		panic(text)
	}
}

func filterFlags(content string) string {
	for i, char := range content {
		if char == ' ' || char == ';' {
			return content[:i]
		}
	}
	return content
}

func chooseData(custom, wildcard interface{}) interface{} {
	if custom == nil {
		if wildcard == nil {
			panic("negotiation config is invalid")
		}
		return wildcard
	}
	return custom
}

func parseAccept(acceptHeader string) []string {
	parts := strings.Split(acceptHeader, ",")
	out := make([]string, 0, len(parts))
	for _, part := range parts {
		if part = strings.TrimSpace(strings.Split(part, ";")[0]); part != "" {
			out = append(out, part)
		}
	}
	return out
}

func lastChar(str string) uint8 {
	if str == "" {
		panic("The length of the string can't be 0")
	}
	return str[len(str)-1]
}

func nameOfFunction(f interface{}) string {
	return runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name()
}

func joinPaths(absolutePath, relativePath string) string {
	if relativePath == "" {
		return absolutePath
	}

	finalPath := path.Join(absolutePath, relativePath)
	appendSlash := lastChar(relativePath) == '/' && lastChar(finalPath) != '/'
	if appendSlash {
		return finalPath + "/"
	}
	return finalPath
}

func resolveAddress(addr []string) string {
	switch len(addr) {
	case 0:
		if port := os.Getenv("PORT"); port != "" {
			debugPrint("Environment variable PORT=\"%s\"", port)
			return ":" + port
		}
		debugPrint("Environment variable PORT is undefined. Using port :8080 by default")
		return ":8080"
	case 1:
		return addr[0]
	default:
		panic("too much parameters")
	}
}
<commit_msg>Typo (#1971)<commit_after>// Copyright 2014 Manu Martinez-Almeida.  All rights reserved.
// Use of this source code is governed by a MIT style
// license that can be found in the LICENSE file.

package gin

import (
	"encoding/xml"
	"net/http"
	"os"
	"path"
	"reflect"
	"runtime"
	"strings"
)

// BindKey indicates a default bind key.
const BindKey = "_gin-gonic/gin/bindkey"

// Bind is a helper function for given interface object and returns a Gin middleware.
func Bind(val interface{}) HandlerFunc {
	value := reflect.ValueOf(val)
	if value.Kind() == reflect.Ptr {
		panic(`Bind struct can not be a pointer. Example:
	Use: gin.Bind(Struct{}) instead of gin.Bind(&Struct{})
`)
	}
	typ := value.Type()

	return func(c *Context) {
		obj := reflect.New(typ).Interface()
		if c.Bind(obj) == nil {
			c.Set(BindKey, obj)
		}
	}
}

// WrapF is a helper function for wrapping http.HandlerFunc and returns a Gin middleware.
func WrapF(f http.HandlerFunc) HandlerFunc {
	return func(c *Context) {
		f(c.Writer, c.Request)
	}
}

// WrapH is a helper function for wrapping http.Handler and returns a Gin middleware.
func WrapH(h http.Handler) HandlerFunc {
	return func(c *Context) {
		h.ServeHTTP(c.Writer, c.Request)
	}
}

// H is a shortcut for map[string]interface{}
type H map[string]interface{}

// MarshalXML allows type H to be used with xml.Marshal.
func (h H) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
	start.Name = xml.Name{
		Space: "",
		Local: "map",
	}
	if err := e.EncodeToken(start); err != nil {
		return err
	}
	for key, value := range h {
		elem := xml.StartElement{
			Name: xml.Name{Space: "", Local: key},
			Attr: []xml.Attr{},
		}
		if err := e.EncodeElement(value, elem); err != nil {
			return err
		}
	}

	return e.EncodeToken(xml.EndElement{Name: start.Name})
}

func assert1(guard bool, text string) {
	if !guard {
		panic(text)
	}
}

func filterFlags(content string) string {
	for i, char := range content {
		if char == ' ' || char == ';' {
			return content[:i]
		}
	}
	return content
}

func chooseData(custom, wildcard interface{}) interface{} {
	if custom == nil {
		if wildcard == nil {
			panic("negotiation config is invalid")
		}
		return wildcard
	}
	return custom
}

func parseAccept(acceptHeader string) []string {
	parts := strings.Split(acceptHeader, ",")
	out := make([]string, 0, len(parts))
	for _, part := range parts {
		if part = strings.TrimSpace(strings.Split(part, ";")[0]); part != "" {
			out = append(out, part)
		}
	}
	return out
}

func lastChar(str string) uint8 {
	if str == "" {
		panic("The length of the string can't be 0")
	}
	return str[len(str)-1]
}

func nameOfFunction(f interface{}) string {
	return runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name()
}

func joinPaths(absolutePath, relativePath string) string {
	if relativePath == "" {
		return absolutePath
	}

	finalPath := path.Join(absolutePath, relativePath)
	appendSlash := lastChar(relativePath) == '/' && lastChar(finalPath) != '/'
	if appendSlash {
		return finalPath + "/"
	}
	return finalPath
}

func resolveAddress(addr []string) string {
	switch len(addr) {
	case 0:
		if port := os.Getenv("PORT"); port != "" {
			debugPrint("Environment variable PORT=\"%s\"", port)
			return ":" + port
		}
		debugPrint("Environment variable PORT is undefined. Using port :8080 by default")
		return ":8080"
	case 1:
		return addr[0]
	default:
		panic("too many parameters")
	}
}
<|endoftext|>


<commit_before>package gobot

import (
	"math"
	"math/rand"
	"reflect"
	"time"
)

func Every(t string, f func()) {
	dur := parseDuration(t)
	go func() {
		for {
			time.Sleep(dur)
			go f()
		}
	}()
}

func After(t string, f func()) {
	dur := parseDuration(t)
	go func() {
		time.Sleep(dur)
		f()
	}()
}

func Publish(c chan interface{}, val interface{}) {
	select {
	case c <- val:
	default:
	}
}

func On(c chan interface{}, f func(s interface{})) {
	go func() {
		for s := range c {
			f(s)
		}
	}()
}

func Rand(max int) int {
	rand.Seed(time.Now().UTC().UnixNano())
	return rand.Intn(max)
}

func Call(thing interface{}, method string, params ...interface{}) []reflect.Value {
	in := make([]reflect.Value, len(params))
	for k, param := range params {
		in[k] = reflect.ValueOf(param)
	}
	return reflect.ValueOf(thing).MethodByName(method).Call(in)
}

func FieldByName(thing interface{}, field string) reflect.Value {
	return reflect.ValueOf(thing).FieldByName(field)
}
func FieldByNamePtr(thing interface{}, field string) reflect.Value {
	return reflect.ValueOf(thing).Elem().FieldByName(field)
}

func FromScale(input, min, max float64) float64 {
	return (input - math.Min(min, max)) / (math.Max(min, max) - math.Min(min, max))
}

func ToScale(input, min, max float64) float64 {
	i := input*(math.Max(min, max)-math.Min(min, max)) + math.Min(min, max)
	if i < math.Min(min, max) {
		return math.Min(min, max)
	} else if i > math.Max(min, max) {
		return math.Max(min, max)
	} else {
		return i
	}
}

func parseDuration(t string) time.Duration {
	dur, err := time.ParseDuration(t)
	if err != nil {
		panic(err)
	}
	return dur
}
<commit_msg>removed sleeps from every and after funcs<commit_after>package gobot

import (
	"math"
	"math/rand"
	"reflect"
	"time"
)

// Every triggers f every `t` time until the end of days.
func Every(t string, f func()) {
	c := time.Tick(parseDuration(t))
	// start a go routine to not bloc the function
	go func() {
		for {
			// wait for the ticker to tell us to run
			<-c
			// run the passed function in another go routine
			// so we don't slow down the loop.
			go f()
		}
	}()
}

// After triggers the passed function after `t` duration.
func After(t string, f func()) {
	time.AfterFunc(parseDuration(t), f)
}

func Publish(c chan interface{}, val interface{}) {
	select {
	case c <- val:
	default:
	}
}

func On(c chan interface{}, f func(s interface{})) {
	go func() {
		for s := range c {
			f(s)
		}
	}()
}

func Rand(max int) int {
	rand.Seed(time.Now().UTC().UnixNano())
	return rand.Intn(max)
}

func Call(thing interface{}, method string, params ...interface{}) []reflect.Value {
	in := make([]reflect.Value, len(params))
	for k, param := range params {
		in[k] = reflect.ValueOf(param)
	}
	return reflect.ValueOf(thing).MethodByName(method).Call(in)
}

func FieldByName(thing interface{}, field string) reflect.Value {
	return reflect.ValueOf(thing).FieldByName(field)
}
func FieldByNamePtr(thing interface{}, field string) reflect.Value {
	return reflect.ValueOf(thing).Elem().FieldByName(field)
}

func FromScale(input, min, max float64) float64 {
	return (input - math.Min(min, max)) / (math.Max(min, max) - math.Min(min, max))
}

func ToScale(input, min, max float64) float64 {
	i := input*(math.Max(min, max)-math.Min(min, max)) + math.Min(min, max)
	if i < math.Min(min, max) {
		return math.Min(min, max)
	} else if i > math.Max(min, max) {
		return math.Max(min, max)
	} else {
		return i
	}
}

func parseDuration(t string) time.Duration {
	dur, err := time.ParseDuration(t)
	if err != nil {
		panic(err)
	}
	return dur
}
<|endoftext|>


<commit_before>package oidc

import (
	"context"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"sync"
	"time"

	jose "gopkg.in/square/go-jose.v2"
)

// NewRemoteKeySet returns a KeySet that can validate JSON web tokens by using HTTP
// GETs to fetch JSON web token sets hosted at a remote URL. This is automatically
// used by NewProvider using the URLs returned by OpenID Connect discovery, but is
// exposed for providers that don't support discovery or to prevent round trips to the
// discovery URL.
//
// The returned KeySet is a long lived verifier that caches keys based on cache-control
// headers. Reuse a common remote key set instead of creating new ones as needed.
//
// The behavior of the returned KeySet is undefined once the context is canceled.
func NewRemoteKeySet(ctx context.Context, jwksURL string) KeySet {
	return newRemoteKeySet(ctx, jwksURL, time.Now)
}

func newRemoteKeySet(ctx context.Context, jwksURL string, now func() time.Time) *remoteKeySet {
	if now == nil {
		now = time.Now
	}
	return &remoteKeySet{jwksURL: jwksURL, ctx: ctx, now: now}
}

type remoteKeySet struct {
	jwksURL string
	ctx     context.Context
	now     func() time.Time

	// guard all other fields
	mu sync.Mutex

	// inflight suppresses parallel execution of updateKeys and allows
	// multiple goroutines to wait for its result.
	inflight *inflight

	// A set of cached keys.
	cachedKeys []jose.JSONWebKey
}

// inflight is used to wait on some in-flight request from multiple goroutines.
type inflight struct {
	doneCh chan struct{}

	keys []jose.JSONWebKey
	err  error
}

func newInflight() *inflight {
	return &inflight{doneCh: make(chan struct{})}
}

// wait returns a channel that multiple goroutines can receive on. Once it returns
// a value, the inflight request is done and result() can be inspected.
func (i *inflight) wait() <-chan struct{} {
	return i.doneCh
}

// done can only be called by a single goroutine. It records the result of the
// inflight request and signals other goroutines that the result is safe to
// inspect.
func (i *inflight) done(keys []jose.JSONWebKey, err error) {
	i.keys = keys
	i.err = err
	close(i.doneCh)
}

// result cannot be called until the wait() channel has returned a value.
func (i *inflight) result() ([]jose.JSONWebKey, error) {
	return i.keys, i.err
}

func (r *remoteKeySet) VerifySignature(ctx context.Context, jwt string) ([]byte, error) {
	jws, err := jose.ParseSigned(jwt)
	if err != nil {
		return nil, fmt.Errorf("oidc: malformed jwt: %v", err)
	}
	return r.verify(ctx, jws)
}

func (r *remoteKeySet) verify(ctx context.Context, jws *jose.JSONWebSignature) ([]byte, error) {
	// We don't support JWTs signed with multiple signatures.
	keyID := ""
	for _, sig := range jws.Signatures {
		keyID = sig.Header.KeyID
		break
	}

	keys := r.keysFromCache()
	for _, key := range keys {
		if keyID == "" || key.KeyID == keyID {
			if payload, err := jws.Verify(&key); err == nil {
				return payload, nil
			}
		}
	}

	// If the kid doesn't match, check for new keys from the remote. This is the
	// strategy recommended by the spec.
	//
	// https://openid.net/specs/openid-connect-core-1_0.html#RotateSigKeys
	keys, err := r.keysFromRemote(ctx)
	if err != nil {
		return nil, fmt.Errorf("fetching keys %v", err)
	}

	for _, key := range keys {
		if keyID == "" || key.KeyID == keyID {
			if payload, err := jws.Verify(&key); err == nil {
				return payload, nil
			}
		}
	}
	return nil, errors.New("failed to verify id token signature")
}

func (r *remoteKeySet) keysFromCache() (keys []jose.JSONWebKey) {
	r.mu.Lock()
	defer r.mu.Unlock()
	return r.cachedKeys
}

// keysFromRemote syncs the key set from the remote set, records the values in the
// cache, and returns the key set.
func (r *remoteKeySet) keysFromRemote(ctx context.Context) ([]jose.JSONWebKey, error) {
	// Need to lock to inspect the inflight request field.
	r.mu.Lock()
	// If there's not a current inflight request, create one.
	if r.inflight == nil {
		r.inflight = newInflight()

		// This goroutine has exclusive ownership over the current inflight
		// request. It releases the resource by nil'ing the inflight field
		// once the goroutine is done.
		go func() {
			// Sync keys and finish inflight when that's done.
			keys, err := r.updateKeys()

			r.inflight.done(keys, err)

			// Lock to update the keys and indicate that there is no longer an
			// inflight request.
			r.mu.Lock()
			defer r.mu.Unlock()

			if err == nil {
				r.cachedKeys = keys
			}

			// Free inflight so a different request can run.
			r.inflight = nil
		}()
	}
	inflight := r.inflight
	r.mu.Unlock()

	select {
	case <-ctx.Done():
		return nil, ctx.Err()
	case <-inflight.wait():
		return inflight.result()
	}
}

func (r *remoteKeySet) updateKeys() ([]jose.JSONWebKey, error) {
	req, err := http.NewRequest("GET", r.jwksURL, nil)
	if err != nil {
		return nil, fmt.Errorf("oidc: can't create request: %v", err)
	}

	resp, err := doRequest(r.ctx, req)
	if err != nil {
		return nil, fmt.Errorf("oidc: get keys failed %v", err)
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, fmt.Errorf("unable to read response body: %v", err)
	}

	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("oidc: get keys failed: %s %s", resp.Status, body)
	}

	var keySet jose.JSONWebKeySet
	err = unmarshalResp(resp, body, &keySet)
	if err != nil {
		return nil, fmt.Errorf("oidc: failed to decode keys: %v %s", err, body)
	}
	return keySet.Keys, nil
}
<commit_msg>oidc: have NewRemoteKeySet return a concrete type<commit_after>package oidc

import (
	"context"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"sync"
	"time"

	jose "gopkg.in/square/go-jose.v2"
)

// NewRemoteKeySet returns a KeySet that can validate JSON web tokens by using HTTP
// GETs to fetch JSON web token sets hosted at a remote URL. This is automatically
// used by NewProvider using the URLs returned by OpenID Connect discovery, but is
// exposed for providers that don't support discovery or to prevent round trips to the
// discovery URL.
//
// The returned KeySet is a long lived verifier that caches keys based on cache-control
// headers. Reuse a common remote key set instead of creating new ones as needed.
func NewRemoteKeySet(ctx context.Context, jwksURL string) *RemoteKeySet {
	return newRemoteKeySet(ctx, jwksURL, time.Now)
}

func newRemoteKeySet(ctx context.Context, jwksURL string, now func() time.Time) *RemoteKeySet {
	if now == nil {
		now = time.Now
	}
	return &RemoteKeySet{jwksURL: jwksURL, ctx: cloneContext(ctx), now: now}
}

// RemoteKeySet is a KeySet implementation that validates JSON web tokens against
// a jwks_uri endpoint.
type RemoteKeySet struct {
	jwksURL string
	ctx     context.Context
	now     func() time.Time

	// guard all other fields
	mu sync.Mutex

	// inflight suppresses parallel execution of updateKeys and allows
	// multiple goroutines to wait for its result.
	inflight *inflight

	// A set of cached keys.
	cachedKeys []jose.JSONWebKey
}

// inflight is used to wait on some in-flight request from multiple goroutines.
type inflight struct {
	doneCh chan struct{}

	keys []jose.JSONWebKey
	err  error
}

func newInflight() *inflight {
	return &inflight{doneCh: make(chan struct{})}
}

// wait returns a channel that multiple goroutines can receive on. Once it returns
// a value, the inflight request is done and result() can be inspected.
func (i *inflight) wait() <-chan struct{} {
	return i.doneCh
}

// done can only be called by a single goroutine. It records the result of the
// inflight request and signals other goroutines that the result is safe to
// inspect.
func (i *inflight) done(keys []jose.JSONWebKey, err error) {
	i.keys = keys
	i.err = err
	close(i.doneCh)
}

// result cannot be called until the wait() channel has returned a value.
func (i *inflight) result() ([]jose.JSONWebKey, error) {
	return i.keys, i.err
}

// VerifySignature validates a payload against a signature from the jwks_uri.
//
// Users MUST NOT call this method directly and should use an IDTokenVerifier
// instead. This method skips critical validations such as 'alg' values and is
// only exported to implement the KeySet interface.
func (r *RemoteKeySet) VerifySignature(ctx context.Context, jwt string) ([]byte, error) {
	jws, err := jose.ParseSigned(jwt)
	if err != nil {
		return nil, fmt.Errorf("oidc: malformed jwt: %v", err)
	}
	return r.verify(ctx, jws)
}

func (r *RemoteKeySet) verify(ctx context.Context, jws *jose.JSONWebSignature) ([]byte, error) {
	// We don't support JWTs signed with multiple signatures.
	keyID := ""
	for _, sig := range jws.Signatures {
		keyID = sig.Header.KeyID
		break
	}

	keys := r.keysFromCache()
	for _, key := range keys {
		if keyID == "" || key.KeyID == keyID {
			if payload, err := jws.Verify(&key); err == nil {
				return payload, nil
			}
		}
	}

	// If the kid doesn't match, check for new keys from the remote. This is the
	// strategy recommended by the spec.
	//
	// https://openid.net/specs/openid-connect-core-1_0.html#RotateSigKeys
	keys, err := r.keysFromRemote(ctx)
	if err != nil {
		return nil, fmt.Errorf("fetching keys %v", err)
	}

	for _, key := range keys {
		if keyID == "" || key.KeyID == keyID {
			if payload, err := jws.Verify(&key); err == nil {
				return payload, nil
			}
		}
	}
	return nil, errors.New("failed to verify id token signature")
}

func (r *RemoteKeySet) keysFromCache() (keys []jose.JSONWebKey) {
	r.mu.Lock()
	defer r.mu.Unlock()
	return r.cachedKeys
}

// keysFromRemote syncs the key set from the remote set, records the values in the
// cache, and returns the key set.
func (r *RemoteKeySet) keysFromRemote(ctx context.Context) ([]jose.JSONWebKey, error) {
	// Need to lock to inspect the inflight request field.
	r.mu.Lock()
	// If there's not a current inflight request, create one.
	if r.inflight == nil {
		r.inflight = newInflight()

		// This goroutine has exclusive ownership over the current inflight
		// request. It releases the resource by nil'ing the inflight field
		// once the goroutine is done.
		go func() {
			// Sync keys and finish inflight when that's done.
			keys, err := r.updateKeys()

			r.inflight.done(keys, err)

			// Lock to update the keys and indicate that there is no longer an
			// inflight request.
			r.mu.Lock()
			defer r.mu.Unlock()

			if err == nil {
				r.cachedKeys = keys
			}

			// Free inflight so a different request can run.
			r.inflight = nil
		}()
	}
	inflight := r.inflight
	r.mu.Unlock()

	select {
	case <-ctx.Done():
		return nil, ctx.Err()
	case <-inflight.wait():
		return inflight.result()
	}
}

func (r *RemoteKeySet) updateKeys() ([]jose.JSONWebKey, error) {
	req, err := http.NewRequest("GET", r.jwksURL, nil)
	if err != nil {
		return nil, fmt.Errorf("oidc: can't create request: %v", err)
	}

	resp, err := doRequest(r.ctx, req)
	if err != nil {
		return nil, fmt.Errorf("oidc: get keys failed %v", err)
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, fmt.Errorf("unable to read response body: %v", err)
	}

	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("oidc: get keys failed: %s %s", resp.Status, body)
	}

	var keySet jose.JSONWebKeySet
	err = unmarshalResp(resp, body, &keySet)
	if err != nil {
		return nil, fmt.Errorf("oidc: failed to decode keys: %v %s", err, body)
	}
	return keySet.Keys, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package ghcache

import (
	"bufio"
	"bytes"
	"net/http"
	"net/http/httputil"
	"strconv"
	"sync"
	"time"

	"github.com/sirupsen/logrus"

	"k8s.io/test-infra/ghproxy/ghmetrics"
)

// requestCoalescer allows concurrent requests for the same URI to share a
// single upstream request and response.
type requestCoalescer struct {
	sync.Mutex
	keys map[string]*responseWaiter

	delegate http.RoundTripper

	hasher ghmetrics.Hasher
}

type responseWaiter struct {
	*sync.Cond

	waiting bool
	resp    []byte
	err     error
}

// RoundTrip coalesces concurrent GET requests for the same URI by blocking
// the later requests until the first request returns and then sharing the
// response between all requests.
//
// Notes: Deadlock shouldn't be possible because the map lock is always
// acquired before responseWaiter lock if both locks are to be held and we
// never hold multiple responseWaiter locks.
func (r *requestCoalescer) RoundTrip(req *http.Request) (*http.Response, error) {
	// Only coalesce GET requests
	if req.Method != http.MethodGet {
		return r.delegate.RoundTrip(req)
	}

	var cacheMode = ModeError
	resp, err := func() (*http.Response, error) {
		key := req.URL.String()
		r.Lock()
		waiter, ok := r.keys[key]
		if ok {
			// Earlier request in flight. Wait for it's response.
			if req.Body != nil {
				defer req.Body.Close() // Since we won't pass the request we must close it.
			}
			waiter.L.Lock()
			r.Unlock()
			waiter.waiting = true
			// The documentation for Wait() says:
			// "Because c.L is not locked when Wait first resumes, the caller typically
			// cannot assume that the condition is true when Wait returns. Instead, the
			// caller should Wait in a loop."
			// This does not apply to this use of Wait() because the condition we are
			// waiting for remains true once it becomes true. This lets us avoid the
			// normal check to see if the condition has switched back to false between
			// the signal being sent and this thread acquiring the lock.
			waiter.Wait()
			waiter.L.Unlock()
			// Earlier request completed.

			if waiter.err != nil {
				// Don't log the error, it will be logged by requester.
				return nil, waiter.err
			}
			resp, err := http.ReadResponse(bufio.NewReader(bytes.NewBuffer(waiter.resp)), nil)
			if err != nil {
				logrus.WithField("cache-key", key).WithError(err).Error("Error loading response.")
				return nil, err
			}

			cacheMode = ModeCoalesced
			return resp, nil
		}
		// No earlier request in flight (common case).
		// Register a new responseWaiter and make the request ourself.
		waiter = &responseWaiter{Cond: sync.NewCond(&sync.Mutex{})}
		r.keys[key] = waiter
		r.Unlock()

		resp, err := r.delegate.RoundTrip(req)
		// Real response received. Remove this responseWaiter from the map THEN
		// wake any requesters that were waiting on this response.
		r.Lock()
		delete(r.keys, key)
		r.Unlock()

		waiter.L.Lock()
		if waiter.waiting {
			if err != nil {
				waiter.resp, waiter.err = nil, err
			} else {
				// Copy the response before releasing to waiter(s).
				waiter.resp, waiter.err = httputil.DumpResponse(resp, true)
			}
			waiter.Broadcast()
		}
		waiter.L.Unlock()

		if err != nil {
			logrus.WithField("cache-key", key).WithError(err).Warn("Error from cache transport layer.")
			return nil, err
		}
		cacheMode = cacheResponseMode(resp.Header)
		return resp, nil
	}()

	var tokenBudgetName string
	if val := req.Header.Get(TokenBudgetIdentifierHeader); val != "" {
		tokenBudgetName = val
	} else {
		tokenBudgetName = r.hasher.Hash(req)
	}

	ghmetrics.CollectCacheRequestMetrics(string(cacheMode), req.URL.Path, req.Header.Get("User-Agent"), tokenBudgetName)
	if resp != nil {
		resp.Header.Set(CacheModeHeader, string(cacheMode))
		if cacheMode == ModeRevalidated && resp.Header.Get(cacheEntryCreationDateHeader) != "" {
			intVal, err := strconv.Atoi(resp.Header.Get(cacheEntryCreationDateHeader))
			if err != nil {
				logrus.WithError(err).WithField("header-value", resp.Header.Get(cacheEntryCreationDateHeader)).Warn("Failed to convert cacheEntryCreationDateHeader value to int")
			} else {
				ghmetrics.CollectCacheEntryAgeMetrics(float64(time.Now().Unix()-int64(intVal)), req.URL.Path, req.Header.Get("User-Agent"), r.hasher.Hash(req))
			}
		}
	}
	return resp, err
}
<commit_msg>GHProxy: Manage cache metrics for v4 api<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package ghcache

import (
	"bufio"
	"bytes"
	"net/http"
	"net/http/httputil"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/sirupsen/logrus"

	"k8s.io/test-infra/ghproxy/ghmetrics"
)

// requestCoalescer allows concurrent requests for the same URI to share a
// single upstream request and response.
type requestCoalescer struct {
	sync.Mutex
	keys map[string]*responseWaiter

	delegate http.RoundTripper

	hasher ghmetrics.Hasher
}

type responseWaiter struct {
	*sync.Cond

	waiting bool
	resp    []byte
	err     error
}

// RoundTrip coalesces concurrent GET requests for the same URI by blocking
// the later requests until the first request returns and then sharing the
// response between all requests.
//
// Notes: Deadlock shouldn't be possible because the map lock is always
// acquired before responseWaiter lock if both locks are to be held and we
// never hold multiple responseWaiter locks.
func (r *requestCoalescer) RoundTrip(req *http.Request) (*http.Response, error) {
	// Only coalesce GET requests
	if req.Method != http.MethodGet {
		resp, err := r.delegate.RoundTrip(req)
		if strings.HasPrefix(req.URL.Path, "graphql") || strings.HasPrefix(req.URL.Path, "/graphql") {
			var tokenBudgetName string
			if val := req.Header.Get(TokenBudgetIdentifierHeader); val != "" {
				tokenBudgetName = val
			} else {
				tokenBudgetName = r.hasher.Hash(req)
			}
			collectMetrics(ModeNoStore, req, resp, tokenBudgetName)
		}
		return resp, err
	}

	var cacheMode = ModeError
	resp, err := func() (*http.Response, error) {
		key := req.URL.String()
		r.Lock()
		waiter, ok := r.keys[key]
		if ok {
			// Earlier request in flight. Wait for it's response.
			if req.Body != nil {
				defer req.Body.Close() // Since we won't pass the request we must close it.
			}
			waiter.L.Lock()
			r.Unlock()
			waiter.waiting = true
			// The documentation for Wait() says:
			// "Because c.L is not locked when Wait first resumes, the caller typically
			// cannot assume that the condition is true when Wait returns. Instead, the
			// caller should Wait in a loop."
			// This does not apply to this use of Wait() because the condition we are
			// waiting for remains true once it becomes true. This lets us avoid the
			// normal check to see if the condition has switched back to false between
			// the signal being sent and this thread acquiring the lock.
			waiter.Wait()
			waiter.L.Unlock()
			// Earlier request completed.

			if waiter.err != nil {
				// Don't log the error, it will be logged by requester.
				return nil, waiter.err
			}
			resp, err := http.ReadResponse(bufio.NewReader(bytes.NewBuffer(waiter.resp)), nil)
			if err != nil {
				logrus.WithField("cache-key", key).WithError(err).Error("Error loading response.")
				return nil, err
			}

			cacheMode = ModeCoalesced
			return resp, nil
		}
		// No earlier request in flight (common case).
		// Register a new responseWaiter and make the request ourself.
		waiter = &responseWaiter{Cond: sync.NewCond(&sync.Mutex{})}
		r.keys[key] = waiter
		r.Unlock()

		resp, err := r.delegate.RoundTrip(req)
		// Real response received. Remove this responseWaiter from the map THEN
		// wake any requesters that were waiting on this response.
		r.Lock()
		delete(r.keys, key)
		r.Unlock()

		waiter.L.Lock()
		if waiter.waiting {
			if err != nil {
				waiter.resp, waiter.err = nil, err
			} else {
				// Copy the response before releasing to waiter(s).
				waiter.resp, waiter.err = httputil.DumpResponse(resp, true)
			}
			waiter.Broadcast()
		}
		waiter.L.Unlock()

		if err != nil {
			logrus.WithField("cache-key", key).WithError(err).Warn("Error from cache transport layer.")
			return nil, err
		}
		cacheMode = cacheResponseMode(resp.Header)
		return resp, nil
	}()

	var tokenBudgetName string
	if val := req.Header.Get(TokenBudgetIdentifierHeader); val != "" {
		tokenBudgetName = val
	} else {
		tokenBudgetName = r.hasher.Hash(req)
	}

	collectMetrics(cacheMode, req, resp, tokenBudgetName)
	return resp, err
}

func collectMetrics(cacheMode CacheResponseMode, req *http.Request, resp *http.Response, tokenBudgetName string) {
	ghmetrics.CollectCacheRequestMetrics(string(cacheMode), req.URL.Path, req.Header.Get("User-Agent"), tokenBudgetName)
	if resp != nil {
		resp.Header.Set(CacheModeHeader, string(cacheMode))
		if cacheMode == ModeRevalidated && resp.Header.Get(cacheEntryCreationDateHeader) != "" {
			intVal, err := strconv.Atoi(resp.Header.Get(cacheEntryCreationDateHeader))
			if err != nil {
				logrus.WithError(err).WithField("header-value", resp.Header.Get(cacheEntryCreationDateHeader)).Warn("Failed to convert cacheEntryCreationDateHeader value to int")
			} else {
				ghmetrics.CollectCacheEntryAgeMetrics(float64(time.Now().Unix()-int64(intVal)), req.URL.Path, req.Header.Get("User-Agent"), tokenBudgetName)
			}
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Dorival de Moraes Pedroso. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package goga

import (
	"math"

	"github.com/cpmech/gosl/rnd"
)

// DiffEvol performs the differential-evolution operation
func DiffEvol(xnew, x, x0, x1, x2 []float64, prms *Parameters) {
	n := len(x)
	C := rnd.Float64(0.0, 1.0/math.Pow(float64(n), 0.1))
	F := rnd.Float64(0.0, 1.0)
	I := rnd.Int(0, n-1)
	for i := 0; i < n; i++ {
		if rnd.FlipCoin(C) || i == I {
			xnew[i] = x0[i] + F*(x1[i]-x2[i])
			if xnew[i] < prms.FltMin[i] {
				xnew[i] = prms.FltMin[i]
			}
			if xnew[i] > prms.FltMax[i] {
				xnew[i] = prms.FltMax[i]
			}
		} else {
			xnew[i] = x[i]
		}
	}
}
<commit_msg>C coefficient in operators simplified<commit_after>// Copyright 2015 Dorival de Moraes Pedroso. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package goga

import "github.com/cpmech/gosl/rnd"

// DiffEvol performs the differential-evolution operation
func DiffEvol(xnew, x, x0, x1, x2 []float64, prms *Parameters) {
	n := len(x)
	C := rnd.Float64(0.0, 1.0)
	F := rnd.Float64(0.0, 1.0)
	I := rnd.Int(0, n-1)
	for i := 0; i < n; i++ {
		if rnd.FlipCoin(C) || i == I {
			xnew[i] = x0[i] + F*(x1[i]-x2[i])
			if xnew[i] < prms.FltMin[i] {
				xnew[i] = prms.FltMin[i]
			}
			if xnew[i] > prms.FltMax[i] {
				xnew[i] = prms.FltMax[i]
			}
		} else {
			xnew[i] = x[i]
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"html/template"
	"io"
	"log"
	"net/http"
	"os"
	"path"
	"time"

	"github.com/cenkalti/backoff"
	"github.com/facebookgo/stackerr"
	"github.com/fsouza/go-dockerclient"
	"github.com/gorilla/mux"
)

// If |useTLS| is enabled, look for the cert files here.
var defaultCertPath = os.Getenv("DOCKER_CERT_PATH")

var (
	staticPath  = flag.String("static", "./static", "serve static files located in this directory")
	dockerHost  = flag.String("docker", "tcp://192.168.59.103:2376", "address of docker daemon host")
	tlsCertPath = flag.String("tls-cert-path", defaultCertPath, "path to docker host TLS certificate")
	host        = flag.String("host", ":8080", "address:port for HTTP listener")
	useTLS      = flag.Bool("tls", true, "use Docker TLS")
)

func main() {
	flag.Parse()
	if err := run(); err != nil {
		log.Fatal(err)
	}
}

func run() error {
	var client *docker.Client
	var err error
	if *useTLS {
		client, err = docker.NewTLSClient(
			*dockerHost,
			path.Join(*tlsCertPath, "cert.pem"),
			path.Join(*tlsCertPath, "key.pem"),
			path.Join(*tlsCertPath, "ca.pem"),
		)
		if err != nil {
			return stackerr.Wrap(err)
		}
	} else {
		client, err = docker.NewClient(*dockerHost)
		if err != nil {
			return err
		}
	}
	s := &AgentSupervisor{Client: client}
	log.Fatal(http.ListenAndServe(*host, Router(s)))
	return nil
}

// Router defines URL routes
func Router(s *AgentSupervisor) *mux.Router {
	r := mux.NewRouter()
	WireAPIRoutes(r, s)
	WireHTMLRoutes(r, s) // after API routes

	// finally, if no other routes match, see if the client-provided path names
	// a static asset
	r.NotFoundHandler = http.FileServer(http.Dir(*staticPath))
	return r
}

var templateHome = `

fxx >>= ai

	<form action=/api/v0/validate method=GET>

	validate your AI... Enter the name of the Docker image:
		<input type=text name=image>
		</input>
	</form>

`

func WireHTMLRoutes(r *mux.Router, s *AgentSupervisor) {

	r.HandleFunc("/html", func(w http.ResponseWriter, r *http.Request) {
		tmpl, err := template.New("name").Parse(templateHome)
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			return
		}
		// Error checking elided
		if err := tmpl.Execute(w, nil); err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			return
		}
	})
}

func WireAPIRoutes(r *mux.Router, s *AgentSupervisor) {

	apiV0 := r.PathPrefix("/api/v0").Subrouter()

	// TODO Build an agent from a GitHub repo
	// TODO Run Game between two running agents

	agents := apiV0.Path("/agents").Subrouter()
	agents.Methods("GET").HandlerFunc(ListAgentsHandler(s)) // list all available agents

	// pull an agent from Dockerhub
	dockerhub := apiV0.Path("/pull/dockerhub").Subrouter()
	dockerhub.Methods("GET").HandlerFunc(PullDockerHubAgentHandler(s)) // FIXME make this a POST

	// TODO List running agents with parameter
	apiV0.Path("/running").HandlerFunc(ListActiveAgentsHandler(s))

	// checks that the agent implements the right protocol
	apiV0.Path("/validate").HandlerFunc(ValidateAgentHandler(s))

	apiV0.Path("/image").HandlerFunc(ShowAgentInfoHandler(s))
}

// PullDockerHubAgentHandler pulls the Docker image named |image| from
// DockerHub.
func PullDockerHubAgentHandler(s *AgentSupervisor) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		if r.FormValue("image") == "" {
			w.WriteHeader(http.StatusBadRequest)
			return
		}
		if err := s.Client.PullImage(docker.PullImageOptions{
			OutputStream: w,
			Registry:     "https://index.docker.io",
			Repository:   r.FormValue("image"),
			Tag:          "latest",
		}, docker.AuthConfiguration{}); err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintln(w, err)
		}
	}
}

// ListAgentsHandler lists AI agents.
func ListAgentsHandler(s *AgentSupervisor) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		images, err := s.Client.ListImages(docker.ListImagesOptions{
			All: false,
		})
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			_, err := io.WriteString(w, err.Error())
			if err != nil {
				log.Println("error writing err: %s", err)
			}
			return
		}
		for _, img := range images {
			_, err := fmt.Fprintln(w, fmt.Sprintln(append([]string{img.ID, "\t"}, img.RepoTags...)))
			if err != nil {
				w.WriteHeader(http.StatusInternalServerError)
				_, err := io.WriteString(w, err.Error())
				if err != nil {
					log.Println("error writing err: %s", err)
				}
			}
		}
	}
}

// ListActiveAgentsHandler lists agents that are currently running.
func ListActiveAgentsHandler(s *AgentSupervisor) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		containers, err := s.Client.ListContainers(docker.ListContainersOptions{})
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			return
		}
		for _, ps := range containers {
			fmt.Fprintln(w, fmt.Sprintf("%+v\n", ps))
		}
	}
}

// ValidateAgentHandler ensures that the Agent image responds to the PING
// command.
func ValidateAgentHandler(s *AgentSupervisor) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		defer log.Println("returned")

		// client gives the |image| as a URL parameter
		if r.FormValue("image") == "" {
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprintln(w, "`image` parameter is required")
			return
		}
		image := r.FormValue("image")

		if err := s.ValidateImage(image); err != nil {
			fmt.Fprintf(w, "image %s is not valid. error: %s", image, err)
			return
		}
		fmt.Fprintf(w, "image %s is valid", image)
	}
}

type AgentInfo struct {
	Owner  string
	Taunts []string
}

// ShowImageHandler shows information about an AI Agent
func ShowAgentInfoHandler(s *AgentSupervisor) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		image, err := s.Client.InspectImage("jbenet/go-ipfs")
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			return
		}
		json.NewEncoder(w).Encode(image.Config.ExposedPorts)
	}
}

// AgentSupervisor manages AI agents running in Docker containers
type AgentSupervisor struct {
	Client *docker.Client
}

func (s *AgentSupervisor) ValidateImage(image string) error {

	// run the container in a two-phase process. First, create the
	// container.
	container, err := s.Client.CreateContainer(docker.CreateContainerOptions{
		Config: &docker.Config{
			Image: image, // the only required argument
		},
	})
	if err != nil {
		return fmt.Errorf("error creating container:", err.Error())
	}

	hc := &docker.HostConfig{
		PublishAllPorts: true,
	}

	// run the created container
	if err := s.Client.StartContainer(container.ID, hc); err != nil {
		return fmt.Errorf("error starting container: %s", err.Error())
	}

	// ensure the proper port is exposed
	info, err := s.Client.InspectContainer(container.ID)
	if err != nil {
		return fmt.Errorf("error inspecting container: %s", err.Error())
	}

	mappings, ok := info.NetworkSettings.Ports[docker.Port("3423/tcp")]
	if !ok {
		return fmt.Errorf(
			"container must expose port 3423/tcp. Found: %+v",
			info.NetworkSettings.Ports)
	}
	if len(mappings) != 1 {
		return fmt.Errorf(
			"error. expected one port mapping. found: %+v",
			info.NetworkSettings.Ports)
	}
	ip, port := mappings[0].HostIP, mappings[0].HostPort

	backoffConfig := backoff.NewExponentialBackOff()
	backoffConfig.InitialInterval = time.Second
	backoffConfig.MaxInterval = 10
	backoffConfig.MaxElapsedTime = 10 * time.Second
	err = backoff.Retry(func() error {
		resp, err := http.Get(fmt.Sprintf("http://%s:%s/ping", ip, port))
		if err != nil {
			log.Println("error pinging agent. found:", err)
			// TODO handle err
			return err
		}
		defer resp.Body.Close()
		var agentInfo AgentInfo
		if err := json.NewDecoder(resp.Body).Decode(&agentInfo); err != nil {
			return err
		}
		if agentInfo.Owner == "btc" {
			log.Println("yay!")
		}
		return nil
	}, backoffConfig)

	const kStopContainerTimeout = 5 // seconds
	if err := s.Client.StopContainer(container.ID, kStopContainerTimeout); err != nil {
		return err
	}
	return nil
}
<commit_msg>fix(httpd) rewrite the path to allow the client to route<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"html/template"
	"io"
	"log"
	"net/http"
	"os"
	"path"
	"time"

	"github.com/cenkalti/backoff"
	"github.com/facebookgo/stackerr"
	"github.com/fsouza/go-dockerclient"
	"github.com/gorilla/mux"
)

// If |useTLS| is enabled, look for the cert files here.
var defaultCertPath = os.Getenv("DOCKER_CERT_PATH")

var (
	staticPath  = flag.String("static", "./static", "serve static files located in this directory")
	dockerHost  = flag.String("docker", "tcp://192.168.59.103:2376", "address of docker daemon host")
	tlsCertPath = flag.String("tls-cert-path", defaultCertPath, "path to docker host TLS certificate")
	host        = flag.String("host", ":8080", "address:port for HTTP listener")
	useTLS      = flag.Bool("tls", true, "use Docker TLS")
)

func main() {
	flag.Parse()
	if err := run(); err != nil {
		log.Fatal(err)
	}
}

func run() error {
	var client *docker.Client
	var err error
	if *useTLS {
		client, err = docker.NewTLSClient(
			*dockerHost,
			path.Join(*tlsCertPath, "cert.pem"),
			path.Join(*tlsCertPath, "key.pem"),
			path.Join(*tlsCertPath, "ca.pem"),
		)
		if err != nil {
			return stackerr.Wrap(err)
		}
	} else {
		client, err = docker.NewClient(*dockerHost)
		if err != nil {
			return err
		}
	}
	s := &AgentSupervisor{Client: client}
	log.Fatal(http.ListenAndServe(*host, Router(s)))
	return nil
}

// Router defines URL routes
func Router(s *AgentSupervisor) *mux.Router {
	r := mux.NewRouter()
	WireAPIRoutes(r, s)
	WireHTMLRoutes(r, s) // after API routes

	// Finally, if none of the above routes match, delegate to the single-page
	// app's client-side router. Rewrite the path in order to load the
	// single-page app's root HTML entrypoint. The app will handle the route.
	r.NotFoundHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		r.URL.Path = "/"
		http.FileServer(http.Dir(*staticPath)).ServeHTTP(w, r)
	})
	return r
}

var templateHome = `

fxx >>= ai

	<form action=/api/v0/validate method=GET>

	validate your AI... Enter the name of the Docker image:
		<input type=text name=image>
		</input>
	</form>

`

func WireHTMLRoutes(r *mux.Router, s *AgentSupervisor) {

	r.HandleFunc("/html", func(w http.ResponseWriter, r *http.Request) {
		tmpl, err := template.New("name").Parse(templateHome)
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			return
		}
		// Error checking elided
		if err := tmpl.Execute(w, nil); err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			return
		}
	})
}

func WireAPIRoutes(r *mux.Router, s *AgentSupervisor) {

	apiV0 := r.PathPrefix("/api/v0").Subrouter()

	// TODO Build an agent from a GitHub repo
	// TODO Run Game between two running agents

	agents := apiV0.Path("/agents").Subrouter()
	agents.Methods("GET").HandlerFunc(ListAgentsHandler(s)) // list all available agents

	// pull an agent from Dockerhub
	dockerhub := apiV0.Path("/pull/dockerhub").Subrouter()
	dockerhub.Methods("GET").HandlerFunc(PullDockerHubAgentHandler(s)) // FIXME make this a POST

	// TODO List running agents with parameter
	apiV0.Path("/running").HandlerFunc(ListActiveAgentsHandler(s))

	// checks that the agent implements the right protocol
	apiV0.Path("/validate").HandlerFunc(ValidateAgentHandler(s))

	apiV0.Path("/image").HandlerFunc(ShowAgentInfoHandler(s))
}

// PullDockerHubAgentHandler pulls the Docker image named |image| from
// DockerHub.
func PullDockerHubAgentHandler(s *AgentSupervisor) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		if r.FormValue("image") == "" {
			w.WriteHeader(http.StatusBadRequest)
			return
		}
		if err := s.Client.PullImage(docker.PullImageOptions{
			OutputStream: w,
			Registry:     "https://index.docker.io",
			Repository:   r.FormValue("image"),
			Tag:          "latest",
		}, docker.AuthConfiguration{}); err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintln(w, err)
		}
	}
}

// ListAgentsHandler lists AI agents.
func ListAgentsHandler(s *AgentSupervisor) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		images, err := s.Client.ListImages(docker.ListImagesOptions{
			All: false,
		})
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			_, err := io.WriteString(w, err.Error())
			if err != nil {
				log.Println("error writing err: %s", err)
			}
			return
		}
		for _, img := range images {
			_, err := fmt.Fprintln(w, fmt.Sprintln(append([]string{img.ID, "\t"}, img.RepoTags...)))
			if err != nil {
				w.WriteHeader(http.StatusInternalServerError)
				_, err := io.WriteString(w, err.Error())
				if err != nil {
					log.Println("error writing err: %s", err)
				}
			}
		}
	}
}

// ListActiveAgentsHandler lists agents that are currently running.
func ListActiveAgentsHandler(s *AgentSupervisor) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		containers, err := s.Client.ListContainers(docker.ListContainersOptions{})
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			return
		}
		for _, ps := range containers {
			fmt.Fprintln(w, fmt.Sprintf("%+v\n", ps))
		}
	}
}

// ValidateAgentHandler ensures that the Agent image responds to the PING
// command.
func ValidateAgentHandler(s *AgentSupervisor) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		defer log.Println("returned")

		// client gives the |image| as a URL parameter
		if r.FormValue("image") == "" {
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprintln(w, "`image` parameter is required")
			return
		}
		image := r.FormValue("image")

		if err := s.ValidateImage(image); err != nil {
			fmt.Fprintf(w, "image %s is not valid. error: %s", image, err)
			return
		}
		fmt.Fprintf(w, "image %s is valid", image)
	}
}

type AgentInfo struct {
	Owner  string
	Taunts []string
}

// ShowImageHandler shows information about an AI Agent
func ShowAgentInfoHandler(s *AgentSupervisor) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		image, err := s.Client.InspectImage("jbenet/go-ipfs")
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			return
		}
		json.NewEncoder(w).Encode(image.Config.ExposedPorts)
	}
}

// AgentSupervisor manages AI agents running in Docker containers
type AgentSupervisor struct {
	Client *docker.Client
}

func (s *AgentSupervisor) ValidateImage(image string) error {

	// run the container in a two-phase process. First, create the
	// container.
	container, err := s.Client.CreateContainer(docker.CreateContainerOptions{
		Config: &docker.Config{
			Image: image, // the only required argument
		},
	})
	if err != nil {
		return fmt.Errorf("error creating container:", err.Error())
	}

	hc := &docker.HostConfig{
		PublishAllPorts: true,
	}

	// run the created container
	if err := s.Client.StartContainer(container.ID, hc); err != nil {
		return fmt.Errorf("error starting container: %s", err.Error())
	}

	// ensure the proper port is exposed
	info, err := s.Client.InspectContainer(container.ID)
	if err != nil {
		return fmt.Errorf("error inspecting container: %s", err.Error())
	}

	mappings, ok := info.NetworkSettings.Ports[docker.Port("3423/tcp")]
	if !ok {
		return fmt.Errorf(
			"container must expose port 3423/tcp. Found: %+v",
			info.NetworkSettings.Ports)
	}
	if len(mappings) != 1 {
		return fmt.Errorf(
			"error. expected one port mapping. found: %+v",
			info.NetworkSettings.Ports)
	}
	ip, port := mappings[0].HostIP, mappings[0].HostPort

	backoffConfig := backoff.NewExponentialBackOff()
	backoffConfig.InitialInterval = time.Second
	backoffConfig.MaxInterval = 10
	backoffConfig.MaxElapsedTime = 10 * time.Second
	err = backoff.Retry(func() error {
		resp, err := http.Get(fmt.Sprintf("http://%s:%s/ping", ip, port))
		if err != nil {
			log.Println("error pinging agent. found:", err)
			// TODO handle err
			return err
		}
		defer resp.Body.Close()
		var agentInfo AgentInfo
		if err := json.NewDecoder(resp.Body).Decode(&agentInfo); err != nil {
			return err
		}
		if agentInfo.Owner == "btc" {
			log.Println("yay!")
		}
		return nil
	}, backoffConfig)

	const kStopContainerTimeout = 5 // seconds
	if err := s.Client.StopContainer(container.ID, kStopContainerTimeout); err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/kierdavis/KMail/go"
	"net/http"
	"os"
	"sync"
    "github.com/kierdavis/zlog"
    "strings"
    "time"
)

var Logger = zlog.NewLogger("kmailqueue")

type Queue struct {
	sync.Mutex
	IPAddress string
	Messages []*kmail.Message
}

func (q *Queue) Put(msg *kmail.Message) {
	q.Lock()
	defer q.Unlock()

	q.Messages = append(q.Messages, msg)
}

func (q *Queue) Get() (msgs []*kmail.Message) {
	q.Lock()
	defer q.Unlock()

	msgs = q.Messages
	q.Messages = nil
	return msgs
}

var Queues = make(map[string]*Queue)
var QueuesLock sync.Mutex

func GetQueue(name string) (queue *Queue, ok bool) {
	QueuesLock.Lock()
	defer QueuesLock.Unlock()

	queue, ok = Queues[name]
	return queue, ok
}

func NewQueue(name string, ipaddress string) (queue *Queue) {
	queue = new(Queue)
	queue.IPAddress = ipaddress
	queue.Messages = nil

	QueuesLock.Lock()
	defer QueuesLock.Unlock()

	Queues[name] = queue
	return queue
}

var Address = kmail.Address{"QUEUE", ""}

func Reply(msg *kmail.Message, body string) {
	Logger.Info("Replying to %s with: %s", msg.Dest.String(), body)

	go func() {
		err := msg.Reply(Address, body)
		if err != nil {
			Logger.Error("Error sending reply: %s", err.Error())
		}
	}()
}

func HandleMessage(msg *kmail.Message) {
	hostname := msg.Dest.Hostname
	queue, ok := GetQueue(hostname)

	if !ok {
		Reply(msg, fmt.Sprintf("No host named '%s' registered on this queue server.", hostname))
		return
	}

	queue.Put(msg)
}

func GetIP(r *http.Request) (ip string) {
	pos := strings.Index(r.RemoteAddr, ":")
	if pos < 0 {
		return r.RemoteAddr
	}

	return r.RemoteAddr[:pos]
}

func HandleFetch(w http.ResponseWriter, r *http.Request) {
	hostname := r.FormValue("hostname")
	if hostname == "" {
		Logger.Info("%s %s -> 400 Bad Request: No hostname parameter given.", r.Method, r.RequestURI)
		w.WriteHeader(400)
		fmt.Fprintf(w, "No hostname parameter given.\r\n")
		return
	}

	queue, ok := GetQueue(hostname)
	if !ok {
		Logger.Info("%s %s -> 404 Not Found: No host named '%s' registered on this queue server.", r.Method, r.RequestURI, hostname)
		w.WriteHeader(404)
		fmt.Fprintf(w, "No host named '%s' registered on this queue server.\r\n", hostname)
		return
	}

	if queue.IPAddress != GetIP(r) {
		Logger.Info("%s %s -> 403 Forbidden: Once created, a queue may only be accessed by the IP address it was created on.", r.Method, r.RequestURI)
		w.WriteHeader(403)
		fmt.Fprintf(w, "Once created, a queue may only be accessed by the IP address it was created on.\r\n")
		return
	}

	messages := queue.Get()

	Logger.Info("%s %s -> 200 OK: %d messages returned.", r.Method, r.RequestURI, len(messages))
	w.WriteHeader(200)

	err := kmail.SerializeXML(w, messages)
	if err != nil {
		Logger.Error("Error serializing messages: %s", err.Error())
		return
	}
}

func HandleRegister(w http.ResponseWriter, r *http.Request) {
	hostname := r.FormValue("hostname")
	if hostname == "" {
		Logger.Info("%s %s -> 400 Bad Request: No hostname parameter given.", r.Method, r.RequestURI)
		w.WriteHeader(400)
		fmt.Fprintf(w, "No hostname parameter given.\r\n")
		return
	}

	NewQueue(hostname, GetIP(r))
	Logger.Info("%s %s -> 200 OK: Queue '%s' created.", r.Method, r.RequestURI, hostname)
	w.WriteHeader(200)
}

func RunMailServer(bindaddr string) {
	sm := http.NewServeMux()
	sm.HandleFunc("/", kmail.HandleMessages(HandleMessage, Logger.SubSource("kmail")))
	sm.HandleFunc("/fetch", HandleFetch)
	sm.HandleFunc("/register", HandleRegister)

	Logger.Info("Listening on %s", bindaddr)
	err := http.ListenAndServe(bindaddr, sm)
	if err != nil {
		Logger.Error("Error running server: %s", err.Error())
		return
	}
}

func main() {
	// Initialise logger
	Logger.AddOutput(zlog.NewConsoleOutput())
	go Logger.DispatchAll()

	// Check command line
	if len(os.Args) < 3 {
		fmt.Fprintf(os.Stderr, "Usage: %s <mailhostname> <bindaddr>\n", os.Args[0])
		return
	}

	Address.Hostname = os.Args[1]

	// Start server
	RunMailServer(os.Args[2])

	// Yield to allow logging to complete
	time.Sleep(time.Millisecond)
}
<commit_msg>(go) Check for pre-existing hosts in register<commit_after>package main

import (
	"fmt"
	"github.com/kierdavis/KMail/go"
	"net/http"
	"os"
	"sync"
    "github.com/kierdavis/zlog"
    "strings"
    "time"
)

var Logger = zlog.NewLogger("kmailqueue")

type Queue struct {
	sync.Mutex
	IPAddress string
	Messages []*kmail.Message
}

func (q *Queue) Put(msg *kmail.Message) {
	q.Lock()
	defer q.Unlock()

	q.Messages = append(q.Messages, msg)
}

func (q *Queue) Get() (msgs []*kmail.Message) {
	q.Lock()
	defer q.Unlock()

	msgs = q.Messages
	q.Messages = nil
	return msgs
}

var Queues = make(map[string]*Queue)
var QueuesLock sync.Mutex

func GetQueue(name string) (queue *Queue, ok bool) {
	QueuesLock.Lock()
	defer QueuesLock.Unlock()

	queue, ok = Queues[name]
	return queue, ok
}

func NewQueue(name string, ipaddress string) (queue *Queue) {
	queue = new(Queue)
	queue.IPAddress = ipaddress
	queue.Messages = nil

	QueuesLock.Lock()
	defer QueuesLock.Unlock()

	Queues[name] = queue
	return queue
}

var Address = kmail.Address{"QUEUE", ""}

func Reply(msg *kmail.Message, body string) {
	Logger.Info("Replying to %s with: %s", msg.Dest.String(), body)

	go func() {
		err := msg.Reply(Address, body)
		if err != nil {
			Logger.Error("Error sending reply: %s", err.Error())
		}
	}()
}

func HandleMessage(msg *kmail.Message) {
	hostname := msg.Dest.Hostname
	queue, ok := GetQueue(hostname)

	if !ok {
		Reply(msg, fmt.Sprintf("No host named '%s' registered on this queue server.", hostname))
		return
	}

	queue.Put(msg)
}

func GetIP(r *http.Request) (ip string) {
	pos := strings.Index(r.RemoteAddr, ":")
	if pos < 0 {
		return r.RemoteAddr
	}

	return r.RemoteAddr[:pos]
}

func HandleFetch(w http.ResponseWriter, r *http.Request) {
	hostname := r.FormValue("hostname")
	if hostname == "" {
		Logger.Info("%s %s -> 400 Bad Request: No hostname parameter given.", r.Method, r.RequestURI)
		w.WriteHeader(400)
		fmt.Fprintf(w, "No hostname parameter given.\r\n")
		return
	}

	queue, ok := GetQueue(hostname)
	if !ok {
		Logger.Info("%s %s -> 404 Not Found: No host named '%s' registered on this queue server.", r.Method, r.RequestURI, hostname)
		w.WriteHeader(404)
		fmt.Fprintf(w, "No host named '%s' registered on this queue server.\r\n", hostname)
		return
	}

	if queue.IPAddress != GetIP(r) {
		Logger.Info("%s %s -> 403 Forbidden: Once created, a queue may only be accessed by the IP address it was created on.", r.Method, r.RequestURI)
		w.WriteHeader(403)
		fmt.Fprintf(w, "Once created, a queue may only be accessed by the IP address it was created on.\r\n")
		return
	}

	messages := queue.Get()

	Logger.Info("%s %s -> 200 OK: %d messages returned.", r.Method, r.RequestURI, len(messages))
	w.WriteHeader(200)

	err := kmail.SerializeXML(w, messages)
	if err != nil {
		Logger.Error("Error serializing messages: %s", err.Error())
		return
	}
}

func HandleRegister(w http.ResponseWriter, r *http.Request) {
	hostname := r.FormValue("hostname")
	if hostname == "" {
		Logger.Info("%s %s -> 400 Bad Request: No hostname parameter given.", r.Method, r.RequestURI)
		w.WriteHeader(400)
		fmt.Fprintf(w, "No hostname parameter given.\r\n")
		return
	}

	_, ok := GetQueue(hostname)
	if ok {
		Logger.Info("%s %s -> 400 Bad Request: Hostname '%s' already registered.", r.Method, r.RequestURI, hostname)
		w.WriteHeader(400)
		fmt.Fprintf(w, "Hostname '%s' already registered.\r\n", hostname)
		return
	}

	NewQueue(hostname, GetIP(r))
	Logger.Info("%s %s -> 200 OK: Queue '%s' created.", r.Method, r.RequestURI, hostname)
	w.WriteHeader(200)
}

func RunMailServer(bindaddr string) {
	sm := http.NewServeMux()
	sm.HandleFunc("/", kmail.HandleMessages(HandleMessage, Logger.SubSource("kmail")))
	sm.HandleFunc("/fetch", HandleFetch)
	sm.HandleFunc("/register", HandleRegister)

	Logger.Info("Listening on %s", bindaddr)
	err := http.ListenAndServe(bindaddr, sm)
	if err != nil {
		Logger.Error("Error running server: %s", err.Error())
		return
	}
}

func main() {
	// Initialise logger
	Logger.AddOutput(zlog.NewConsoleOutput())
	go Logger.DispatchAll()

	// Check command line
	if len(os.Args) < 3 {
		fmt.Fprintf(os.Stderr, "Usage: %s <mailhostname> <bindaddr>\n", os.Args[0])
		return
	}

	Address.Hostname = os.Args[1]

	// Start server
	RunMailServer(os.Args[2])

	// Yield to allow logging to complete
	time.Sleep(time.Millisecond)
}
<|endoftext|>


<commit_before>// Copyright 2012, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package wrangler

import (
	"fmt"
	"net"
	"path"
	"strings"
	"sync"
	"time"

	"code.google.com/p/vitess/go/relog"
	rpc "code.google.com/p/vitess/go/rpcplus"
	vtrpc "code.google.com/p/vitess/go/vt/rpc"
	tm "code.google.com/p/vitess/go/vt/tabletmanager"
	"code.google.com/p/vitess/go/zk"
)

// As with all distributed systems, things can skew. These functions
// explore data in zookeeper and attempt to square that with reality.
//
// Given the node counts are usually large, this work should be done
// with as much parallelism as is viable.
//
// This may eventually move into a separate package.

type vresult struct {
	zkPath string
	err    error
}

func (wr *Wrangler) waitForResults(wg *sync.WaitGroup, results chan vresult) error {
	timer := time.NewTimer(wr.actionTimeout())
	done := make(chan bool, 1)
	go func() {
		wg.Wait()
		done <- true
	}()

	var err error
wait:
	for {
		select {
		case vd := <-results:
			relog.Info("checking %v", vd.zkPath)
			if vd.err != nil {
				err = fmt.Errorf("some validation errors - see log")
				relog.Error("%v: %v", vd.zkPath, vd.err)
			}
		case <-timer.C:
			err = fmt.Errorf("timed out during validate")
			break wait
		case <-done:
			// To prevent a false positive, once we are 'done',
			// drain the result channel completely.
			for {
				select {
				case vd := <-results:
					relog.Info("checking %v", vd.zkPath)
					if vd.err != nil {
						err = fmt.Errorf("some validation errors - see log")
						relog.Error("%v: %v", vd.zkPath, vd.err)
					}
				default:
					break wait
				}
			}
		}
	}

	return err
}

// Validate all tablets in all discoverable cells, even if they are
// not in the replication graph.
func (wr *Wrangler) validateAllTablets(zkKeyspacesPath string, wg *sync.WaitGroup, results chan<- vresult) {
	replicationPaths, err := zk.ChildrenRecursive(wr.zconn, zkKeyspacesPath)
	if err != nil {
		results <- vresult{zkKeyspacesPath, err}
		return
	}

	cellSet := make(map[string]bool, 16)
	for _, p := range replicationPaths {
		p := path.Join(zkKeyspacesPath, p)
		if tm.IsTabletReplicationPath(p) {
			cell, _ := tm.ParseTabletReplicationPath(p)
			cellSet[cell] = true
		}
	}

	for cell, _ := range cellSet {
		zkTabletsPath := path.Join("/zk", cell, tm.VtSubtree(zkKeyspacesPath), "tablets")
		tabletUids, _, err := wr.zconn.Children(zkTabletsPath)
		if err != nil {
			results <- vresult{zkTabletsPath, err}
		} else {
			for _, tabletUid := range tabletUids {
				tabletPath := path.Join(zkTabletsPath, tabletUid)
				wg.Add(1)
				go func() {
					results <- vresult{tabletPath, tm.Validate(wr.zconn, tabletPath, "")}
					wg.Done()
				}()
			}
		}
	}
}

func (wr *Wrangler) validateKeyspace(zkKeyspacePath string, pingTablets bool, wg *sync.WaitGroup, results chan<- vresult) {
	// Validate replication graph by traversing each shard.
	zkShardsPath := path.Join(zkKeyspacePath, "shards")
	shards, _, err := wr.zconn.Children(zkShardsPath)
	if err != nil {
		results <- vresult{zkShardsPath, err}
	}
	for _, shard := range shards {
		zkShardPath := path.Join(zkShardsPath, shard)
		wg.Add(1)
		go func() {
			wr.validateShard(zkShardPath, pingTablets, wg, results)
			wg.Done()
		}()
	}
}

func (wr *Wrangler) validateShard(zkShardPath string, pingTablets bool, wg *sync.WaitGroup, results chan<- vresult) {
	shardInfo, err := tm.ReadShard(wr.zconn, zkShardPath)
	if err != nil {
		results <- vresult{zkShardPath, err}
		return
	}

	aliases, err := tm.FindAllTabletAliasesInShard(wr.zconn, zkShardPath)
	if err != nil {
		results <- vresult{zkShardPath, err}
	}

	shardTablets := make([]string, 0, 16)
	for _, alias := range aliases {
		shardTablets = append(shardTablets, tm.TabletPathForAlias(alias))
	}

	tabletMap, _ := GetTabletMap(wr.zconn, shardTablets)

	var masterAlias tm.TabletAlias
	for _, alias := range aliases {
		zkTabletPath := tm.TabletPathForAlias(alias)
		tabletInfo, ok := tabletMap[zkTabletPath]
		if !ok {
			results <- vresult{zkTabletPath, fmt.Errorf("tablet not found in map: %v", zkTabletPath)}
			continue
		}
		if tabletInfo.Parent.Uid == tm.NO_TABLET {
			if masterAlias.Cell != "" {
				results <- vresult{zkTabletPath, fmt.Errorf("%v: already has a master %v", zkTabletPath, masterAlias)}
			} else {
				masterAlias = alias
			}
		}
	}

	if masterAlias.Cell == "" {
		results <- vresult{zkShardPath, fmt.Errorf("no master for shard %v", zkShardPath)}
	} else if shardInfo.MasterAlias != masterAlias {
		results <- vresult{zkShardPath, fmt.Errorf("master mismatch for shard %v: found %v, expected %v", zkShardPath, masterAlias, shardInfo.MasterAlias)}
	}

	for _, alias := range aliases {
		zkTabletPath := tm.TabletPathForAlias(alias)
		zkTabletReplicationPath := path.Join(zkShardPath, masterAlias.String())
		if alias != masterAlias {
			zkTabletReplicationPath += "/" + alias.String()
		}
		wg.Add(1)
		go func() {
			results <- vresult{zkTabletReplicationPath, tm.Validate(wr.zconn, zkTabletPath, zkTabletReplicationPath)}
			wg.Done()
		}()
	}

	if pingTablets {
		wr.validateReplication(shardInfo, tabletMap, results)
		wr.pingTablets(tabletMap, wg, results)
	}

	return
}

func strInList(sl []string, s string) bool {
	for _, x := range sl {
		if x == s {
			return true
		}
	}
	return false
}

func (wr *Wrangler) validateReplication(shardInfo *tm.ShardInfo, tabletMap map[string]*tm.TabletInfo, results chan<- vresult) {
	masterTabletPath := tm.TabletPathForAlias(shardInfo.MasterAlias)
	masterTablet, ok := tabletMap[masterTabletPath]
	if !ok {
		err := fmt.Errorf("master not in tablet map: %v", masterTabletPath)
		results <- vresult{masterTabletPath, err}
		return
	}

	slaveAddrs, err := getSlaves(masterTablet.Tablet)
	if err != nil {
		results <- vresult{masterTabletPath, err}
		return
	}

	if len(slaveAddrs) == 0 {
		results <- vresult{masterTabletPath, fmt.Errorf("no slaves found: %v", masterTabletPath)}
		return
	}

	// Some addresses don't resolve in all locations, just use IP address
	if err != nil {
		results <- vresult{masterTabletPath, fmt.Errorf("resolve slaves failed: %v", err)}
		return
	}

	tabletIpMap := make(map[string]*tm.Tablet)
	for tabletPath, tablet := range tabletMap {
		ipAddr, _, err := net.SplitHostPort(tablet.MysqlIpAddr)
		if err != nil {
			results <- vresult{tabletPath, fmt.Errorf("bad mysql addr: %v %v %v", tablet.MysqlIpAddr, tabletPath, err)}
			continue
		}
		tabletIpMap[ipAddr] = tablet.Tablet
	}

	// See if every slave is in the replication graph.
	for _, slaveAddr := range slaveAddrs {
		if tabletIpMap[slaveAddr] == nil {
			results <- vresult{shardInfo.ShardPath(), fmt.Errorf("slave not in replication graph: %v (mysql instance without vttablet?)", slaveAddr)}
		}
	}

	// See if every entry in the replication graph is connected to the master.
	for tabletPath, tablet := range tabletMap {
		if !tablet.IsSlaveType() {
			continue
		}

		ipAddr, _, err := net.SplitHostPort(tablet.MysqlIpAddr)
		if err != nil {
			results <- vresult{tabletPath, fmt.Errorf("bad mysql addr: %v %v", tabletPath, err)}
		} else if !strInList(slaveAddrs, ipAddr) {
			results <- vresult{tabletPath, fmt.Errorf("slave not replicating: %v %v %q", tabletPath, ipAddr, slaveAddrs)}
		}
	}
}

func (wr *Wrangler) pingTablets(tabletMap map[string]*tm.TabletInfo, wg *sync.WaitGroup, results chan<- vresult) {
	for zkTabletPath, tabletInfo := range tabletMap {
		wg.Add(1)
		go func(zkTabletPath string, tabletInfo *tm.TabletInfo) {
			defer wg.Done()

			zkTabletPid := path.Join(tabletInfo.Path(), "pid")
			_, _, err := wr.zconn.Get(zkTabletPid)
			if err != nil {
				results <- vresult{zkTabletPath, fmt.Errorf("no pid node %v: %v %v", zkTabletPid, err, tabletInfo.Hostname())}
				return
			}

			actionPath, err := wr.ai.Ping(zkTabletPath)
			if err != nil {
				results <- vresult{zkTabletPath, fmt.Errorf("%v: %v %v", actionPath, err, tabletInfo.Hostname())}
				return
			}

			err = wr.ai.WaitForCompletion(actionPath, wr.actionTimeout())
			if err != nil {
				results <- vresult{zkTabletPath, fmt.Errorf("%v: %v %v", actionPath, err, tabletInfo.Hostname())}
			}
		}(zkTabletPath, tabletInfo)
	}
}

// Validate a whole zk tree
func (wr *Wrangler) Validate(zkKeyspacesPath string, pingTablets bool) error {
	// Results from various actions feed here.
	results := make(chan vresult, 16)
	wg := &sync.WaitGroup{}

	// Validate all tablets in all cells, even if they are not discoverable
	// by the replication graph.
	wr.validateAllTablets(zkKeyspacesPath, wg, results)

	// Validate replication graph by traversing each keyspace and then each shard.
	keyspaces, _, err := wr.zconn.Children(zkKeyspacesPath)
	if err != nil {
		results <- vresult{zkKeyspacesPath, err}
	} else {
		for _, keyspace := range keyspaces {
			zkKeyspace := path.Join(zkKeyspacesPath, keyspace)
			wg.Add(1)
			go func() {
				wr.validateKeyspace(zkKeyspace, pingTablets, wg, results)
				wg.Done()
			}()
		}
	}
	return wr.waitForResults(wg, results)
}

func (wr *Wrangler) ValidateKeyspace(zkKeyspacePath string, pingTablets bool) error {
	wg := &sync.WaitGroup{}
	results := make(chan vresult, 16)
	wr.validateKeyspace(zkKeyspacePath, pingTablets, wg, results)
	return wr.waitForResults(wg, results)
}

func (wr *Wrangler) ValidateShard(zkShardPath string, pingTablets bool) error {
	wg := &sync.WaitGroup{}
	results := make(chan vresult, 16)
	wr.validateShard(zkShardPath, pingTablets, wg, results)
	return wr.waitForResults(wg, results)
}

// Slaves come back as IP addrs, resolve to host names.
func resolveSlaveNames(addrs []string) (hostnames []string, err error) {
	hostnames = make([]string, len(addrs))
	for i, addr := range addrs {
		if net.ParseIP(addr) != nil {
			ipAddrs, err := net.LookupAddr(addr)
			if err != nil {
				return nil, err
			}
			addr = ipAddrs[0]
		}
		cname, err := net.LookupCNAME(addr)
		if err != nil {
			return nil, err
		}
		hostnames[i] = strings.TrimRight(cname, ".")
	}
	return
}

// Get list of slave ip addresses from the tablet.
// FIXME(alainjobart) this should not use RPC, but an actionNode
// FIXME(alainjobart) once this uses an actionNode, remove TabletManager RPC
func getSlaves(tablet *tm.Tablet) ([]string, error) {
	timer := time.NewTimer(10 * time.Second)
	defer timer.Stop()

	callChan := make(chan *rpc.Call, 1)

	go func() {
		client, clientErr := rpc.DialHTTP("tcp", tablet.Addr)
		if clientErr != nil {
			callChan <- &rpc.Call{Error: fmt.Errorf("dial failed: %v", clientErr)}
			return
		}

		slaveList := new(tm.SlaveList)
		// Forward the message so we close the connection after the rpc is done.
		done := make(chan *rpc.Call, 1)
		client.Go("TabletManager.GetSlaves", vtrpc.NilRequest, slaveList, done)
		callChan <- <-done
		client.Close()
	}()

	select {
	case <-timer.C:
		return nil, fmt.Errorf("TabletManager.GetSlaves deadline exceeded %v", tablet.Addr)
	case call := <-callChan:
		if call.Error != nil {
			return nil, call.Error
		}
		return call.Reply.(*tm.SlaveList).Addrs, nil
	}

	panic("unreachable")
}
<commit_msg>prevent validation from blocking indefinitely while waiting for data<commit_after>// Copyright 2012, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package wrangler

import (
	"fmt"
	"net"
	"path"
	"strings"
	"sync"
	"time"

	"code.google.com/p/vitess/go/relog"
	rpc "code.google.com/p/vitess/go/rpcplus"
	vtrpc "code.google.com/p/vitess/go/vt/rpc"
	tm "code.google.com/p/vitess/go/vt/tabletmanager"
	"code.google.com/p/vitess/go/zk"
)

// As with all distributed systems, things can skew. These functions
// explore data in zookeeper and attempt to square that with reality.
//
// Given the node counts are usually large, this work should be done
// with as much parallelism as is viable.
//
// This may eventually move into a separate package.

type vresult struct {
	zkPath string
	err    error
}

func (wr *Wrangler) waitForResults(wg *sync.WaitGroup, results chan vresult) error {
	timer := time.NewTimer(wr.actionTimeout())
	done := make(chan bool, 1)
	go func() {
		wg.Wait()
		done <- true
	}()

	var err error
wait:
	for {
		select {
		case vd := <-results:
			relog.Info("checking %v", vd.zkPath)
			if vd.err != nil {
				err = fmt.Errorf("some validation errors - see log")
				relog.Error("%v: %v", vd.zkPath, vd.err)
			}
		case <-timer.C:
			err = fmt.Errorf("timed out during validate")
			break wait
		case <-done:
			// To prevent a false positive, once we are 'done',
			// drain the result channel completely.
			for {
				select {
				case vd := <-results:
					relog.Info("checking %v", vd.zkPath)
					if vd.err != nil {
						err = fmt.Errorf("some validation errors - see log")
						relog.Error("%v: %v", vd.zkPath, vd.err)
					}
				default:
					break wait
				}
			}
		}
	}

	return err
}

// Validate all tablets in all discoverable cells, even if they are
// not in the replication graph.
func (wr *Wrangler) validateAllTablets(zkKeyspacesPath string, wg *sync.WaitGroup, results chan<- vresult) {
	replicationPaths, err := zk.ChildrenRecursive(wr.zconn, zkKeyspacesPath)
	if err != nil {
		results <- vresult{zkKeyspacesPath, err}
		return
	}

	cellSet := make(map[string]bool, 16)
	for _, p := range replicationPaths {
		p := path.Join(zkKeyspacesPath, p)
		if tm.IsTabletReplicationPath(p) {
			cell, _ := tm.ParseTabletReplicationPath(p)
			cellSet[cell] = true
		}
	}

	for cell, _ := range cellSet {
		zkTabletsPath := path.Join("/zk", cell, tm.VtSubtree(zkKeyspacesPath), "tablets")
		tabletUids, _, err := wr.zconn.Children(zkTabletsPath)
		if err != nil {
			results <- vresult{zkTabletsPath, err}
		} else {
			for _, tabletUid := range tabletUids {
				tabletPath := path.Join(zkTabletsPath, tabletUid)
				wg.Add(1)
				go func() {
					results <- vresult{tabletPath, tm.Validate(wr.zconn, tabletPath, "")}
					wg.Done()
				}()
			}
		}
	}
}

func (wr *Wrangler) validateKeyspace(zkKeyspacePath string, pingTablets bool, wg *sync.WaitGroup, results chan<- vresult) {
	// Validate replication graph by traversing each shard.
	zkShardsPath := path.Join(zkKeyspacePath, "shards")
	shards, _, err := wr.zconn.Children(zkShardsPath)
	if err != nil {
		results <- vresult{zkShardsPath, err}
	}
	for _, shard := range shards {
		zkShardPath := path.Join(zkShardsPath, shard)
		wg.Add(1)
		go func() {
			wr.validateShard(zkShardPath, pingTablets, wg, results)
			wg.Done()
		}()
	}
}

func (wr *Wrangler) validateShard(zkShardPath string, pingTablets bool, wg *sync.WaitGroup, results chan<- vresult) {
	shardInfo, err := tm.ReadShard(wr.zconn, zkShardPath)
	if err != nil {
		results <- vresult{zkShardPath, err}
		return
	}

	aliases, err := tm.FindAllTabletAliasesInShard(wr.zconn, zkShardPath)
	if err != nil {
		results <- vresult{zkShardPath, err}
	}

	shardTablets := make([]string, 0, 16)
	for _, alias := range aliases {
		shardTablets = append(shardTablets, tm.TabletPathForAlias(alias))
	}

	tabletMap, _ := GetTabletMap(wr.zconn, shardTablets)

	var masterAlias tm.TabletAlias
	for _, alias := range aliases {
		zkTabletPath := tm.TabletPathForAlias(alias)
		tabletInfo, ok := tabletMap[zkTabletPath]
		if !ok {
			results <- vresult{zkTabletPath, fmt.Errorf("tablet not found in map: %v", zkTabletPath)}
			continue
		}
		if tabletInfo.Parent.Uid == tm.NO_TABLET {
			if masterAlias.Cell != "" {
				results <- vresult{zkTabletPath, fmt.Errorf("%v: already has a master %v", zkTabletPath, masterAlias)}
			} else {
				masterAlias = alias
			}
		}
	}

	if masterAlias.Cell == "" {
		results <- vresult{zkShardPath, fmt.Errorf("no master for shard %v", zkShardPath)}
	} else if shardInfo.MasterAlias != masterAlias {
		results <- vresult{zkShardPath, fmt.Errorf("master mismatch for shard %v: found %v, expected %v", zkShardPath, masterAlias, shardInfo.MasterAlias)}
	}

	for _, alias := range aliases {
		zkTabletPath := tm.TabletPathForAlias(alias)
		zkTabletReplicationPath := path.Join(zkShardPath, masterAlias.String())
		if alias != masterAlias {
			zkTabletReplicationPath += "/" + alias.String()
		}
		wg.Add(1)
		go func() {
			results <- vresult{zkTabletReplicationPath, tm.Validate(wr.zconn, zkTabletPath, zkTabletReplicationPath)}
			wg.Done()
		}()
	}

	if pingTablets {
		wr.validateReplication(shardInfo, tabletMap, results)
		wr.pingTablets(tabletMap, wg, results)
	}

	return
}

func strInList(sl []string, s string) bool {
	for _, x := range sl {
		if x == s {
			return true
		}
	}
	return false
}

func (wr *Wrangler) validateReplication(shardInfo *tm.ShardInfo, tabletMap map[string]*tm.TabletInfo, results chan<- vresult) {
	masterTabletPath := tm.TabletPathForAlias(shardInfo.MasterAlias)
	masterTablet, ok := tabletMap[masterTabletPath]
	if !ok {
		err := fmt.Errorf("master not in tablet map: %v", masterTabletPath)
		results <- vresult{masterTabletPath, err}
		return
	}

	slaveAddrs, err := getSlaves(masterTablet.Tablet)
	if err != nil {
		results <- vresult{masterTabletPath, err}
		return
	}

	if len(slaveAddrs) == 0 {
		results <- vresult{masterTabletPath, fmt.Errorf("no slaves found: %v", masterTabletPath)}
		return
	}

	// Some addresses don't resolve in all locations, just use IP address
	if err != nil {
		results <- vresult{masterTabletPath, fmt.Errorf("resolve slaves failed: %v", err)}
		return
	}

	tabletIpMap := make(map[string]*tm.Tablet)
	for tabletPath, tablet := range tabletMap {
		ipAddr, _, err := net.SplitHostPort(tablet.MysqlIpAddr)
		if err != nil {
			results <- vresult{tabletPath, fmt.Errorf("bad mysql addr: %v %v %v", tablet.MysqlIpAddr, tabletPath, err)}
			continue
		}
		tabletIpMap[ipAddr] = tablet.Tablet
	}

	// See if every slave is in the replication graph.
	for _, slaveAddr := range slaveAddrs {
		if tabletIpMap[slaveAddr] == nil {
			results <- vresult{shardInfo.ShardPath(), fmt.Errorf("slave not in replication graph: %v (mysql instance without vttablet?)", slaveAddr)}
		}
	}

	// See if every entry in the replication graph is connected to the master.
	for tabletPath, tablet := range tabletMap {
		if !tablet.IsSlaveType() {
			continue
		}

		ipAddr, _, err := net.SplitHostPort(tablet.MysqlIpAddr)
		if err != nil {
			results <- vresult{tabletPath, fmt.Errorf("bad mysql addr: %v %v", tabletPath, err)}
		} else if !strInList(slaveAddrs, ipAddr) {
			results <- vresult{tabletPath, fmt.Errorf("slave not replicating: %v %v %q", tabletPath, ipAddr, slaveAddrs)}
		}
	}
}

func (wr *Wrangler) pingTablets(tabletMap map[string]*tm.TabletInfo, wg *sync.WaitGroup, results chan<- vresult) {
	for zkTabletPath, tabletInfo := range tabletMap {
		wg.Add(1)
		go func(zkTabletPath string, tabletInfo *tm.TabletInfo) {
			defer wg.Done()

			zkTabletPid := path.Join(tabletInfo.Path(), "pid")
			_, _, err := wr.zconn.Get(zkTabletPid)
			if err != nil {
				results <- vresult{zkTabletPath, fmt.Errorf("no pid node %v: %v %v", zkTabletPid, err, tabletInfo.Hostname())}
				return
			}

			actionPath, err := wr.ai.Ping(zkTabletPath)
			if err != nil {
				results <- vresult{zkTabletPath, fmt.Errorf("%v: %v %v", actionPath, err, tabletInfo.Hostname())}
				return
			}

			err = wr.ai.WaitForCompletion(actionPath, wr.actionTimeout())
			if err != nil {
				results <- vresult{zkTabletPath, fmt.Errorf("%v: %v %v", actionPath, err, tabletInfo.Hostname())}
			}
		}(zkTabletPath, tabletInfo)
	}
}

// Validate a whole zk tree
func (wr *Wrangler) Validate(zkKeyspacesPath string, pingTablets bool) error {
	// Results from various actions feed here.
	results := make(chan vresult, 16)
	wg := &sync.WaitGroup{}

	// Validate all tablets in all cells, even if they are not discoverable
	// by the replication graph.
	wg.Add(1)
	go func() {
		wr.validateAllTablets(zkKeyspacesPath, wg, results)
		wg.Done()
	}()

	// Validate replication graph by traversing each keyspace and then each shard.
	keyspaces, _, err := wr.zconn.Children(zkKeyspacesPath)
	if err != nil {
		results <- vresult{zkKeyspacesPath, err}
	} else {
		for _, keyspace := range keyspaces {
			zkKeyspace := path.Join(zkKeyspacesPath, keyspace)
			wg.Add(1)
			go func() {
				wr.validateKeyspace(zkKeyspace, pingTablets, wg, results)
				wg.Done()
			}()
		}
	}
	return wr.waitForResults(wg, results)
}

func (wr *Wrangler) ValidateKeyspace(zkKeyspacePath string, pingTablets bool) error {
	wg := &sync.WaitGroup{}
	results := make(chan vresult, 16)
	wg.Add(1)
	go func() {
		wr.validateKeyspace(zkKeyspacePath, pingTablets, wg, results)
		wg.Done()
	}()
	return wr.waitForResults(wg, results)
}

func (wr *Wrangler) ValidateShard(zkShardPath string, pingTablets bool) error {
	wg := &sync.WaitGroup{}
	results := make(chan vresult, 16)
	wg.Add(1)
	go func() {
		wr.validateShard(zkShardPath, pingTablets, wg, results)
		wg.Done()
	}()
	return wr.waitForResults(wg, results)
}

// Slaves come back as IP addrs, resolve to host names.
func resolveSlaveNames(addrs []string) (hostnames []string, err error) {
	hostnames = make([]string, len(addrs))
	for i, addr := range addrs {
		if net.ParseIP(addr) != nil {
			ipAddrs, err := net.LookupAddr(addr)
			if err != nil {
				return nil, err
			}
			addr = ipAddrs[0]
		}
		cname, err := net.LookupCNAME(addr)
		if err != nil {
			return nil, err
		}
		hostnames[i] = strings.TrimRight(cname, ".")
	}
	return
}

// Get list of slave ip addresses from the tablet.
// FIXME(alainjobart) this should not use RPC, but an actionNode
// FIXME(alainjobart) once this uses an actionNode, remove TabletManager RPC
func getSlaves(tablet *tm.Tablet) ([]string, error) {
	timer := time.NewTimer(10 * time.Second)
	defer timer.Stop()

	callChan := make(chan *rpc.Call, 1)

	go func() {
		client, clientErr := rpc.DialHTTP("tcp", tablet.Addr)
		if clientErr != nil {
			callChan <- &rpc.Call{Error: fmt.Errorf("dial failed: %v", clientErr)}
			return
		}

		slaveList := new(tm.SlaveList)
		// Forward the message so we close the connection after the rpc is done.
		done := make(chan *rpc.Call, 1)
		client.Go("TabletManager.GetSlaves", vtrpc.NilRequest, slaveList, done)
		callChan <- <-done
		client.Close()
	}()

	select {
	case <-timer.C:
		return nil, fmt.Errorf("TabletManager.GetSlaves deadline exceeded %v", tablet.Addr)
	case call := <-callChan:
		if call.Error != nil {
			return nil, call.Error
		}
		return call.Reply.(*tm.SlaveList).Addrs, nil
	}

	panic("unreachable")
}
<|endoftext|>