<commit_before>// Copyright © 2017 Alexander Sosna <alexander@xxor.de>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package storage

import (
	"fmt"
	"io"
	"sync"

	"github.com/spf13/viper"
	"github.com/xxorde/pgglaskugel/backup"
	"github.com/xxorde/pgglaskugel/storage/backends/awsS3"
	"github.com/xxorde/pgglaskugel/storage/backends/local"
	"github.com/xxorde/pgglaskugel/storage/backends/minioCs"
	"github.com/xxorde/pgglaskugel/storage/backends/minios3"

	log "github.com/Sirupsen/logrus"
)

var (
	// Definition in function below
	backends map[string]Backend
)

/*
 Storage Interface "Backend"" functions below
*/

// GetMyBackups does something
func GetMyBackups(viper *viper.Viper, subDirWal string) (backups backup.Backups) {
	bn := viper.GetString("backup_to")
	return backends[bn].GetBackups(viper, subDirWal)
}

// GetWals returns all Wal-Files for a Backup
func GetWals(viper *viper.Viper) (archive backup.Archive, err error) {
	bn := viper.GetString("backup_to")
	return backends[bn].GetWals(viper)
}

// WriteStream writes the stream to the configured archive_to
func WriteStream(viper *viper.Viper, input *io.Reader, name string, backuptype string) {
	bn := viper.GetString("backup_to")
	backends[bn].WriteStream(viper, input, name, backuptype)
}

// Fetch fetches
func Fetch(viper *viper.Viper) error {
	bn := viper.GetString("backup_to")
	return backends[bn].Fetch(viper)
}

// GetBasebackup gets basebackups
func GetBasebackup(viper *viper.Viper, bp *backup.Backup, backupStream *io.Reader, wgStart *sync.WaitGroup, wgDone *sync.WaitGroup) {
	bn := viper.GetString("backup_to")
	backends[bn].GetBasebackup(viper, bp, backupStream, wgStart, wgDone)
}

// DeleteAll deletes all backups in the struct
func DeleteAll(viper *viper.Viper, backups *backup.Backups) (count int, err error) {
	bn := viper.GetString("backup_to")
	return backends[bn].DeleteAll(backups)
}

// GetStartWalLocation returns the oldest needed WAL file
// Every older WAL file is not required to use this backup
func GetStartWalLocation(viper *viper.Viper, bp *backup.Backup) (startWalLocation string, err error) {
	bn := viper.GetString("backup_to")
	return backends[bn].GetStartWalLocation(bp)
}

// DeleteWal deletes the given WAL-file
func DeleteWal(viper *viper.Viper, w *backup.Wal) (err error) {
	bn := viper.GetString("backup_to")
	return backends[bn].DeleteWal(viper, w)
}

/*
	Not Interface functions below
*/

func init() {
	initbackends()
}

func initbackends() map[string]Backend {
	backends := make(map[string]Backend)

	var awsS3 awsS3.S3backend
	var minios3 minios3.S3backend
	var minioCs minioCs.S3backend
	var localb local.Localbackend
	backends["awss3"] = awsS3
	backends["minios3"] = minios3
	backends["minioCs"] = minioCs
	backends["file"] = localb
	return backends
}

// CheckBackend checks if the configured backend is supported
func CheckBackend(backend string) error {
	if _, ok := backends[backend]; ok {
		return nil
	}
	return fmt.Errorf("Backend %s not supported", backend)
}

// TODO Maybe we can move the function below to backup/wal.go. actually there is an import-circle

// DeleteOldWal deletes all WAL files that are older than lastWalToKeep
func DeleteOldWal(viper *viper.Viper, a *backup.Archive, lastWalToKeep backup.Wal) (deleted int) {
	// WAL files are deleted sequential
	// Due to the file system architecture parallel delete
	// Maybe this can be done in parallel for other storage systems
	visited := 0
	for _, wal := range a.WalFiles {
		// Count up
		visited++

		// Check if current visited WAL is older than lastWalToKeep
		old, err := wal.OlderThan(lastWalToKeep)
		if err != nil {
			log.Warn(err)
			continue
		}

		// If it is older, delete it
		if old {
			log.Debugf("Older than %s => going to delete: %s", lastWalToKeep.Name, wal.Name)
			err := DeleteWal(viper, &wal)
			if err != nil {
				log.Warn(err)
				continue
			}
			deleted++
		}
	}
	log.Debugf("Checked %d files and deleted %d", visited, deleted)
	return deleted
}
<commit_msg>fix backends function<commit_after>// Copyright © 2017 Alexander Sosna <alexander@xxor.de>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package storage

import (
	"fmt"
	"io"
	"sync"

	"github.com/spf13/viper"
	"github.com/xxorde/pgglaskugel/backup"
	"github.com/xxorde/pgglaskugel/storage/backends/awsS3"
	"github.com/xxorde/pgglaskugel/storage/backends/local"
	"github.com/xxorde/pgglaskugel/storage/backends/minioCs"
	"github.com/xxorde/pgglaskugel/storage/backends/minios3"

	log "github.com/Sirupsen/logrus"
)

var (
	// Definition in function below
	backends map[string]Backend
)

/*
 Storage Interface "Backend"" functions below
*/

// GetMyBackups does something
func GetMyBackups(viper *viper.Viper, subDirWal string) (backups backup.Backups) {
	bn := viper.GetString("backup_to")
	return backends[bn].GetBackups(viper, subDirWal)
}

// GetWals returns all Wal-Files for a Backup
func GetWals(viper *viper.Viper) (archive backup.Archive, err error) {
	bn := viper.GetString("backup_to")
	return backends[bn].GetWals(viper)
}

// WriteStream writes the stream to the configured archive_to
func WriteStream(viper *viper.Viper, input *io.Reader, name string, backuptype string) {
	bn := viper.GetString("backup_to")
	backends[bn].WriteStream(viper, input, name, backuptype)
}

// Fetch fetches
func Fetch(viper *viper.Viper) error {
	bn := viper.GetString("backup_to")
	return backends[bn].Fetch(viper)
}

// GetBasebackup gets basebackups
func GetBasebackup(viper *viper.Viper, bp *backup.Backup, backupStream *io.Reader, wgStart *sync.WaitGroup, wgDone *sync.WaitGroup) {
	bn := viper.GetString("backup_to")
	backends[bn].GetBasebackup(viper, bp, backupStream, wgStart, wgDone)
}

// DeleteAll deletes all backups in the struct
func DeleteAll(viper *viper.Viper, backups *backup.Backups) (count int, err error) {
	bn := viper.GetString("backup_to")
	return backends[bn].DeleteAll(backups)
}

// GetStartWalLocation returns the oldest needed WAL file
// Every older WAL file is not required to use this backup
func GetStartWalLocation(viper *viper.Viper, bp *backup.Backup) (startWalLocation string, err error) {
	bn := viper.GetString("backup_to")
	return backends[bn].GetStartWalLocation(bp)
}

// DeleteWal deletes the given WAL-file
func DeleteWal(viper *viper.Viper, w *backup.Wal) (err error) {
	bn := viper.GetString("backup_to")
	return backends[bn].DeleteWal(viper, w)
}

/*
	Not Interface functions below
*/

func init() {
	backends = initbackends()
}

func initbackends() map[string]Backend {
	fbackends := make(map[string]Backend)

	var awsS3 awsS3.S3backend
	var minios3 minios3.S3backend
	var minioCs minioCs.S3backend
	var localb local.Localbackend
	fbackends["awss3"] = awsS3
	fbackends["minios3"] = minios3
	fbackends["minioCs"] = minioCs
	fbackends["file"] = localb
	return fbackends
}

// CheckBackend checks if the configured backend is supported
func CheckBackend(backend string) error {
	if _, ok := backends[backend]; ok {
		return nil
	}
	return fmt.Errorf("Backend %s not supported", backend)
}

// TODO Maybe we can move the function below to backup/wal.go. actually there is an import-circle

// DeleteOldWal deletes all WAL files that are older than lastWalToKeep
func DeleteOldWal(viper *viper.Viper, a *backup.Archive, lastWalToKeep backup.Wal) (deleted int) {
	// WAL files are deleted sequential
	// Due to the file system architecture parallel delete
	// Maybe this can be done in parallel for other storage systems
	visited := 0
	for _, wal := range a.WalFiles {
		// Count up
		visited++

		// Check if current visited WAL is older than lastWalToKeep
		old, err := wal.OlderThan(lastWalToKeep)
		if err != nil {
			log.Warn(err)
			continue
		}

		// If it is older, delete it
		if old {
			log.Debugf("Older than %s => going to delete: %s", lastWalToKeep.Name, wal.Name)
			err := DeleteWal(viper, &wal)
			if err != nil {
				log.Warn(err)
				continue
			}
			deleted++
		}
	}
	log.Debugf("Checked %d files and deleted %d", visited, deleted)
	return deleted
}
<|endoftext|>


<commit_before>//auto-generated by difficulty-convert.py DO NOT EDIT

package sudoku

func init() {
	difficultySignalWeights = map[string]float64{
		"Block Block Interactions Count" : -0.0225,
		"Block Block Interactions Percentage" : 0.0012,
		"Constant" : 0.1076,
		"Forcing Chain Count" : 0.1286,
		"Forcing Chain Percentage" : 0.0072,
		"Guess Count" : -0.111,
		"Guess Percentage" : -0.0016,
		"Hidden Pair Block Count" : -0.0309,
		"Hidden Pair Block Percentage" : -0.003,
		"Hidden Pair Col Count" : -0.041,
		"Hidden Pair Col Percentage" : 0.0036,
		"Hidden Pair Row Count" : -0.0648,
		"Hidden Pair Row Percentage" : 0.0043,
		"Hidden Quad Block Count" : 0.219,
		"Hidden Quad Block Percentage" : 0.0036,
		"Hidden Quad Col Count" : 0.0229,
		"Hidden Quad Col Percentage" : 0.0003,
		"Hidden Quad Row Count" : 0.3,
		"Hidden Quad Row Percentage" : 0.0047,
		"Hidden Triple Block Count" : 0.0376,
		"Hidden Triple Block Percentage" : 0.0009,
		"Hidden Triple Col Count" : 0.0444,
		"Hidden Triple Col Percentage" : 0.0011,
		"Hidden Triple Row Count" : 0.1498,
		"Hidden Triple Row Percentage" : 0.0027,
		"Naked Pair Block Count" : -0.0645,
		"Naked Pair Block Percentage" : 0.0107,
		"Naked Pair Col Count" : -0.0609,
		"Naked Pair Col Percentage" : 0.0064,
		"Naked Pair Row Count" : -0.0651,
		"Naked Pair Row Percentage" : -0.0016,
		"Naked Quad Block Count" : -0.0315,
		"Naked Quad Block Percentage" : 0.0109,
		"Naked Quad Col Count" : -0.0516,
		"Naked Quad Col Percentage" : 0.0048,
		"Naked Quad Row Count" : 0.0076,
		"Naked Quad Row Percentage" : -0.0001,
		"Naked Triple Block Count" : -0.0825,
		"Naked Triple Block Percentage" : 0.0202,
		"Naked Triple Col Count" : -0.0362,
		"Naked Triple Col Percentage" : 0.0131,
		"Naked Triple Row Count" : -0.0273,
		"Naked Triple Row Percentage" : 0.0038,
		"Necessary In Block Count" : -0.0243,
		"Necessary In Block Percentage" : 0.0686,
		"Necessary In Col Count" : -0.0007,
		"Necessary In Col Percentage" : -0.0586,
		"Necessary In Row Count" : -0.0021,
		"Necessary In Row Percentage" : 0.0237,
		"Number Unfilled Cells" : -0.0394,
		"Number of Steps" : 0.0585,
		"Obvious In Block Count" : -0.0205,
		"Obvious In Block Percentage" : 0.028,
		"Obvious In Col Count" : 0.0058,
		"Obvious In Col Percentage" : -0.0782,
		"Obvious In Row Count" : -0.007,
		"Obvious In Row Percentage" : 0.0469,
		"Only Legal Number Count" : -0.0084,
		"Only Legal Number Percentage" : -0.1435,
		"Percentage Fill Steps" : -0.1074,
		"Pointing Pair Col Count" : 0.0183,
		"Pointing Pair Col Percentage" : 0.0097,
		"Pointing Pair Row Count" : -0.0297,
		"Pointing Pair Row Percentage" : 0.0034,
		"Steps Until Nonfill" : -0.0089,
		"XWing Col Count" : -0.0442,
		"XWing Col Percentage" : 0.0003,
		"XWing Row Count" : -0.0489,
		"XWing Row Percentage" : 0.0065,
	}
}
<commit_msg>New weights trained with new solve data. 0.7581, including forcing chains. CF 0.766 without forcing chains on same relativedifficulties<commit_after>//auto-generated by difficulty-convert.py DO NOT EDIT

package sudoku

func init() {
	difficultySignalWeights = map[string]float64{
		"Block Block Interactions Count" : 0.0542,
		"Block Block Interactions Percentage" : 0.0024,
		"Constant" : 0.1622,
		"Forcing Chain Count" : 0.106,
		"Forcing Chain Percentage" : 0.0078,
		"Guess Count" : -0.0031,
		"Guess Percentage" : -0.0,
		"Hidden Pair Block Count" : 0.0091,
		"Hidden Pair Block Percentage" : 0.0011,
		"Hidden Pair Col Count" : -0.0167,
		"Hidden Pair Col Percentage" : 0.0023,
		"Hidden Pair Row Count" : -0.097,
		"Hidden Pair Row Percentage" : -0.0032,
		"Hidden Quad Block Count" : 0.1883,
		"Hidden Quad Block Percentage" : 0.0031,
		"Hidden Quad Col Count" : 0.2229,
		"Hidden Quad Col Percentage" : 0.0031,
		"Hidden Quad Row Count" : -0.253,
		"Hidden Quad Row Percentage" : -0.0039,
		"Hidden Triple Block Count" : -0.0255,
		"Hidden Triple Block Percentage" : 0.0019,
		"Hidden Triple Col Count" : 0.0238,
		"Hidden Triple Col Percentage" : 0.0018,
		"Hidden Triple Row Count" : -0.0344,
		"Hidden Triple Row Percentage" : -0.0013,
		"Naked Pair Block Count" : -0.0515,
		"Naked Pair Block Percentage" : -0.0051,
		"Naked Pair Col Count" : -0.0459,
		"Naked Pair Col Percentage" : 0.0065,
		"Naked Pair Row Count" : -0.0341,
		"Naked Pair Row Percentage" : 0.0426,
		"Naked Quad Block Count" : -0.0273,
		"Naked Quad Block Percentage" : 0.0078,
		"Naked Quad Col Count" : -0.0143,
		"Naked Quad Col Percentage" : 0.0045,
		"Naked Quad Row Count" : 0.0267,
		"Naked Quad Row Percentage" : 0.0019,
		"Naked Triple Block Count" : -0.0471,
		"Naked Triple Block Percentage" : 0.0273,
		"Naked Triple Col Count" : -0.0094,
		"Naked Triple Col Percentage" : 0.0212,
		"Naked Triple Row Count" : 0.0186,
		"Naked Triple Row Percentage" : 0.0107,
		"Necessary In Block Count" : -0.0267,
		"Necessary In Block Percentage" : -0.0516,
		"Necessary In Col Count" : 0.002,
		"Necessary In Col Percentage" : -0.2068,
		"Necessary In Row Count" : -0.0129,
		"Necessary In Row Percentage" : 0.1994,
		"Number Unfilled Cells" : -0.0083,
		"Number of Steps" : 0.031,
		"Obvious In Block Count" : -0.0496,
		"Obvious In Block Percentage" : 0.0792,
		"Obvious In Col Count" : -0.0075,
		"Obvious In Col Percentage" : 0.0124,
		"Obvious In Row Count" : -0.0164,
		"Obvious In Row Percentage" : 0.0452,
		"Only Legal Number Count" : -0.0001,
		"Only Legal Number Percentage" : -0.2399,
		"Percentage Fill Steps" : -0.1544,
		"Pointing Pair Col Count" : 0.1042,
		"Pointing Pair Col Percentage" : 0.0141,
		"Pointing Pair Row Count" : 0.0026,
		"Pointing Pair Row Percentage" : 0.0034,
		"Steps Until Nonfill" : -0.0054,
		"XWing Col Count" : 0.0453,
		"XWing Col Percentage" : 0.0052,
		"XWing Row Count" : -0.0002,
		"XWing Row Percentage" : 0.0069,
	}
}
<|endoftext|>


<commit_before>// Package strings2 is the supplement of the standard library of strings.
package strings2

import (
	"fmt"
	"strings"
)

var (
	leftDelimiter  = "{"
	rightDelimiter = "}"
)

// SetFmtDelimiter sets the delimiters which are used by KvFmt.
//
// The left delimiter is "{", and the right delimiter is "}".
func SetFmtDelimiter(left, right string) {
	if left == "" || right == "" {
		panic("The arguments cannot be empty")
	}
	leftDelimiter = left
	rightDelimiter = right
}

// KvFmt formats the string like the key-value method format of str in Python,
// which the placeholder is appointed by the key name of the values.
//
// Notice: the formatter will use %v to convert the value of the key to string.
// The delimiters are "{" and "}" by default, and you can reset them by the
// function SetFmtDelimiter.
func KvFmt(s string, values map[string]interface{}) string {
	for key, value := range values {
		key = fmt.Sprintf("%s%s%s", leftDelimiter, key, rightDelimiter)
		s = strings.Replace(s, key, fmt.Sprintf("%v", value), -1)
	}
	return s
}
<commit_msg>deprecate string2<commit_after>// Package strings2 is the supplement of the standard library of strings.
//
// DEPRECATED!!!
package strings2

import (
	"fmt"
	"strings"
)

var (
	leftDelimiter  = "{"
	rightDelimiter = "}"
)

// SetFmtDelimiter sets the delimiters which are used by KvFmt.
//
// The left delimiter is "{", and the right delimiter is "}".
func SetFmtDelimiter(left, right string) {
	if left == "" || right == "" {
		panic("The arguments cannot be empty")
	}
	leftDelimiter = left
	rightDelimiter = right
}

// KvFmt formats the string like the key-value method format of str in Python,
// which the placeholder is appointed by the key name of the values.
//
// Notice: the formatter will use %v to convert the value of the key to string.
// The delimiters are "{" and "}" by default, and you can reset them by the
// function SetFmtDelimiter.
func KvFmt(s string, values map[string]interface{}) string {
	for key, value := range values {
		key = fmt.Sprintf("%s%s%s", leftDelimiter, key, rightDelimiter)
		s = strings.Replace(s, key, fmt.Sprintf("%v", value), -1)
	}
	return s
}
<|endoftext|>


<commit_before>package scanner

import (
	"crypto/sha512"
	"errors"
	"github.com/Symantec/Dominator/lib/filesystem"
	"github.com/Symantec/Dominator/lib/objectcache"
	"io"
	"os"
	"path"
	"runtime"
	"sort"
	"syscall"
)

var myCountGC int

func myGC() {
	if myCountGC > 1000 {
		runtime.GC()
		myCountGC = 0
	}
	myCountGC++
}

func makeRegularInode(stat *syscall.Stat_t) *filesystem.RegularInode {
	var inode filesystem.RegularInode
	inode.Mode = filesystem.FileMode(stat.Mode)
	inode.Uid = stat.Uid
	inode.Gid = stat.Gid
	inode.MtimeSeconds = stat.Mtim.Sec
	inode.MtimeNanoSeconds = int32(stat.Mtim.Nsec)
	inode.Size = uint64(stat.Size)
	return &inode
}

func makeSymlinkInode(stat *syscall.Stat_t) *filesystem.SymlinkInode {
	var inode filesystem.SymlinkInode
	inode.Uid = stat.Uid
	inode.Gid = stat.Gid
	return &inode
}

func makeSpecialInode(stat *syscall.Stat_t) *filesystem.SpecialInode {
	var inode filesystem.SpecialInode
	inode.Mode = filesystem.FileMode(stat.Mode)
	inode.Uid = stat.Uid
	inode.Gid = stat.Gid
	inode.MtimeSeconds = stat.Mtim.Sec
	inode.MtimeNanoSeconds = int32(stat.Mtim.Nsec)
	inode.Rdev = stat.Rdev
	return &inode
}

func scanFileSystem(rootDirectoryName string, cacheDirectoryName string,
	configuration *Configuration, oldFS *FileSystem) (*FileSystem, error) {
	var fileSystem FileSystem
	fileSystem.configuration = configuration
	fileSystem.rootDirectoryName = rootDirectoryName
	fileSystem.cacheDirectoryName = cacheDirectoryName
	var stat syscall.Stat_t
	if err := syscall.Lstat(rootDirectoryName, &stat); err != nil {
		return nil, err
	}
	fileSystem.InodeTable = make(filesystem.InodeTable)
	fileSystem.dev = stat.Dev
	fileSystem.inodeNumber = stat.Ino
	fileSystem.Mode = filesystem.FileMode(stat.Mode)
	fileSystem.Uid = stat.Uid
	fileSystem.Gid = stat.Gid
	fileSystem.DirectoryCount++
	var tmpInode filesystem.RegularInode
	if sha512.New().Size() != len(tmpInode.Hash) {
		return nil, errors.New("Incompatible hash size")
	}
	var oldDirectory *filesystem.DirectoryInode
	if oldFS != nil && oldFS.InodeTable != nil {
		oldDirectory = &oldFS.DirectoryInode
	}
	err, _ := scanDirectory(&fileSystem.FileSystem.DirectoryInode, oldDirectory,
		&fileSystem, oldFS, "/")
	oldFS = nil
	oldDirectory = nil
	if err != nil {
		return nil, err
	}
	if err = fileSystem.scanObjectCache(); err != nil {
		return nil, err
	}
	fileSystem.ComputeTotalDataBytes()
	if err = fileSystem.RebuildInodePointers(); err != nil {
		panic(err)
	}
	return &fileSystem, nil
}

func (fs *FileSystem) scanObjectCache() error {
	if fs.cacheDirectoryName == "" {
		return nil
	}
	var err error
	fs.ObjectCache, err = objectcache.ScanObjectCache(fs.cacheDirectoryName)
	return err
}

func scanDirectory(directory, oldDirectory *filesystem.DirectoryInode,
	fileSystem, oldFS *FileSystem, myPathName string) (error, bool) {
	file, err := os.Open(path.Join(fileSystem.rootDirectoryName, myPathName))
	if err != nil {
		return err, false
	}
	names, err := file.Readdirnames(-1)
	file.Close()
	if err != nil {
		return err, false
	}
	sort.Strings(names)
	entryList := make([]*filesystem.DirectoryEntry, 0, len(names))
	var copiedDirents int
	for _, name := range names {
		if directory == &fileSystem.DirectoryInode && name == ".subd" {
			continue
		}
		filename := path.Join(myPathName, name)
		if fileSystem.configuration.ScanFilter.Match(filename) {
			continue
		}
		var stat syscall.Stat_t
		err := syscall.Lstat(path.Join(fileSystem.rootDirectoryName, filename),
			&stat)
		if err != nil {
			if err == syscall.ENOENT {
				continue
			}
			return err, false
		}
		if stat.Dev != fileSystem.dev {
			continue
		}
		if checkScanDisableRequest() {
			return errors.New("DisableScan"), false
		}
		myGC()
		dirent := new(filesystem.DirectoryEntry)
		dirent.Name = name
		dirent.InodeNumber = stat.Ino
		var oldDirent *filesystem.DirectoryEntry
		if oldDirectory != nil {
			index := len(entryList)
			if len(oldDirectory.EntryList) > index &&
				oldDirectory.EntryList[index].Name == name {
				oldDirent = oldDirectory.EntryList[index]
			}
		}
		if stat.Mode&syscall.S_IFMT == syscall.S_IFDIR {
			err = addDirectory(dirent, oldDirent, fileSystem, oldFS, myPathName,
				&stat)
		} else if stat.Mode&syscall.S_IFMT == syscall.S_IFREG {
			err = addRegularFile(dirent, fileSystem, oldFS, myPathName, &stat)
		} else if stat.Mode&syscall.S_IFMT == syscall.S_IFLNK {
			err = addSymlink(dirent, fileSystem, oldFS, myPathName, &stat)
		} else if stat.Mode&syscall.S_IFMT == syscall.S_IFSOCK {
			continue
		} else {
			err = addSpecialFile(dirent, fileSystem, oldFS, &stat)
		}
		if err != nil {
			if err == syscall.ENOENT {
				continue
			}
			return err, false
		}
		if oldDirent != nil && *dirent == *oldDirent {
			dirent = oldDirent
			copiedDirents++
		}
		entryList = append(entryList, dirent)
	}
	if oldDirectory != nil && len(entryList) == copiedDirents &&
		len(entryList) == len(oldDirectory.EntryList) {
		directory.EntryList = oldDirectory.EntryList
		return nil, true
	} else {
		directory.EntryList = entryList
		return nil, false
	}
}

func addDirectory(dirent, oldDirent *filesystem.DirectoryEntry,
	fileSystem, oldFS *FileSystem,
	directoryPathName string, stat *syscall.Stat_t) error {
	myPathName := path.Join(directoryPathName, dirent.Name)
	if stat.Ino == fileSystem.inodeNumber {
		return errors.New("Recursive directory: " + myPathName)
	}
	if _, ok := fileSystem.InodeTable[stat.Ino]; ok {
		return errors.New("Hardlinked directory: " + myPathName)
	}
	inode := new(filesystem.DirectoryInode)
	dirent.SetInode(inode)
	fileSystem.InodeTable[stat.Ino] = inode
	inode.Mode = filesystem.FileMode(stat.Mode)
	inode.Uid = stat.Uid
	inode.Gid = stat.Gid
	var oldInode *filesystem.DirectoryInode
	if oldDirent != nil {
		if oi, ok := oldDirent.Inode().(*filesystem.DirectoryInode); ok {
			oldInode = oi
		}
	}
	err, copied := scanDirectory(inode, oldInode, fileSystem, oldFS, myPathName)
	if err != nil {
		return err
	}
	if copied && filesystem.CompareDirectoriesMetadata(inode, oldInode, nil) {
		dirent.SetInode(oldInode)
		fileSystem.InodeTable[stat.Ino] = oldInode
	}
	fileSystem.DirectoryCount++
	return nil
}

func addRegularFile(dirent *filesystem.DirectoryEntry,
	fileSystem, oldFS *FileSystem,
	directoryPathName string, stat *syscall.Stat_t) error {
	if inode, ok := fileSystem.InodeTable[stat.Ino]; ok {
		if inode, ok := inode.(*filesystem.RegularInode); ok {
			dirent.SetInode(inode)
			return nil
		}
		return errors.New("Inode changed type: " + dirent.Name)
	}
	inode := makeRegularInode(stat)
	if inode.Size > 0 {
		err := scanRegularInode(inode, fileSystem,
			path.Join(directoryPathName, dirent.Name))
		if err != nil {
			return err
		}
	}
	if oldFS != nil && oldFS.InodeTable != nil {
		if oldInode, found := oldFS.InodeTable[stat.Ino]; found {
			if oldInode, ok := oldInode.(*filesystem.RegularInode); ok {
				if filesystem.CompareRegularInodes(inode, oldInode, nil) {
					inode = oldInode
				}
			}
		}
	}
	dirent.SetInode(inode)
	fileSystem.InodeTable[stat.Ino] = inode
	return nil
}

func addSymlink(dirent *filesystem.DirectoryEntry,
	fileSystem, oldFS *FileSystem,
	directoryPathName string, stat *syscall.Stat_t) error {
	if inode, ok := fileSystem.InodeTable[stat.Ino]; ok {
		if inode, ok := inode.(*filesystem.SymlinkInode); ok {
			dirent.SetInode(inode)
			return nil
		}
		return errors.New("Inode changed type: " + dirent.Name)
	}
	inode := makeSymlinkInode(stat)
	err := scanSymlinkInode(inode, fileSystem,
		path.Join(directoryPathName, dirent.Name))
	if err != nil {
		return err
	}
	if oldFS != nil && oldFS.InodeTable != nil {
		if oldInode, found := oldFS.InodeTable[stat.Ino]; found {
			if oldInode, ok := oldInode.(*filesystem.SymlinkInode); ok {
				if filesystem.CompareSymlinkInodes(inode, oldInode, nil) {
					inode = oldInode
				}
			}
		}
	}
	dirent.SetInode(inode)
	fileSystem.InodeTable[stat.Ino] = inode
	return nil
}

func addSpecialFile(dirent *filesystem.DirectoryEntry,
	fileSystem, oldFS *FileSystem, stat *syscall.Stat_t) error {
	if inode, ok := fileSystem.InodeTable[stat.Ino]; ok {
		if inode, ok := inode.(*filesystem.SpecialInode); ok {
			dirent.SetInode(inode)
			return nil
		}
		return errors.New("Inode changed type: " + dirent.Name)
	}
	inode := makeSpecialInode(stat)
	if oldFS != nil && oldFS.InodeTable != nil {
		if oldInode, found := oldFS.InodeTable[stat.Ino]; found {
			if oldInode, ok := oldInode.(*filesystem.SpecialInode); ok {
				if filesystem.CompareSpecialInodes(inode, oldInode, nil) {
					inode = oldInode
				}
			}
		}
	}
	dirent.SetInode(inode)
	fileSystem.InodeTable[stat.Ino] = inode
	return nil
}

func scanRegularInode(inode *filesystem.RegularInode, fileSystem *FileSystem,
	myPathName string) error {
	f, err := os.Open(path.Join(fileSystem.rootDirectoryName, myPathName))
	if err != nil {
		return err
	}
	reader := fileSystem.configuration.FsScanContext.NewReader(f)
	hash := sha512.New()
	io.Copy(hash, reader)
	f.Close()
	copy(inode.Hash[:], hash.Sum(nil))
	return nil
}

func scanSymlinkInode(inode *filesystem.SymlinkInode, fileSystem *FileSystem,
	myPathName string) error {
	target, err := os.Readlink(path.Join(fileSystem.rootDirectoryName,
		myPathName))
	if err != nil {
		return err
	}
	inode.Symlink = target
	return nil
}
<commit_msg>Add error checking to scanRegularInode().<commit_after>package scanner

import (
	"crypto/sha512"
	"errors"
	"fmt"
	"github.com/Symantec/Dominator/lib/filesystem"
	"github.com/Symantec/Dominator/lib/objectcache"
	"io"
	"os"
	"path"
	"runtime"
	"sort"
	"syscall"
)

var myCountGC int

func myGC() {
	if myCountGC > 1000 {
		runtime.GC()
		myCountGC = 0
	}
	myCountGC++
}

func makeRegularInode(stat *syscall.Stat_t) *filesystem.RegularInode {
	var inode filesystem.RegularInode
	inode.Mode = filesystem.FileMode(stat.Mode)
	inode.Uid = stat.Uid
	inode.Gid = stat.Gid
	inode.MtimeSeconds = stat.Mtim.Sec
	inode.MtimeNanoSeconds = int32(stat.Mtim.Nsec)
	inode.Size = uint64(stat.Size)
	return &inode
}

func makeSymlinkInode(stat *syscall.Stat_t) *filesystem.SymlinkInode {
	var inode filesystem.SymlinkInode
	inode.Uid = stat.Uid
	inode.Gid = stat.Gid
	return &inode
}

func makeSpecialInode(stat *syscall.Stat_t) *filesystem.SpecialInode {
	var inode filesystem.SpecialInode
	inode.Mode = filesystem.FileMode(stat.Mode)
	inode.Uid = stat.Uid
	inode.Gid = stat.Gid
	inode.MtimeSeconds = stat.Mtim.Sec
	inode.MtimeNanoSeconds = int32(stat.Mtim.Nsec)
	inode.Rdev = stat.Rdev
	return &inode
}

func scanFileSystem(rootDirectoryName string, cacheDirectoryName string,
	configuration *Configuration, oldFS *FileSystem) (*FileSystem, error) {
	var fileSystem FileSystem
	fileSystem.configuration = configuration
	fileSystem.rootDirectoryName = rootDirectoryName
	fileSystem.cacheDirectoryName = cacheDirectoryName
	var stat syscall.Stat_t
	if err := syscall.Lstat(rootDirectoryName, &stat); err != nil {
		return nil, err
	}
	fileSystem.InodeTable = make(filesystem.InodeTable)
	fileSystem.dev = stat.Dev
	fileSystem.inodeNumber = stat.Ino
	fileSystem.Mode = filesystem.FileMode(stat.Mode)
	fileSystem.Uid = stat.Uid
	fileSystem.Gid = stat.Gid
	fileSystem.DirectoryCount++
	var tmpInode filesystem.RegularInode
	if sha512.New().Size() != len(tmpInode.Hash) {
		return nil, errors.New("Incompatible hash size")
	}
	var oldDirectory *filesystem.DirectoryInode
	if oldFS != nil && oldFS.InodeTable != nil {
		oldDirectory = &oldFS.DirectoryInode
	}
	err, _ := scanDirectory(&fileSystem.FileSystem.DirectoryInode, oldDirectory,
		&fileSystem, oldFS, "/")
	oldFS = nil
	oldDirectory = nil
	if err != nil {
		return nil, err
	}
	if err = fileSystem.scanObjectCache(); err != nil {
		return nil, err
	}
	fileSystem.ComputeTotalDataBytes()
	if err = fileSystem.RebuildInodePointers(); err != nil {
		panic(err)
	}
	return &fileSystem, nil
}

func (fs *FileSystem) scanObjectCache() error {
	if fs.cacheDirectoryName == "" {
		return nil
	}
	var err error
	fs.ObjectCache, err = objectcache.ScanObjectCache(fs.cacheDirectoryName)
	return err
}

func scanDirectory(directory, oldDirectory *filesystem.DirectoryInode,
	fileSystem, oldFS *FileSystem, myPathName string) (error, bool) {
	file, err := os.Open(path.Join(fileSystem.rootDirectoryName, myPathName))
	if err != nil {
		return err, false
	}
	names, err := file.Readdirnames(-1)
	file.Close()
	if err != nil {
		return err, false
	}
	sort.Strings(names)
	entryList := make([]*filesystem.DirectoryEntry, 0, len(names))
	var copiedDirents int
	for _, name := range names {
		if directory == &fileSystem.DirectoryInode && name == ".subd" {
			continue
		}
		filename := path.Join(myPathName, name)
		if fileSystem.configuration.ScanFilter.Match(filename) {
			continue
		}
		var stat syscall.Stat_t
		err := syscall.Lstat(path.Join(fileSystem.rootDirectoryName, filename),
			&stat)
		if err != nil {
			if err == syscall.ENOENT {
				continue
			}
			return err, false
		}
		if stat.Dev != fileSystem.dev {
			continue
		}
		if checkScanDisableRequest() {
			return errors.New("DisableScan"), false
		}
		myGC()
		dirent := new(filesystem.DirectoryEntry)
		dirent.Name = name
		dirent.InodeNumber = stat.Ino
		var oldDirent *filesystem.DirectoryEntry
		if oldDirectory != nil {
			index := len(entryList)
			if len(oldDirectory.EntryList) > index &&
				oldDirectory.EntryList[index].Name == name {
				oldDirent = oldDirectory.EntryList[index]
			}
		}
		if stat.Mode&syscall.S_IFMT == syscall.S_IFDIR {
			err = addDirectory(dirent, oldDirent, fileSystem, oldFS, myPathName,
				&stat)
		} else if stat.Mode&syscall.S_IFMT == syscall.S_IFREG {
			err = addRegularFile(dirent, fileSystem, oldFS, myPathName, &stat)
		} else if stat.Mode&syscall.S_IFMT == syscall.S_IFLNK {
			err = addSymlink(dirent, fileSystem, oldFS, myPathName, &stat)
		} else if stat.Mode&syscall.S_IFMT == syscall.S_IFSOCK {
			continue
		} else {
			err = addSpecialFile(dirent, fileSystem, oldFS, &stat)
		}
		if err != nil {
			if err == syscall.ENOENT {
				continue
			}
			return err, false
		}
		if oldDirent != nil && *dirent == *oldDirent {
			dirent = oldDirent
			copiedDirents++
		}
		entryList = append(entryList, dirent)
	}
	if oldDirectory != nil && len(entryList) == copiedDirents &&
		len(entryList) == len(oldDirectory.EntryList) {
		directory.EntryList = oldDirectory.EntryList
		return nil, true
	} else {
		directory.EntryList = entryList
		return nil, false
	}
}

func addDirectory(dirent, oldDirent *filesystem.DirectoryEntry,
	fileSystem, oldFS *FileSystem,
	directoryPathName string, stat *syscall.Stat_t) error {
	myPathName := path.Join(directoryPathName, dirent.Name)
	if stat.Ino == fileSystem.inodeNumber {
		return errors.New("Recursive directory: " + myPathName)
	}
	if _, ok := fileSystem.InodeTable[stat.Ino]; ok {
		return errors.New("Hardlinked directory: " + myPathName)
	}
	inode := new(filesystem.DirectoryInode)
	dirent.SetInode(inode)
	fileSystem.InodeTable[stat.Ino] = inode
	inode.Mode = filesystem.FileMode(stat.Mode)
	inode.Uid = stat.Uid
	inode.Gid = stat.Gid
	var oldInode *filesystem.DirectoryInode
	if oldDirent != nil {
		if oi, ok := oldDirent.Inode().(*filesystem.DirectoryInode); ok {
			oldInode = oi
		}
	}
	err, copied := scanDirectory(inode, oldInode, fileSystem, oldFS, myPathName)
	if err != nil {
		return err
	}
	if copied && filesystem.CompareDirectoriesMetadata(inode, oldInode, nil) {
		dirent.SetInode(oldInode)
		fileSystem.InodeTable[stat.Ino] = oldInode
	}
	fileSystem.DirectoryCount++
	return nil
}

func addRegularFile(dirent *filesystem.DirectoryEntry,
	fileSystem, oldFS *FileSystem,
	directoryPathName string, stat *syscall.Stat_t) error {
	if inode, ok := fileSystem.InodeTable[stat.Ino]; ok {
		if inode, ok := inode.(*filesystem.RegularInode); ok {
			dirent.SetInode(inode)
			return nil
		}
		return errors.New("Inode changed type: " + dirent.Name)
	}
	inode := makeRegularInode(stat)
	if inode.Size > 0 {
		err := scanRegularInode(inode, fileSystem,
			path.Join(directoryPathName, dirent.Name))
		if err != nil {
			return err
		}
	}
	if oldFS != nil && oldFS.InodeTable != nil {
		if oldInode, found := oldFS.InodeTable[stat.Ino]; found {
			if oldInode, ok := oldInode.(*filesystem.RegularInode); ok {
				if filesystem.CompareRegularInodes(inode, oldInode, nil) {
					inode = oldInode
				}
			}
		}
	}
	dirent.SetInode(inode)
	fileSystem.InodeTable[stat.Ino] = inode
	return nil
}

func addSymlink(dirent *filesystem.DirectoryEntry,
	fileSystem, oldFS *FileSystem,
	directoryPathName string, stat *syscall.Stat_t) error {
	if inode, ok := fileSystem.InodeTable[stat.Ino]; ok {
		if inode, ok := inode.(*filesystem.SymlinkInode); ok {
			dirent.SetInode(inode)
			return nil
		}
		return errors.New("Inode changed type: " + dirent.Name)
	}
	inode := makeSymlinkInode(stat)
	err := scanSymlinkInode(inode, fileSystem,
		path.Join(directoryPathName, dirent.Name))
	if err != nil {
		return err
	}
	if oldFS != nil && oldFS.InodeTable != nil {
		if oldInode, found := oldFS.InodeTable[stat.Ino]; found {
			if oldInode, ok := oldInode.(*filesystem.SymlinkInode); ok {
				if filesystem.CompareSymlinkInodes(inode, oldInode, nil) {
					inode = oldInode
				}
			}
		}
	}
	dirent.SetInode(inode)
	fileSystem.InodeTable[stat.Ino] = inode
	return nil
}

func addSpecialFile(dirent *filesystem.DirectoryEntry,
	fileSystem, oldFS *FileSystem, stat *syscall.Stat_t) error {
	if inode, ok := fileSystem.InodeTable[stat.Ino]; ok {
		if inode, ok := inode.(*filesystem.SpecialInode); ok {
			dirent.SetInode(inode)
			return nil
		}
		return errors.New("Inode changed type: " + dirent.Name)
	}
	inode := makeSpecialInode(stat)
	if oldFS != nil && oldFS.InodeTable != nil {
		if oldInode, found := oldFS.InodeTable[stat.Ino]; found {
			if oldInode, ok := oldInode.(*filesystem.SpecialInode); ok {
				if filesystem.CompareSpecialInodes(inode, oldInode, nil) {
					inode = oldInode
				}
			}
		}
	}
	dirent.SetInode(inode)
	fileSystem.InodeTable[stat.Ino] = inode
	return nil
}

func scanRegularInode(inode *filesystem.RegularInode, fileSystem *FileSystem,
	myPathName string) error {
	f, err := os.Open(path.Join(fileSystem.rootDirectoryName, myPathName))
	if err != nil {
		return err
	}
	reader := fileSystem.configuration.FsScanContext.NewReader(f)
	hash := sha512.New()
	nCopied, err := io.Copy(hash, reader)
	f.Close()
	if err != nil {
		return err
	}
	if nCopied != int64(inode.Size) {
		return errors.New(fmt.Sprintf(
			"scanRegularInode(%s): read: %d, expected: %d bytes",
			myPathName, nCopied, inode.Size))
	}
	copy(inode.Hash[:], hash.Sum(nil))
	return nil
}

func scanSymlinkInode(inode *filesystem.SymlinkInode, fileSystem *FileSystem,
	myPathName string) error {
	target, err := os.Readlink(path.Join(fileSystem.rootDirectoryName,
		myPathName))
	if err != nil {
		return err
	}
	inode.Symlink = target
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2022 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kubernetes

import (
	"context"
	"fmt"
	"net"
	"reflect"
	"strconv"
	"strings"
	"testing"
	"time"

	"github.com/stretchr/testify/require"
	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/apimachinery/pkg/util/wait"
	"sigs.k8s.io/controller-runtime/pkg/client"

	"sigs.k8s.io/gateway-api/apis/v1alpha2"
)

// GWCMustBeAccepted waits until the specified GatewayClass has an Accepted
// condition set to true. It also returns the ControllerName for the
// GatewayClass. This will cause the test to halt if the specified timeout is
// exceeded.
func GWCMustBeAccepted(t *testing.T, c client.Client, gwcName string, seconds int) string {
	t.Helper()

	var controllerName string
	waitFor := time.Duration(seconds) * time.Second
	waitErr := wait.PollImmediate(1*time.Second, waitFor, func() (bool, error) {
		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
		defer cancel()

		gwc := &v1alpha2.GatewayClass{}
		err := c.Get(ctx, types.NamespacedName{Name: gwcName}, gwc)
		if err != nil {
			return false, fmt.Errorf("error fetching GatewayClass: %w", err)
		}

		controllerName = string(gwc.Spec.ControllerName)
		return findConditionInList(t, gwc.Status.Conditions, "Accepted", "True"), nil
	})
	require.NoErrorf(t, waitErr, "error waiting for %s GatewayClass to have Accepted condition set to True: %v", gwcName, waitErr)

	return controllerName
}

// NamespacesMustBeReady waits until all Pods and Gateways in the provided
// namespaces are marked as ready. This will cause the test to halt if the
// specified timeout is exceeded.
func NamespacesMustBeReady(t *testing.T, c client.Client, namespaces []string, seconds int) {
	t.Helper()

	waitFor := time.Duration(seconds) * time.Second
	waitErr := wait.PollImmediate(1*time.Second, waitFor, func() (bool, error) {
		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
		defer cancel()

		for _, ns := range namespaces {
			gwList := &v1alpha2.GatewayList{}
			err := c.List(ctx, gwList, client.InNamespace(ns))
			if err != nil {
				t.Errorf("Error listing Gateways: %v", err)
			}
			for _, gw := range gwList.Items {
				if !findConditionInList(t, gw.Status.Conditions, "Ready", "True") {
					t.Logf("%s/%s Gateway not ready yet", ns, gw.Name)
					return false, nil
				}
			}

			podList := &v1.PodList{}
			err = c.List(ctx, podList, client.InNamespace(ns))
			if err != nil {
				t.Errorf("Error listing Pods: %v", err)
			}
			for _, pod := range podList.Items {
				if !findPodConditionInList(t, pod.Status.Conditions, "Ready", "True") &&
					pod.Status.Phase != v1.PodSucceeded {
					t.Logf("%s/%s Pod not ready yet", ns, pod.Name)
					return false, nil
				}
			}
		}
		t.Logf("Gateways and Pods in %s namespaces ready", strings.Join(namespaces, ", "))
		return true, nil
	})
	require.NoErrorf(t, waitErr, "error waiting for %s namespaces to be ready", strings.Join(namespaces, ", "))
}

// GatewayAndHTTPRoutesMustBeReady waits until the specified Gateway has an IP
// address assigned to it and the Route has a ParentRef referring to the
// Gateway. The test will fail if these conditions are not met before the
// timeouts.
func GatewayAndHTTPRoutesMustBeReady(t *testing.T, c client.Client, controllerName string, gwNN types.NamespacedName, routeNNs ...types.NamespacedName) string {
	t.Helper()

	gwAddr, err := WaitForGatewayAddress(t, c, gwNN, 180)
	require.NoErrorf(t, err, "timed out waiting for Gateway address to be assigned")

	ns := v1alpha2.Namespace(gwNN.Namespace)
	kind := v1alpha2.Kind("Gateway")

	for _, routeNN := range routeNNs {
		namespaceRequired := true
		if routeNN.Namespace == gwNN.Namespace {
			namespaceRequired = false
		}
		parents := []v1alpha2.RouteParentStatus{{
			ParentRef: v1alpha2.ParentReference{
				Group:     (*v1alpha2.Group)(&v1alpha2.GroupVersion.Group),
				Kind:      &kind,
				Name:      v1alpha2.ObjectName(gwNN.Name),
				Namespace: &ns,
			},
			ControllerName: v1alpha2.GatewayController(controllerName),
			Conditions: []metav1.Condition{{
				Type:   string(v1alpha2.RouteConditionAccepted),
				Status: metav1.ConditionTrue,
			}},
		}}
		HTTPRouteMustHaveParents(t, c, routeNN, parents, namespaceRequired, 60)
	}

	return gwAddr
}

// WaitForGatewayAddress waits until at least one IP Address has been set in the
// status of the specified Gateway.
func WaitForGatewayAddress(t *testing.T, client client.Client, gwName types.NamespacedName, seconds int) (string, error) {
	t.Helper()

	var ipAddr, port string
	waitFor := time.Duration(seconds) * time.Second
	waitErr := wait.PollImmediate(1*time.Second, waitFor, func() (bool, error) {
		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
		defer cancel()

		gw := &v1alpha2.Gateway{}
		err := client.Get(ctx, gwName, gw)
		if err != nil {
			t.Logf("error fetching Gateway: %v", err)
			return false, fmt.Errorf("error fetching Gateway: %w", err)
		}

		port = strconv.FormatInt(int64(gw.Spec.Listeners[0].Port), 10)

		// TODO: Support more than IPAddress
		for _, address := range gw.Status.Addresses {
			if address.Type != nil && *address.Type == v1alpha2.IPAddressType {
				ipAddr = address.Value
				return true, nil
			}
		}

		return false, nil
	})
	require.NoErrorf(t, waitErr, "error waiting for Gateway to have at least one IP address in status")
	return net.JoinHostPort(ipAddr, port), waitErr
}

// HTTPRouteMustHaveParents waits for the specified HTTPRoute to have either no parents
// or a single parent that is not acceptted. This is used to validate HTTPRoute errors.
func HTTPRouteMustHaveNoAcceptedParents(t *testing.T, client client.Client, routeName types.NamespacedName, seconds int) {
	t.Helper()

	var actual []v1alpha2.RouteParentStatus
	waitFor := time.Duration(seconds) * time.Second
	waitErr := wait.PollImmediate(1*time.Second, waitFor, func() (bool, error) {
		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
		defer cancel()

		route := &v1alpha2.HTTPRoute{}
		err := client.Get(ctx, routeName, route)
		if err != nil {
			return false, fmt.Errorf("error fetching HTTPRoute: %w", err)
		}

		actual = route.Status.Parents

		if len(actual) == 0 {
			return true, nil
		}
		if len(actual) > 1 {
			// Only expect one parent
			return false, nil
		}
		return conditionsMatch(t, []metav1.Condition{{
			Type:   string(v1alpha2.RouteConditionAccepted),
			Status: "False",
		}}, actual[0].Conditions), nil

	})
	require.NoErrorf(t, waitErr, "error waiting for HTTPRoute to have no accepted parents")
}

// HTTPRouteMustHaveParents waits for the specified HTTPRoute to have parents
// in status that match the expected parents. This will cause the test to halt
// if the specified timeout is exceeded.
func HTTPRouteMustHaveParents(t *testing.T, client client.Client, routeName types.NamespacedName, parents []v1alpha2.RouteParentStatus, namespaceRequired bool, seconds int) {
	t.Helper()

	var actual []v1alpha2.RouteParentStatus
	waitFor := time.Duration(seconds) * time.Second
	waitErr := wait.PollImmediate(1*time.Second, waitFor, func() (bool, error) {
		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
		defer cancel()

		route := &v1alpha2.HTTPRoute{}
		err := client.Get(ctx, routeName, route)
		if err != nil {
			return false, fmt.Errorf("error fetching HTTPRoute: %w", err)
		}

		actual = route.Status.Parents

		return parentsMatch(t, parents, actual, namespaceRequired), nil
	})
	require.NoErrorf(t, waitErr, "error waiting for HTTPRoute to have parents matching expectations")
}

func parentsMatch(t *testing.T, expected, actual []v1alpha2.RouteParentStatus, namespaceRequired bool) bool {
	t.Helper()

	if len(expected) != len(actual) {
		t.Logf("Expected %d Route parents, got %d", len(expected), len(actual))
		return false
	}

	// TODO(robscott): Allow for arbitrarily ordered parents
	for i, eParent := range expected {
		aParent := actual[i]
		if aParent.ControllerName != eParent.ControllerName {
			t.Logf("ControllerName doesn't match")
			return false
		}
		if !reflect.DeepEqual(aParent.ParentRef.Group, eParent.ParentRef.Group) {
			t.Logf("Expected ParentReference.Group to be %v, got %v", eParent.ParentRef.Group, aParent.ParentRef.Group)
			return false
		}
		if !reflect.DeepEqual(aParent.ParentRef.Kind, eParent.ParentRef.Kind) {
			t.Logf("Expected ParentReference.Kind to be %v, got %v", eParent.ParentRef.Kind, aParent.ParentRef.Kind)
			return false
		}
		if aParent.ParentRef.Name != eParent.ParentRef.Name {
			t.Logf("ParentReference.Name doesn't match")
			return false
		}
		if !reflect.DeepEqual(aParent.ParentRef.Namespace, eParent.ParentRef.Namespace) {
			if namespaceRequired || aParent.ParentRef.Namespace != nil {
				t.Logf("Expected ParentReference.Namespace to be %v, got %v", eParent.ParentRef.Namespace, aParent.ParentRef.Namespace)
				return false
			}
		}
		if !conditionsMatch(t, eParent.Conditions, aParent.Conditions) {
			return false
		}
	}

	t.Logf("Route parents matched expectations")
	return true
}

// GatewayStatusMustHaveListeners waits for the specified Gateway to have listeners
// in status that match the expected listeners. This will cause the test to halt
// if the specified timeout is exceeded.
func GatewayStatusMustHaveListeners(t *testing.T, client client.Client, gwNN types.NamespacedName, listeners []v1alpha2.ListenerStatus, seconds int) {
	t.Helper()

	var actual []v1alpha2.ListenerStatus
	waitFor := time.Duration(seconds) * time.Second
	waitErr := wait.PollImmediate(1*time.Second, waitFor, func() (bool, error) {
		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
		defer cancel()

		gw := &v1alpha2.Gateway{}
		err := client.Get(ctx, gwNN, gw)
		if err != nil {
			return false, fmt.Errorf("error fetching Gateway: %w", err)
		}

		actual = gw.Status.Listeners

		return listenersMatch(t, listeners, actual), nil
	})
	require.NoErrorf(t, waitErr, "error waiting for Gateway status to have listeners matching expectations")
}

// TODO(mikemorris): this and parentsMatch could possibly be rewritten as a generic function?
func listenersMatch(t *testing.T, expected, actual []v1alpha2.ListenerStatus) bool {
	t.Helper()

	if len(expected) != len(actual) {
		t.Logf("Expected %d Gateway status listeners, got %d", len(expected), len(actual))
		return false
	}

	// TODO(mikemorris): Allow for arbitrarily ordered listeners
	for i, eListener := range expected {
		aListener := actual[i]
		if aListener.Name != eListener.Name {
			t.Logf("Name doesn't match")
			return false
		}
		if !reflect.DeepEqual(aListener.SupportedKinds, eListener.SupportedKinds) {
			t.Logf("Expected SupportedKinds to be %v, got %v", eListener.SupportedKinds, aListener.SupportedKinds)
			return false
		}
		if aListener.AttachedRoutes != eListener.AttachedRoutes {
			t.Logf("Expected AttachedRoutes to be %v, got %v", eListener.AttachedRoutes, aListener.AttachedRoutes)
			return false
		}
		if !conditionsMatch(t, eListener.Conditions, aListener.Conditions) {
			return false
		}
	}

	t.Logf("Gateway status listeners matched expectations")
	return true
}

func conditionsMatch(t *testing.T, expected, actual []metav1.Condition) bool {
	if len(actual) < len(expected) {
		t.Logf("Expected more conditions to be present")
		return false
	}
	for _, condition := range expected {
		if !findConditionInList(t, actual, condition.Type, string(condition.Status)) {
			return false
		}
	}

	t.Logf("Conditions matched expectations")
	return true
}

func findConditionInList(t *testing.T, conditions []metav1.Condition, condName, condValue string) bool {
	for _, cond := range conditions {
		if cond.Type == condName {
			if cond.Status == metav1.ConditionStatus(condValue) {
				return true
			}
			t.Logf("%s condition set to %s, expected %s", condName, cond.Status, condValue)
		}
	}

	t.Logf("%s was not in conditions list", condName)
	return false
}

func findPodConditionInList(t *testing.T, conditions []v1.PodCondition, condName, condValue string) bool {
	for _, cond := range conditions {
		if cond.Type == v1.PodConditionType(condName) {
			if cond.Status == v1.ConditionStatus(condValue) {
				return true
			}
			t.Logf("%s condition set to %s, expected %s", condName, cond.Status, condValue)
		}
	}

	t.Logf("%s was not in conditions list", condName)
	return false
}
<commit_msg>Add delay<commit_after>/*
Copyright 2022 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kubernetes

import (
	"context"
	"fmt"
	"net"
	"reflect"
	"strconv"
	"strings"
	"testing"
	"time"

	"github.com/stretchr/testify/require"
	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/apimachinery/pkg/util/wait"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/gateway-api/apis/v1alpha2"
)

// GWCMustBeAccepted waits until the specified GatewayClass has an Accepted
// condition set to true. It also returns the ControllerName for the
// GatewayClass. This will cause the test to halt if the specified timeout is
// exceeded.
func GWCMustBeAccepted(t *testing.T, c client.Client, gwcName string, seconds int) string {
	t.Helper()

	var controllerName string
	waitFor := time.Duration(seconds) * time.Second
	waitErr := wait.PollImmediate(1*time.Second, waitFor, func() (bool, error) {
		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
		defer cancel()

		gwc := &v1alpha2.GatewayClass{}
		err := c.Get(ctx, types.NamespacedName{Name: gwcName}, gwc)
		if err != nil {
			return false, fmt.Errorf("error fetching GatewayClass: %w", err)
		}

		controllerName = string(gwc.Spec.ControllerName)
		return findConditionInList(t, gwc.Status.Conditions, "Accepted", "True"), nil
	})
	require.NoErrorf(t, waitErr, "error waiting for %s GatewayClass to have Accepted condition set to True: %v", gwcName, waitErr)

	return controllerName
}

// NamespacesMustBeReady waits until all Pods and Gateways in the provided
// namespaces are marked as ready. This will cause the test to halt if the
// specified timeout is exceeded.
func NamespacesMustBeReady(t *testing.T, c client.Client, namespaces []string, seconds int) {
	t.Helper()

	waitFor := time.Duration(seconds) * time.Second
	waitErr := wait.PollImmediate(1*time.Second, waitFor, func() (bool, error) {
		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
		defer cancel()

		for _, ns := range namespaces {
			gwList := &v1alpha2.GatewayList{}
			err := c.List(ctx, gwList, client.InNamespace(ns))
			if err != nil {
				t.Errorf("Error listing Gateways: %v", err)
			}
			for _, gw := range gwList.Items {
				if !findConditionInList(t, gw.Status.Conditions, "Ready", "True") {
					t.Logf("%s/%s Gateway not ready yet", ns, gw.Name)
					return false, nil
				}
			}

			podList := &v1.PodList{}
			err = c.List(ctx, podList, client.InNamespace(ns))
			if err != nil {
				t.Errorf("Error listing Pods: %v", err)
			}
			for _, pod := range podList.Items {
				if !findPodConditionInList(t, pod.Status.Conditions, "Ready", "True") &&
					pod.Status.Phase != v1.PodSucceeded {
					t.Logf("%s/%s Pod not ready yet", ns, pod.Name)
					return false, nil
				}
			}
		}
		t.Logf("Gateways and Pods in %s namespaces ready", strings.Join(namespaces, ", "))
		return true, nil
	})
	require.NoErrorf(t, waitErr, "error waiting for %s namespaces to be ready", strings.Join(namespaces, ", "))
}

// GatewayAndHTTPRoutesMustBeReady waits until the specified Gateway has an IP
// address assigned to it and the Route has a ParentRef referring to the
// Gateway. The test will fail if these conditions are not met before the
// timeouts.
func GatewayAndHTTPRoutesMustBeReady(t *testing.T, c client.Client, controllerName string, gwNN types.NamespacedName, routeNNs ...types.NamespacedName) string {
	t.Helper()

	gwAddr, err := WaitForGatewayAddress(t, c, gwNN, 180)
	require.NoErrorf(t, err, "timed out waiting for Gateway address to be assigned")

	ns := v1alpha2.Namespace(gwNN.Namespace)
	kind := v1alpha2.Kind("Gateway")

	for _, routeNN := range routeNNs {
		namespaceRequired := true
		if routeNN.Namespace == gwNN.Namespace {
			namespaceRequired = false
		}
		parents := []v1alpha2.RouteParentStatus{{
			ParentRef: v1alpha2.ParentReference{
				Group:     (*v1alpha2.Group)(&v1alpha2.GroupVersion.Group),
				Kind:      &kind,
				Name:      v1alpha2.ObjectName(gwNN.Name),
				Namespace: &ns,
			},
			ControllerName: v1alpha2.GatewayController(controllerName),
			Conditions: []metav1.Condition{{
				Type:   string(v1alpha2.RouteConditionAccepted),
				Status: metav1.ConditionTrue,
			}},
		}}
		HTTPRouteMustHaveParents(t, c, routeNN, parents, namespaceRequired, 60)
	}

	return gwAddr
}

// WaitForGatewayAddress waits until at least one IP Address has been set in the
// status of the specified Gateway.
func WaitForGatewayAddress(t *testing.T, client client.Client, gwName types.NamespacedName, seconds int) (string, error) {
	t.Helper()

	var ipAddr, port string
	waitFor := time.Duration(seconds) * time.Second
	waitErr := wait.PollImmediate(1*time.Second, waitFor, func() (bool, error) {
		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
		defer cancel()

		gw := &v1alpha2.Gateway{}
		err := client.Get(ctx, gwName, gw)
		if err != nil {
			t.Logf("error fetching Gateway: %v", err)
			return false, fmt.Errorf("error fetching Gateway: %w", err)
		}

		port = strconv.FormatInt(int64(gw.Spec.Listeners[0].Port), 10)

		// TODO: Support more than IPAddress
		for _, address := range gw.Status.Addresses {
			if address.Type != nil && *address.Type == v1alpha2.IPAddressType {
				ipAddr = address.Value
				return true, nil
			}
		}

		return false, nil
	})
	require.NoErrorf(t, waitErr, "error waiting for Gateway to have at least one IP address in status")
	return net.JoinHostPort(ipAddr, port), waitErr
}

// HTTPRouteMustHaveNoAcceptedParents waits for the specified HTTPRoute to have either no parents
// or a single parent that is not accepted. This is used to validate HTTPRoute errors.
func HTTPRouteMustHaveNoAcceptedParents(t *testing.T, client client.Client, routeName types.NamespacedName, seconds int) {
	t.Helper()

	var actual []v1alpha2.RouteParentStatus
	waitFor := time.Duration(seconds) * time.Second
	emptyChecked := false
	waitErr := wait.PollImmediate(1*time.Second, waitFor, func() (bool, error) {
		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
		defer cancel()

		route := &v1alpha2.HTTPRoute{}
		err := client.Get(ctx, routeName, route)
		if err != nil {
			return false, fmt.Errorf("error fetching HTTPRoute: %w", err)
		}

		actual = route.Status.Parents

		if len(actual) == 0 {
			// For empty status, we need to distinguish between "correctly did not set" and "hasn't set yet"
			// Ensure we iterate at least two times (taking advantage of the 1s poll delay) to give it some time.
			if !emptyChecked {
				emptyChecked = true
				return false, nil
			}
			return true, nil
		}
		if len(actual) > 1 {
			// Only expect one parent
			return false, nil
		}
		return conditionsMatch(t, []metav1.Condition{{
			Type:   string(v1alpha2.RouteConditionAccepted),
			Status: "False",
		}}, actual[0].Conditions), nil
	})
	require.NoErrorf(t, waitErr, "error waiting for HTTPRoute to have no accepted parents")
}

// HTTPRouteMustHaveParents waits for the specified HTTPRoute to have parents
// in status that match the expected parents. This will cause the test to halt
// if the specified timeout is exceeded.
func HTTPRouteMustHaveParents(t *testing.T, client client.Client, routeName types.NamespacedName, parents []v1alpha2.RouteParentStatus, namespaceRequired bool, seconds int) {
	t.Helper()

	var actual []v1alpha2.RouteParentStatus
	waitFor := time.Duration(seconds) * time.Second
	waitErr := wait.PollImmediate(1*time.Second, waitFor, func() (bool, error) {
		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
		defer cancel()

		route := &v1alpha2.HTTPRoute{}
		err := client.Get(ctx, routeName, route)
		if err != nil {
			return false, fmt.Errorf("error fetching HTTPRoute: %w", err)
		}

		actual = route.Status.Parents

		return parentsMatch(t, parents, actual, namespaceRequired), nil
	})
	require.NoErrorf(t, waitErr, "error waiting for HTTPRoute to have parents matching expectations")
}

func parentsMatch(t *testing.T, expected, actual []v1alpha2.RouteParentStatus, namespaceRequired bool) bool {
	t.Helper()

	if len(expected) != len(actual) {
		t.Logf("Expected %d Route parents, got %d", len(expected), len(actual))
		return false
	}

	// TODO(robscott): Allow for arbitrarily ordered parents
	for i, eParent := range expected {
		aParent := actual[i]
		if aParent.ControllerName != eParent.ControllerName {
			t.Logf("ControllerName doesn't match")
			return false
		}
		if !reflect.DeepEqual(aParent.ParentRef.Group, eParent.ParentRef.Group) {
			t.Logf("Expected ParentReference.Group to be %v, got %v", eParent.ParentRef.Group, aParent.ParentRef.Group)
			return false
		}
		if !reflect.DeepEqual(aParent.ParentRef.Kind, eParent.ParentRef.Kind) {
			t.Logf("Expected ParentReference.Kind to be %v, got %v", eParent.ParentRef.Kind, aParent.ParentRef.Kind)
			return false
		}
		if aParent.ParentRef.Name != eParent.ParentRef.Name {
			t.Logf("ParentReference.Name doesn't match")
			return false
		}
		if !reflect.DeepEqual(aParent.ParentRef.Namespace, eParent.ParentRef.Namespace) {
			if namespaceRequired || aParent.ParentRef.Namespace != nil {
				t.Logf("Expected ParentReference.Namespace to be %v, got %v", eParent.ParentRef.Namespace, aParent.ParentRef.Namespace)
				return false
			}
		}
		if !conditionsMatch(t, eParent.Conditions, aParent.Conditions) {
			return false
		}
	}

	t.Logf("Route parents matched expectations")
	return true
}

// GatewayStatusMustHaveListeners waits for the specified Gateway to have listeners
// in status that match the expected listeners. This will cause the test to halt
// if the specified timeout is exceeded.
func GatewayStatusMustHaveListeners(t *testing.T, client client.Client, gwNN types.NamespacedName, listeners []v1alpha2.ListenerStatus, seconds int) {
	t.Helper()

	var actual []v1alpha2.ListenerStatus
	waitFor := time.Duration(seconds) * time.Second
	waitErr := wait.PollImmediate(1*time.Second, waitFor, func() (bool, error) {
		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
		defer cancel()

		gw := &v1alpha2.Gateway{}
		err := client.Get(ctx, gwNN, gw)
		if err != nil {
			return false, fmt.Errorf("error fetching Gateway: %w", err)
		}

		actual = gw.Status.Listeners

		return listenersMatch(t, listeners, actual), nil
	})
	require.NoErrorf(t, waitErr, "error waiting for Gateway status to have listeners matching expectations")
}

// TODO(mikemorris): this and parentsMatch could possibly be rewritten as a generic function?
func listenersMatch(t *testing.T, expected, actual []v1alpha2.ListenerStatus) bool {
	t.Helper()

	if len(expected) != len(actual) {
		t.Logf("Expected %d Gateway status listeners, got %d", len(expected), len(actual))
		return false
	}

	// TODO(mikemorris): Allow for arbitrarily ordered listeners
	for i, eListener := range expected {
		aListener := actual[i]
		if aListener.Name != eListener.Name {
			t.Logf("Name doesn't match")
			return false
		}
		if !reflect.DeepEqual(aListener.SupportedKinds, eListener.SupportedKinds) {
			t.Logf("Expected SupportedKinds to be %v, got %v", eListener.SupportedKinds, aListener.SupportedKinds)
			return false
		}
		if aListener.AttachedRoutes != eListener.AttachedRoutes {
			t.Logf("Expected AttachedRoutes to be %v, got %v", eListener.AttachedRoutes, aListener.AttachedRoutes)
			return false
		}
		if !conditionsMatch(t, eListener.Conditions, aListener.Conditions) {
			return false
		}
	}

	t.Logf("Gateway status listeners matched expectations")
	return true
}

func conditionsMatch(t *testing.T, expected, actual []metav1.Condition) bool {
	if len(actual) < len(expected) {
		t.Logf("Expected more conditions to be present")
		return false
	}
	for _, condition := range expected {
		if !findConditionInList(t, actual, condition.Type, string(condition.Status)) {
			return false
		}
	}

	t.Logf("Conditions matched expectations")
	return true
}

func findConditionInList(t *testing.T, conditions []metav1.Condition, condName, condValue string) bool {
	for _, cond := range conditions {
		if cond.Type == condName {
			if cond.Status == metav1.ConditionStatus(condValue) {
				return true
			}
			t.Logf("%s condition set to %s, expected %s", condName, cond.Status, condValue)
		}
	}

	t.Logf("%s was not in conditions list", condName)
	return false
}

func findPodConditionInList(t *testing.T, conditions []v1.PodCondition, condName, condValue string) bool {
	for _, cond := range conditions {
		if cond.Type == v1.PodConditionType(condName) {
			if cond.Status == v1.ConditionStatus(condValue) {
				return true
			}
			t.Logf("%s condition set to %s, expected %s", condName, cond.Status, condValue)
		}
	}

	t.Logf("%s was not in conditions list", condName)
	return false
}
<|endoftext|>


<commit_before>package server_manager

import (
	"log"
	"math/rand"
	"sync"
	"sync/atomic"
	"time"

	"github.com/hashicorp/consul/consul/server_details"
	"github.com/hashicorp/consul/lib"
)

type consulServerEventTypes int

const (
	// consulServersNodeJoin is used to notify of a new consulServer.
	// The primary effect of this is a reshuffling of consulServers and
	// finding a new preferredServer.
	consulServersNodeJoin = iota

	// consulServersRebalance is used to signal we should rebalance our
	// connection load across servers
	consulServersRebalance

	// consulServersRPCError is used to signal when a server has either
	// timed out or returned an error and we would like to have the
	// server manager find a new preferredServer.
	consulServersRPCError
)

const (
	// clientRPCJitterFraction determines the amount of jitter added to
	// clientRPCMinReuseDuration before a connection is expired and a new
	// connection is established in order to rebalance load across consul
	// servers.  The cluster-wide number of connections per second from
	// rebalancing is applied after this jitter to ensure the CPU impact
	// is always finite.  See newRebalanceConnsPerSecPerServer's comment
	// for additional commentary.
	//
	// For example, in a 10K consul cluster with 5x servers, this default
	// averages out to ~13 new connections from rebalancing per server
	// per second (each connection is reused for 120s to 180s).
	clientRPCJitterFraction = 2

	// clientRPCMinReuseDuration controls the minimum amount of time RPC
	// queries are sent over an established connection to a single server
	clientRPCMinReuseDuration = 120 * time.Second

	// Limit the number of new connections a server receives per second
	// for connection rebalancing.  This limit caps the load caused by
	// continual rebalancing efforts when a cluster is in equilibrium.  A
	// lower value comes at the cost of increased recovery time after a
	// partition.  This parameter begins to take effect when there are
	// more than ~48K clients querying 5x servers or at lower server
	// values when there is a partition.
	//
	// For example, in a 100K consul cluster with 5x servers, it will
	// take ~5min for all servers to rebalance their connections.  If
	// 99,995 agents are in the minority talking to only one server, it
	// will take ~26min for all servers to rebalance.  A 10K cluster in
	// the same scenario will take ~2.6min to rebalance.
	newRebalanceConnsPerSecPerServer = 64
)

// serverCfg is the thread-safe configuration structure that is used to
// maintain the list of consul servers in Client.
//
// NOTE(sean@): We are explicitly relying on the fact that this is copied.
// Please keep this structure light.
type serverConfig struct {
	// servers tracks the locally known servers
	servers []*server_details.ServerDetails

	// Timer used to control rebalancing of servers
	rebalanceTimer *time.Timer
}

type ServerManager struct {
	// serverConfig provides the necessary load/store semantics to
	// serverConfig
	serverConfigValue atomic.Value
	serverConfigLock  sync.Mutex

	// consulServersCh is used to receive events related to the
	// maintenance of the list of consulServers
	consulServersCh chan consulServerEventTypes

	// shutdownCh is a copy of the channel in consul.Client
	shutdownCh chan struct{}

	// Logger uses the provided LogOutput
	logger *log.Logger
}

// AddServer takes out an internal write lock and adds a new server.  If the
// server is not known, it adds the new server and schedules a rebalance.  If
// it is known, we merge the new server details.
func (sm *ServerManager) AddServer(server *server_details.ServerDetails) {
	sm.serverConfigLock.Lock()
	defer sm.serverConfigLock.Unlock()
	serverCfg := sm.getServerConfig()

	// Check if this server is known
	found := false
	for idx, existing := range serverCfg.servers {
		if existing.Name == server.Name {
			// Overwrite the existing server parts in order to
			// possibly update metadata (i.e. server version)
			serverCfg.servers[idx] = server
			found = true
			break
		}
	}

	// Add to the list if not known
	if !found {
		newServers := make([]*server_details.ServerDetails, len(serverCfg.servers)+1)
		copy(newServers, serverCfg.servers)
		serverCfg.servers = newServers

		// Notify the server maintenance task of a new server
		sm.consulServersCh <- consulServersNodeJoin
	}

	sm.serverConfigValue.Store(serverCfg)
}

// CycleFailedServers takes out an internal write lock and dequeues all
// failed servers and re-enqueues them.  This method does not reshuffle the
// server list.  Because this changed the order of servers, we push out the
// time at which a rebalance occurs.
func (sm *ServerManager) CycleFailedServers() {
	sm.serverConfigLock.Lock()
	defer sm.serverConfigLock.Unlock()
	serverCfg := sm.getServerConfig()

	for i := range serverCfg.servers {
		failCount := atomic.LoadUint64(&(serverCfg.servers[i].Disabled))
		if failCount == 0 {
			break
		} else if failCount > 0 {
			serverCfg.servers = serverCfg.cycleServer()
		}
	}

	serverCfg.resetRebalanceTimer(sm)
	sm.serverConfigValue.Store(serverCfg)
}

// cycleServers returns a new list of servers that has dequeued the first
// server and enqueued it at the end of the list.  cycleServers assumes the
// caller is holding the serverConfigLock.
func (sc *serverConfig) cycleServer() (servers []*server_details.ServerDetails) {
	numServers := len(servers)
	if numServers < 2 {
		// No action required
		return servers
	}

	var dequeuedServer *server_details.ServerDetails
	newServers := make([]*server_details.ServerDetails, len(servers)+1)
	dequeuedServer, newServers = servers[0], servers[1:]
	servers = append(newServers, dequeuedServer)
	return servers
}

// FindHealthyServer takes out an internal "read lock" and searches through
// the list of servers to find a healthy server.
func (sm *ServerManager) FindHealthyServer() (server *server_details.ServerDetails) {
	serverCfg := sm.getServerConfig()
	numServers := len(serverCfg.servers)
	if numServers == 0 {
		sm.logger.Printf("[ERR] consul: No servers found in the server config")
		return nil
	}

	// Find the first non-failing server in the server list.  If this is
	// not the first server a prior RPC call marked the first server as
	// failed and we're waiting for the server management task to reorder
	// a working server to the front of the list.
	for i := range serverCfg.servers {
		failCount := atomic.LoadUint64(&(serverCfg.servers[i].Disabled))
		if failCount == 0 {
			server = serverCfg.servers[i]
			break
		}
	}

	return server
}

// GetNumServers takes out an internal "read lock" and returns the number of
// servers.  numServers includes both healthy and unhealthy servers.
func (sm *ServerManager) GetNumServers() (numServers int) {
	serverCfg := sm.getServerConfig()
	numServers = len(serverCfg.servers)
	return numServers
}

// getServerConfig is a convenience method to hide the locking semantics of
// atomic.Value from the caller.
func (sm *ServerManager) getServerConfig() serverConfig {
	return sm.serverConfigValue.Load().(serverConfig)
}

// NewServerManager is the only way to safely create a new ServerManager
// struct.
//
// NOTE(sean@): We can not pass in *consul.Client due to an import cycle
func NewServerManager(logger *log.Logger, shutdownCh chan struct{}) (sm *ServerManager) {
	sm = new(ServerManager)
	// Create the initial serverConfig
	serverCfg := serverConfig{}
	sm.logger = logger
	sm.shutdownCh = shutdownCh
	sm.serverConfigValue.Store(serverCfg)
	return sm
}

// NotifyFailedServer is an exported convenience function that allows callers
// to pass in a server that has failed an RPC request and mark it as failed.
// This will initiate a background task that will optimize the failed server
// to the end of the serer list.  No locks are required here because we are
// bypassing the serverConfig and sending a message to ServerManager's
// channel.
func (sm *ServerManager) NotifyFailedServer(server *server_details.ServerDetails) {
	atomic.AddUint64(&server.Disabled, 1)
	sm.consulServersCh <- consulServersRPCError
}

// RebalanceServers takes out an internal write lock and shuffles the list of
// servers on this agent.  This allows for a redistribution of work across
// consul servers and provides a guarantee that the order list of
// ServerDetails isn't actually ordered, therefore we can sequentially walk
// the array to pick a server without all agents in the cluster dog piling on
// a single node.
func (sm *ServerManager) RebalanceServers() {
	sm.serverConfigLock.Lock()
	defer sm.serverConfigLock.Unlock()
	serverCfg := sm.getServerConfig()

	// Shuffle the server list on server join.  Servers are selected from
	// the head of the list and are moved to the end of the list on
	// failure.
	for i := len(serverCfg.servers) - 1; i > 0; i-- {
		j := rand.Int31n(int32(i + 1))
		serverCfg.servers[i], serverCfg.servers[j] = serverCfg.servers[j], serverCfg.servers[i]
	}

	serverCfg.resetRebalanceTimer(sm)
	sm.serverConfigValue.Store(serverCfg)
}

// RemoveServer takes out an internal write lock and removes a server from
// the server list.  No rebalancing happens as a result of the removed server
// because we do not want a network partition which separated a server from
// this agent to cause an increase in work.  Instead we rely on the internal
// already existing semantics to handle failure detection after a server has
// been removed.
func (sm *ServerManager) RemoveServer(server *server_details.ServerDetails) {
	sm.serverConfigLock.Lock()
	defer sm.serverConfigLock.Unlock()
	serverCfg := sm.getServerConfig()

	// Remove the server if known
	n := len(serverCfg.servers)
	for i := 0; i < n; i++ {
		if serverCfg.servers[i].Name == server.Name {
			serverCfg.servers[i], serverCfg.servers[n-1] = serverCfg.servers[n-1], nil
			serverCfg.servers = serverCfg.servers[:n-1]
			break
		}
	}

	sm.serverConfigValue.Store(serverCfg)
}

// resetRebalanceTimer assumes:
//
// 1) the serverConfigLock is already held by the caller.
// 2) the caller will call serverConfigValue.Store()
func (sc *serverConfig) resetRebalanceTimer(sm *ServerManager) {
	numConsulServers := len(sc.servers)
	// Limit this connection's life based on the size (and health) of the
	// cluster.  Never rebalance a connection more frequently than
	// connReuseLowWatermarkDuration, and make sure we never exceed
	// clusterWideRebalanceConnsPerSec operations/s across numLANMembers.
	clusterWideRebalanceConnsPerSec := float64(numConsulServers * newRebalanceConnsPerSecPerServer)
	connReuseLowWatermarkDuration := clientRPCMinReuseDuration + lib.RandomStagger(clientRPCMinReuseDuration/clientRPCJitterFraction)
	numLANMembers := 16384 // Assume sufficiently large for now. FIXME: numLanMembers := len(c.LANMembers())
	connRebalanceTimeout := lib.RateScaledInterval(clusterWideRebalanceConnsPerSec, connReuseLowWatermarkDuration, numLANMembers)
	sm.logger.Printf("[DEBUG] consul: connection will be rebalanced in %v", connRebalanceTimeout)

	if sc.rebalanceTimer == nil {
		sc.rebalanceTimer = time.NewTimer(connRebalanceTimeout)
	} else {
		sc.rebalanceTimer.Reset(connRebalanceTimeout)
	}
}

// StartServerManager is used to start and manage the task of automatically
// shuffling and rebalance the list of consul servers.  This maintenance
// happens either when a new server is added or when a duration has been
// exceed.
func (sm *ServerManager) StartServerManager() {
	var rebalanceTimer *time.Timer
	func() {
		sm.serverConfigLock.Lock()
		defer sm.serverConfigLock.Unlock()

		serverCfgPtr := sm.serverConfigValue.Load()
		if serverCfgPtr == nil {
			panic("server config has not been initialized")
		}
		var serverCfg serverConfig
		serverCfg = serverCfgPtr.(serverConfig)
		serverCfg.resetRebalanceTimer(sm)
		rebalanceTimer = serverCfg.rebalanceTimer
		sm.serverConfigValue.Store(serverCfg)
	}()

	for {
		select {
		case e := <-sm.consulServersCh:
			switch e {
			case consulServersNodeJoin:
				sm.logger.Printf("[INFO] consul: new node joined cluster")
				sm.RebalanceServers()
			case consulServersRebalance:
				sm.logger.Printf("[INFO] consul: rebalancing servers by request")
				sm.RebalanceServers()
			case consulServersRPCError:
				sm.logger.Printf("[INFO] consul: need to find a new server to talk with")
				sm.CycleFailedServers()
				// FIXME(sean@): wtb preemptive Status.Ping
				// of servers, ideally parallel fan-out of N
				// nodes, then settle on the first node which
				// responds successfully.
				//
				// Is there a distinction between slow and
				// offline?  Do we run the Status.Ping with a
				// fixed timeout (say 30s) that way we can
				// alert administrators that they've set
				// their RPC time too low even though the
				// Ping did return successfully?
			default:
				sm.logger.Printf("[WARN] consul: unhandled LAN Serf Event: %#v", e)
			}
		case <-rebalanceTimer.C:
			sm.logger.Printf("[INFO] consul: server rebalance timeout")
			sm.RebalanceServers()

		case <-sm.shutdownCh:
			return
		}
	}
}
<commit_msg>Mutate copies of serverCfg.servers, not original<commit_after>package server_manager

import (
	"log"
	"math/rand"
	"sync"
	"sync/atomic"
	"time"

	"github.com/hashicorp/consul/consul/server_details"
	"github.com/hashicorp/consul/lib"
)

type consulServerEventTypes int

const (
	// consulServersNodeJoin is used to notify of a new consulServer.
	// The primary effect of this is a reshuffling of consulServers and
	// finding a new preferredServer.
	consulServersNodeJoin = iota

	// consulServersRebalance is used to signal we should rebalance our
	// connection load across servers
	consulServersRebalance

	// consulServersRPCError is used to signal when a server has either
	// timed out or returned an error and we would like to have the
	// server manager find a new preferredServer.
	consulServersRPCError
)

const (
	// clientRPCJitterFraction determines the amount of jitter added to
	// clientRPCMinReuseDuration before a connection is expired and a new
	// connection is established in order to rebalance load across consul
	// servers.  The cluster-wide number of connections per second from
	// rebalancing is applied after this jitter to ensure the CPU impact
	// is always finite.  See newRebalanceConnsPerSecPerServer's comment
	// for additional commentary.
	//
	// For example, in a 10K consul cluster with 5x servers, this default
	// averages out to ~13 new connections from rebalancing per server
	// per second (each connection is reused for 120s to 180s).
	clientRPCJitterFraction = 2

	// clientRPCMinReuseDuration controls the minimum amount of time RPC
	// queries are sent over an established connection to a single server
	clientRPCMinReuseDuration = 120 * time.Second

	// Limit the number of new connections a server receives per second
	// for connection rebalancing.  This limit caps the load caused by
	// continual rebalancing efforts when a cluster is in equilibrium.  A
	// lower value comes at the cost of increased recovery time after a
	// partition.  This parameter begins to take effect when there are
	// more than ~48K clients querying 5x servers or at lower server
	// values when there is a partition.
	//
	// For example, in a 100K consul cluster with 5x servers, it will
	// take ~5min for all servers to rebalance their connections.  If
	// 99,995 agents are in the minority talking to only one server, it
	// will take ~26min for all servers to rebalance.  A 10K cluster in
	// the same scenario will take ~2.6min to rebalance.
	newRebalanceConnsPerSecPerServer = 64
)

// serverCfg is the thread-safe configuration structure that is used to
// maintain the list of consul servers in Client.
//
// NOTE(sean@): We are explicitly relying on the fact that this is copied.
// Please keep this structure light.
type serverConfig struct {
	// servers tracks the locally known servers
	servers []*server_details.ServerDetails

	// Timer used to control rebalancing of servers
	rebalanceTimer *time.Timer
}

type ServerManager struct {
	// serverConfig provides the necessary load/store semantics to
	// serverConfig
	serverConfigValue atomic.Value
	serverConfigLock  sync.Mutex

	// consulServersCh is used to receive events related to the
	// maintenance of the list of consulServers
	consulServersCh chan consulServerEventTypes

	// shutdownCh is a copy of the channel in consul.Client
	shutdownCh chan struct{}

	// Logger uses the provided LogOutput
	logger *log.Logger
}

// AddServer takes out an internal write lock and adds a new server.  If the
// server is not known, it adds the new server and schedules a rebalance.  If
// it is known, we merge the new server details.
func (sm *ServerManager) AddServer(server *server_details.ServerDetails) {
	sm.serverConfigLock.Lock()
	defer sm.serverConfigLock.Unlock()
	serverCfg := sm.getServerConfig()

	// Check if this server is known
	found := false
	for idx, existing := range serverCfg.servers {
		if existing.Name == server.Name {
			newServers := make([]*server_details.ServerDetails, len(serverCfg.servers))
			copy(newServers, serverCfg.servers)

			// Overwrite the existing server details in order to
			// possibly update metadata (e.g. server version)
			newServers[idx] = server

			serverCfg.servers = newServers
			found = true
			break
		}
	}

	// Add to the list if not known
	if !found {
		newServers := make([]*server_details.ServerDetails, len(serverCfg.servers)+1)
		copy(newServers, serverCfg.servers)
		serverCfg.servers = newServers

		// Notify the server maintenance task of a new server
		sm.consulServersCh <- consulServersNodeJoin
	}

	sm.serverConfigValue.Store(serverCfg)
}

// CycleFailedServers takes out an internal write lock and dequeues all
// failed servers and re-enqueues them.  This method does not reshuffle the
// server list.  Because this changed the order of servers, we push out the
// time at which a rebalance occurs.
func (sm *ServerManager) CycleFailedServers() {
	sm.serverConfigLock.Lock()
	defer sm.serverConfigLock.Unlock()
	serverCfg := sm.getServerConfig()

	for i := range serverCfg.servers {
		failCount := atomic.LoadUint64(&(serverCfg.servers[i].Disabled))
		if failCount == 0 {
			break
		} else if failCount > 0 {
			serverCfg.servers = serverCfg.cycleServer()
		}
	}

	serverCfg.resetRebalanceTimer(sm)
	sm.serverConfigValue.Store(serverCfg)
}

// cycleServers returns a new list of servers that has dequeued the first
// server and enqueued it at the end of the list.  cycleServers assumes the
// caller is holding the serverConfigLock.
func (sc *serverConfig) cycleServer() (servers []*server_details.ServerDetails) {
	numServers := len(servers)
	if numServers < 2 {
		// No action required
		return servers
	}

	var dequeuedServer *server_details.ServerDetails
	newServers := make([]*server_details.ServerDetails, len(servers)+1)
	dequeuedServer, newServers = servers[0], servers[1:]
	servers = append(newServers, dequeuedServer)
	return servers
}

// FindHealthyServer takes out an internal "read lock" and searches through
// the list of servers to find a healthy server.
func (sm *ServerManager) FindHealthyServer() (server *server_details.ServerDetails) {
	serverCfg := sm.getServerConfig()
	numServers := len(serverCfg.servers)
	if numServers == 0 {
		sm.logger.Printf("[ERR] consul: No servers found in the server config")
		return nil
	}

	// Find the first non-failing server in the server list.  If this is
	// not the first server a prior RPC call marked the first server as
	// failed and we're waiting for the server management task to reorder
	// a working server to the front of the list.
	for i := range serverCfg.servers {
		failCount := atomic.LoadUint64(&(serverCfg.servers[i].Disabled))
		if failCount == 0 {
			server = serverCfg.servers[i]
			break
		}
	}

	return server
}

// GetNumServers takes out an internal "read lock" and returns the number of
// servers.  numServers includes both healthy and unhealthy servers.
func (sm *ServerManager) GetNumServers() (numServers int) {
	serverCfg := sm.getServerConfig()
	numServers = len(serverCfg.servers)
	return numServers
}

// getServerConfig is a convenience method to hide the locking semantics of
// atomic.Value from the caller.
func (sm *ServerManager) getServerConfig() serverConfig {
	return sm.serverConfigValue.Load().(serverConfig)
}

// NewServerManager is the only way to safely create a new ServerManager
// struct.
//
// NOTE(sean@): We can not pass in *consul.Client due to an import cycle
func NewServerManager(logger *log.Logger, shutdownCh chan struct{}) (sm *ServerManager) {
	sm = new(ServerManager)
	// Create the initial serverConfig
	serverCfg := serverConfig{}
	sm.logger = logger
	sm.shutdownCh = shutdownCh
	sm.serverConfigValue.Store(serverCfg)
	return sm
}

// NotifyFailedServer is an exported convenience function that allows callers
// to pass in a server that has failed an RPC request and mark it as failed.
// This will initiate a background task that will optimize the failed server
// to the end of the serer list.  No locks are required here because we are
// bypassing the serverConfig and sending a message to ServerManager's
// channel.
func (sm *ServerManager) NotifyFailedServer(server *server_details.ServerDetails) {
	atomic.AddUint64(&server.Disabled, 1)
	sm.consulServersCh <- consulServersRPCError
}

// RebalanceServers takes out an internal write lock and shuffles the list of
// servers on this agent.  This allows for a redistribution of work across
// consul servers and provides a guarantee that the order list of
// ServerDetails isn't actually ordered, therefore we can sequentially walk
// the array to pick a server without all agents in the cluster dog piling on
// a single node.
func (sm *ServerManager) RebalanceServers() {
	sm.serverConfigLock.Lock()
	defer sm.serverConfigLock.Unlock()
	serverCfg := sm.getServerConfig()

	newServers := make([]*server_details.ServerDetails, len(serverCfg.servers)+1)
	copy(newServers, serverCfg.servers)

	// Shuffle the server list on server join.  Servers are selected from
	// the head of the list and are moved to the end of the list on
	// failure.
	for i := len(serverCfg.servers) - 1; i > 0; i-- {
		j := rand.Int31n(int32(i + 1))
		newServers[i], newServers[j] = newServers[j], newServers[i]
	}
	serverCfg.servers = newServers

	serverCfg.resetRebalanceTimer(sm)
	sm.serverConfigValue.Store(serverCfg)
}

// RemoveServer takes out an internal write lock and removes a server from
// the server list.  No rebalancing happens as a result of the removed server
// because we do not want a network partition which separated a server from
// this agent to cause an increase in work.  Instead we rely on the internal
// already existing semantics to handle failure detection after a server has
// been removed.
func (sm *ServerManager) RemoveServer(server *server_details.ServerDetails) {
	sm.serverConfigLock.Lock()
	defer sm.serverConfigLock.Unlock()
	serverCfg := sm.getServerConfig()

	// Remove the server if known
	n := len(serverCfg.servers)
	for i := 0; i < n; i++ {
		if serverCfg.servers[i].Name == server.Name {
			newServers := make([]*server_details.ServerDetails, len(serverCfg.servers)-1)
			copy(newServers, serverCfg.servers)

			newServers[i], newServers[n-1] = newServers[n-1], nil
			newServers = newServers[:n-1]
			serverCfg.servers = newServers

			sm.serverConfigValue.Store(serverCfg)
			return
		}
	}
}

// resetRebalanceTimer assumes:
//
// 1) the serverConfigLock is already held by the caller.
// 2) the caller will call serverConfigValue.Store()
func (sc *serverConfig) resetRebalanceTimer(sm *ServerManager) {
	numConsulServers := len(sc.servers)
	// Limit this connection's life based on the size (and health) of the
	// cluster.  Never rebalance a connection more frequently than
	// connReuseLowWatermarkDuration, and make sure we never exceed
	// clusterWideRebalanceConnsPerSec operations/s across numLANMembers.
	clusterWideRebalanceConnsPerSec := float64(numConsulServers * newRebalanceConnsPerSecPerServer)
	connReuseLowWatermarkDuration := clientRPCMinReuseDuration + lib.RandomStagger(clientRPCMinReuseDuration/clientRPCJitterFraction)
	numLANMembers := 16384 // Assume sufficiently large for now. FIXME: numLanMembers := len(c.LANMembers())
	connRebalanceTimeout := lib.RateScaledInterval(clusterWideRebalanceConnsPerSec, connReuseLowWatermarkDuration, numLANMembers)
	sm.logger.Printf("[DEBUG] consul: connection will be rebalanced in %v", connRebalanceTimeout)

	if sc.rebalanceTimer == nil {
		sc.rebalanceTimer = time.NewTimer(connRebalanceTimeout)
	} else {
		sc.rebalanceTimer.Reset(connRebalanceTimeout)
	}
}

// StartServerManager is used to start and manage the task of automatically
// shuffling and rebalance the list of consul servers.  This maintenance
// happens either when a new server is added or when a duration has been
// exceed.
func (sm *ServerManager) StartServerManager() {
	var rebalanceTimer *time.Timer
	func() {
		sm.serverConfigLock.Lock()
		defer sm.serverConfigLock.Unlock()

		serverCfgPtr := sm.serverConfigValue.Load()
		if serverCfgPtr == nil {
			panic("server config has not been initialized")
		}
		var serverCfg serverConfig
		serverCfg = serverCfgPtr.(serverConfig)
		serverCfg.resetRebalanceTimer(sm)
		rebalanceTimer = serverCfg.rebalanceTimer
		sm.serverConfigValue.Store(serverCfg)
	}()

	for {
		select {
		case e := <-sm.consulServersCh:
			switch e {
			case consulServersNodeJoin:
				sm.logger.Printf("[INFO] consul: new node joined cluster")
				sm.RebalanceServers()
			case consulServersRebalance:
				sm.logger.Printf("[INFO] consul: rebalancing servers by request")
				sm.RebalanceServers()
			case consulServersRPCError:
				sm.logger.Printf("[INFO] consul: need to find a new server to talk with")
				sm.CycleFailedServers()
				// FIXME(sean@): wtb preemptive Status.Ping
				// of servers, ideally parallel fan-out of N
				// nodes, then settle on the first node which
				// responds successfully.
				//
				// Is there a distinction between slow and
				// offline?  Do we run the Status.Ping with a
				// fixed timeout (say 30s) that way we can
				// alert administrators that they've set
				// their RPC time too low even though the
				// Ping did return successfully?
			default:
				sm.logger.Printf("[WARN] consul: unhandled LAN Serf Event: %#v", e)
			}
		case <-rebalanceTimer.C:
			sm.logger.Printf("[INFO] consul: server rebalance timeout")
			sm.RebalanceServers()

		case <-sm.shutdownCh:
			return
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2019 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package workflow

import (
	"fmt"
	"time"

	argov1alpha1 "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1"
	kbjobv1alpha2 "github.com/kubeflow/kubebench/controller/pkg/apis/kubebenchjob/v1alpha2"
	"github.com/kubeflow/kubebench/controller/pkg/util"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"github.com/kubeflow/kubebench/controller/pkg/constants"
)

type workflowInfo struct {
	experimentID        string
	namespace           string
	labels              map[string]string
	ownerReferences     []metav1.OwnerReference
	env                 []corev1.EnvVar
	volumes             []corev1.Volume
	volumeMap           map[string]corev1.Volume
	managedVolumes      []corev1.Volume
	managedVolumeMounts []corev1.VolumeMount
	managedVolumeMap    map[string]corev1.Volume
}

func newWorkflowInfo(kbjob *kbjobv1alpha2.KubebenchJob) *workflowInfo {

	// Create an easy-to-read unique experiment ID for each run of the workflow
	// The experiment ID will be placed in the label of all resources created by the workflow
	experimentID := kbjob.Name + "-" + time.Now().Format("0601021504") + "-" + util.RandString(4)

	ownerRefs := []metav1.OwnerReference{
		{
			APIVersion: argov1alpha1.SchemeGroupVersion.Group + "/" + argov1alpha1.SchemeGroupVersion.Version,
			Kind:       "Workflow",
			Name:       "{{workflow.name}}",
			UID:        "{{workflow.uid}}",
		},
	}

	labels := map[string]string{
		"kubebench.kubeflow.org/experiment-id": experimentID,
	}

	envVars := []corev1.EnvVar{
		{
			Name:  constants.WorkflowRootEnvName,
			Value: constants.WorkflowRootPath,
		},
		{
			Name:  constants.WorkflowExpRootEnvName,
			Value: constants.WorkflowExpRootPath,
		},
		{
			Name:  constants.WorkflowExpPathEnvName,
			Value: fmt.Sprintf(constants.WorkflowExpPathFmt, experimentID),
		},
		{
			Name:  constants.ExpRootEnvName,
			Value: constants.ExpRootPath,
		},
		{
			Name:  constants.ExpPathEnvName,
			Value: fmt.Sprintf(constants.ExpPathFmt, experimentID),
		},
		{
			Name:  constants.ExpConfigPathEnvName,
			Value: fmt.Sprintf(constants.ExpConfigPathFmt, experimentID),
		},
		{
			Name:  constants.ExpOutputPathEnvName,
			Value: fmt.Sprintf(constants.ExpOutputPathFmt, experimentID),
		},
		{
			Name:  constants.ExpResultPathEnvName,
			Value: fmt.Sprintf(constants.ExpResultPathFmt, experimentID),
		},
	}

	volMap := map[string]corev1.Volume{}
	for _, v := range kbjob.Spec.Volumes {
		volMap[v.Name] = v
	}
	managedVols := []corev1.Volume{}
	managedVolMnts := []corev1.VolumeMount{}
	managedVolMap := map[string]corev1.Volume{}
	managedVolCands := []*corev1.Volume{
		kbjob.Spec.ManagedVolumes.ExperimentVolume,
		kbjob.Spec.ManagedVolumes.WorkflowVolume,
	}
	managedVolMntPaths := []string{
		constants.ExpRootPath,
		constants.WorkflowExpRootPath,
	}
	for i, v := range managedVolCands {
		if v != nil {
			managedVols = append(managedVols, *v)
			volMnt := corev1.VolumeMount{
				Name:      v.Name,
				MountPath: managedVolMntPaths[i],
			}
			managedVolMnts = append(managedVolMnts, volMnt)
			managedVolMap[v.Name] = *v
		}
	}

	wfInfo := &workflowInfo{
		experimentID:        experimentID,
		namespace:           kbjob.Namespace,
		ownerReferences:     ownerRefs,
		labels:              labels,
		env:                 envVars,
		volumes:             kbjob.Spec.Volumes,
		volumeMap:           volMap,
		managedVolumes:      managedVols,
		managedVolumeMounts: managedVolMnts,
		managedVolumeMap:    managedVolMap,
	}

	return wfInfo
}
<commit_msg>Fix auto-generated environment variables (#209)<commit_after>// Copyright 2019 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package workflow

import (
	"fmt"
	"time"

	argov1alpha1 "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1"
	kbjobv1alpha2 "github.com/kubeflow/kubebench/controller/pkg/apis/kubebenchjob/v1alpha2"
	"github.com/kubeflow/kubebench/controller/pkg/util"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"github.com/kubeflow/kubebench/controller/pkg/constants"
)

type workflowInfo struct {
	experimentID        string
	namespace           string
	labels              map[string]string
	ownerReferences     []metav1.OwnerReference
	env                 []corev1.EnvVar
	volumes             []corev1.Volume
	volumeMap           map[string]corev1.Volume
	managedVolumes      []corev1.Volume
	managedVolumeMounts []corev1.VolumeMount
	managedVolumeMap    map[string]corev1.Volume
}

func newWorkflowInfo(kbjob *kbjobv1alpha2.KubebenchJob) *workflowInfo {

	// Create an easy-to-read unique experiment ID for each run of the workflow
	// The experiment ID will be placed in the label of all resources created by the workflow
	experimentID := kbjob.Name + "-" + time.Now().Format("0601021504") + "-" + util.RandString(4)

	ownerRefs := []metav1.OwnerReference{
		{
			APIVersion: argov1alpha1.SchemeGroupVersion.Group + "/" + argov1alpha1.SchemeGroupVersion.Version,
			Kind:       "Workflow",
			Name:       "{{workflow.name}}",
			UID:        "{{workflow.uid}}",
		},
	}

	labels := map[string]string{
		"kubebench.kubeflow.org/experiment-id": experimentID,
	}

	envVars := []corev1.EnvVar{
		{
			Name:  constants.ExpIDEnvName,
			Value: experimentID,
		},
		// NOTE: WorkflowRootPath is not available to use. Mount point at WorkflowExpRootPath.
		// {
		// 	Name:  constants.WorkflowRootEnvName,
		// 	Value: constants.WorkflowRootPath,
		// },
		{
			Name:  constants.WorkflowExpRootEnvName,
			Value: constants.WorkflowExpRootPath,
		},
		{
			Name:  constants.WorkflowExpPathEnvName,
			Value: fmt.Sprintf(constants.WorkflowExpPathFmt, experimentID),
		},
		{
			Name:  constants.ExpRootEnvName,
			Value: constants.ExpRootPath,
		},
		{
			Name:  constants.ExpPathEnvName,
			Value: fmt.Sprintf(constants.ExpPathFmt, experimentID),
		},
		{
			Name:  constants.ExpConfigPathEnvName,
			Value: fmt.Sprintf(constants.ExpConfigPathFmt, experimentID),
		},
		{
			Name:  constants.ExpOutputPathEnvName,
			Value: fmt.Sprintf(constants.ExpOutputPathFmt, experimentID),
		},
		{
			Name:  constants.ExpResultPathEnvName,
			Value: fmt.Sprintf(constants.ExpResultPathFmt, experimentID),
		},
	}

	volMap := map[string]corev1.Volume{}
	for _, v := range kbjob.Spec.Volumes {
		volMap[v.Name] = v
	}
	managedVols := []corev1.Volume{}
	managedVolMnts := []corev1.VolumeMount{}
	managedVolMap := map[string]corev1.Volume{}
	managedVolCands := []*corev1.Volume{
		kbjob.Spec.ManagedVolumes.ExperimentVolume,
		kbjob.Spec.ManagedVolumes.WorkflowVolume,
	}
	managedVolMntPaths := []string{
		constants.ExpRootPath,
		constants.WorkflowExpRootPath,
	}
	for i, v := range managedVolCands {
		if v != nil {
			managedVols = append(managedVols, *v)
			volMnt := corev1.VolumeMount{
				Name:      v.Name,
				MountPath: managedVolMntPaths[i],
			}
			managedVolMnts = append(managedVolMnts, volMnt)
			managedVolMap[v.Name] = *v
		}
	}

	wfInfo := &workflowInfo{
		experimentID:        experimentID,
		namespace:           kbjob.Namespace,
		ownerReferences:     ownerRefs,
		labels:              labels,
		env:                 envVars,
		volumes:             kbjob.Spec.Volumes,
		volumeMap:           volMap,
		managedVolumes:      managedVols,
		managedVolumeMounts: managedVolMnts,
		managedVolumeMap:    managedVolMap,
	}

	return wfInfo
}
<|endoftext|>