<commit_before>package workspace

import (
	"errors"
	"fmt"
	"net/url"
	"strings"
	"sync"

	"github.com/jmoiron/sqlx"
	log "github.com/sirupsen/logrus"
	"github.com/skeema/tengo"
)

// LocalDocker is a Workspace created inside of a Docker container on localhost.
// The schema is dropped when done interacting with the workspace in Cleanup(),
// but the container remains running. The container may optionally be stopped
// or destroyed via Shutdown().
type LocalDocker struct {
	schemaName        string
	d                 *tengo.DockerizedInstance
	releaseLock       releaseFunc
	cleanupAction     CleanupAction
	defaultConnParams string
}

var cstore struct {
	dockerClient *tengo.DockerClient
	containers   map[string]*tengo.DockerizedInstance
	sync.Mutex
}

// NewLocalDocker finds or creates a containerized MySQL instance, creates a
// temporary schema on it, and returns it.
func NewLocalDocker(opts Options) (ld *LocalDocker, err error) {
	if !opts.Flavor.Supported() {
		return nil, fmt.Errorf("NewLocalDocker: unsupported flavor %s", opts.Flavor)
	}

	cstore.Lock()
	defer cstore.Unlock()
	if cstore.dockerClient == nil {
		if cstore.dockerClient, err = tengo.NewDockerClient(tengo.DockerClientOptions{}); err != nil {
			return
		}
		cstore.containers = make(map[string]*tengo.DockerizedInstance)
		tengo.UseFilteredDriverLogger()
	}

	ld = &LocalDocker{
		schemaName:        opts.SchemaName,
		cleanupAction:     opts.CleanupAction,
		defaultConnParams: opts.DefaultConnParams,
	}
	image := opts.Flavor.String()
	if opts.ContainerName == "" {
		opts.ContainerName = fmt.Sprintf("skeema-%s", strings.Replace(image, ":", "-", -1))
	}
	if cstore.containers[opts.ContainerName] != nil {
		ld.d = cstore.containers[opts.ContainerName]
	} else {
		log.Infof("Using container %s (image=%s) for workspace operations", opts.ContainerName, image)
		ld.d, err = cstore.dockerClient.GetOrCreateInstance(tengo.DockerizedInstanceOptions{
			Name:              opts.ContainerName,
			Image:             image,
			RootPassword:      opts.RootPassword,
			DefaultConnParams: "", // intentionally not set here; see important comment in ConnectionPool()
		})
		if ld.d != nil {
			cstore.containers[opts.ContainerName] = ld.d
			RegisterShutdownFunc(ld.shutdown)
		}
		if err != nil {
			return nil, err
		}
	}

	lockName := fmt.Sprintf("skeema.%s", ld.schemaName)
	if ld.releaseLock, err = getLock(ld.d.Instance, lockName, opts.LockWaitTimeout); err != nil {
		return nil, fmt.Errorf("Unable to obtain lock on %s: %s", ld.d.Instance, err)
	}
	// If this function errors, don't continue to hold the lock
	defer func() {
		if err != nil {
			ld.releaseLock()
			ld = nil
		}
	}()

	if has, err := ld.d.HasSchema(ld.schemaName); err != nil {
		return ld, fmt.Errorf("Unable to check for existence of temp schema on %s: %s", ld.d.Instance, err)
	} else if has {
		// Attempt to drop the schema, so we can recreate it below. (This is safer
		// than attempting to re-use the schema.) Fail if any tables actually have
		// 1 or more rows.
		dropOpts := tengo.BulkDropOptions{
			MaxConcurrency: 10,
			OnlyIfEmpty:    true,
			SkipBinlog:     true,
		}
		if err := ld.d.DropSchema(ld.schemaName, dropOpts); err != nil {
			return ld, fmt.Errorf("Cannot drop existing temporary schema on %s: %s", ld.d.Instance, err)
		}
	}

	createOpts := tengo.SchemaCreationOptions{
		DefaultCharSet:   opts.DefaultCharacterSet,
		DefaultCollation: opts.DefaultCollation,
		SkipBinlog:       true,
	}
	_, err = ld.d.CreateSchema(ld.schemaName, createOpts)
	if err != nil {
		return ld, fmt.Errorf("Cannot create temporary schema on %s: %s", ld.d.Instance, err)
	}
	return ld, nil
}

// ConnectionPool returns a connection pool (*sqlx.DB) to the temporary
// workspace schema, using the supplied connection params (which may be blank).
func (ld *LocalDocker) ConnectionPool(params string) (*sqlx.DB, error) {
	// User-configurable default connection params are stored in the LocalDocker
	// value, NOT in the tengo.DockerizedInstance. This permits re-use of the same
	// DockerizedInstance in multiple LocalDocker workspaces, even if the
	// workspaces have different connection params (e.g. due to being generated by
	// different sibling subdirectories with differing configurations).
	// So, here we must merge the params arg (callsite-dependent) over top of the
	// LocalDocker params (dir-dependent).
	var finalParams string
	if ld.defaultConnParams == "" && params == "" {
		// By default, disable TLS for connections to the DockerizedInstance, since
		// we know it's on the local machine
		finalParams = "tls=false"
	} else {
		v, err := url.ParseQuery(ld.defaultConnParams)
		if err != nil {
			return nil, err
		}
		overrides, err := url.ParseQuery(params)
		if err != nil {
			return nil, err
		}
		for name := range overrides {
			v.Set(name, overrides.Get(name))
		}
		if v.Get("tls") == "" {
			v.Set("tls", "false")
		}
		finalParams = v.Encode()
	}
	return ld.d.Connect(ld.schemaName, finalParams)
}

// IntrospectSchema introspects and returns the temporary workspace schema.
func (ld *LocalDocker) IntrospectSchema() (*tengo.Schema, error) {
	return ld.d.Schema(ld.schemaName)
}

// Cleanup drops the temporary schema from the Dockerized instance. If any
// tables have any rows in the temp schema, the cleanup aborts and an error is
// returned.
// Cleanup does not handle stopping or destroying the container. If requested,
// that is handled by Shutdown() instead, so that containers aren't needlessly
// created and stopped/destroyed multiple times during a program's execution.
func (ld *LocalDocker) Cleanup() error {
	if ld.releaseLock == nil {
		return errors.New("Cleanup() called multiple times on same LocalDocker")
	}
	defer func() {
		ld.releaseLock()
		ld.releaseLock = nil
	}()

	dropOpts := tengo.BulkDropOptions{
		MaxConcurrency: 10,
		OnlyIfEmpty:    true,
		SkipBinlog:     true,
	}
	if err := ld.d.DropSchema(ld.schemaName, dropOpts); err != nil {
		return fmt.Errorf("Cannot drop temporary schema on %s: %s", ld.d.Instance, err)
	}
	return nil
}

// shutdown handles shutdown logic for a specific LocalDocker instance. A single
// string arg may optionally be supplied as a container name prefix: if the
// container name does not begin with the prefix, no shutdown occurs.
func (ld *LocalDocker) shutdown(args ...interface{}) bool {
	if len(args) > 0 {
		if prefix, ok := args[0].(string); !ok || !strings.HasPrefix(ld.d.Name, prefix) {
			return false
		}
	}

	cstore.Lock()
	defer cstore.Unlock()

	if ld.cleanupAction == CleanupActionStop {
		log.Infof("Stopping container %s", ld.d.Name)
		ld.d.Stop()
	} else if ld.cleanupAction == CleanupActionDestroy {
		log.Infof("Destroying container %s", ld.d.Name)
		ld.d.Destroy()
	}
	delete(cstore.containers, ld.d.Name)
	return true
}
<commit_msg>workspace=docker: properly disable workspace conn TLS<commit_after>package workspace

import (
	"errors"
	"fmt"
	"net/url"
	"strings"
	"sync"

	"github.com/jmoiron/sqlx"
	log "github.com/sirupsen/logrus"
	"github.com/skeema/tengo"
)

// LocalDocker is a Workspace created inside of a Docker container on localhost.
// The schema is dropped when done interacting with the workspace in Cleanup(),
// but the container remains running. The container may optionally be stopped
// or destroyed via Shutdown().
type LocalDocker struct {
	schemaName        string
	d                 *tengo.DockerizedInstance
	releaseLock       releaseFunc
	cleanupAction     CleanupAction
	defaultConnParams string
}

var cstore struct {
	dockerClient *tengo.DockerClient
	containers   map[string]*tengo.DockerizedInstance
	sync.Mutex
}

// NewLocalDocker finds or creates a containerized MySQL instance, creates a
// temporary schema on it, and returns it.
func NewLocalDocker(opts Options) (ld *LocalDocker, err error) {
	if !opts.Flavor.Supported() {
		return nil, fmt.Errorf("NewLocalDocker: unsupported flavor %s", opts.Flavor)
	}

	cstore.Lock()
	defer cstore.Unlock()
	if cstore.dockerClient == nil {
		if cstore.dockerClient, err = tengo.NewDockerClient(tengo.DockerClientOptions{}); err != nil {
			return
		}
		cstore.containers = make(map[string]*tengo.DockerizedInstance)
		tengo.UseFilteredDriverLogger()
	}

	ld = &LocalDocker{
		schemaName:        opts.SchemaName,
		cleanupAction:     opts.CleanupAction,
		defaultConnParams: opts.DefaultConnParams,
	}
	image := opts.Flavor.String()
	if opts.ContainerName == "" {
		opts.ContainerName = fmt.Sprintf("skeema-%s", strings.Replace(image, ":", "-", -1))
	}
	if cstore.containers[opts.ContainerName] != nil {
		ld.d = cstore.containers[opts.ContainerName]
	} else {
		log.Infof("Using container %s (image=%s) for workspace operations", opts.ContainerName, image)
		ld.d, err = cstore.dockerClient.GetOrCreateInstance(tengo.DockerizedInstanceOptions{
			Name:              opts.ContainerName,
			Image:             image,
			RootPassword:      opts.RootPassword,
			DefaultConnParams: "", // intentionally not set here; see important comment in ConnectionPool()
		})
		if ld.d != nil {
			cstore.containers[opts.ContainerName] = ld.d
			RegisterShutdownFunc(ld.shutdown)
		}
		if err != nil {
			return nil, err
		}
	}

	lockName := fmt.Sprintf("skeema.%s", ld.schemaName)
	if ld.releaseLock, err = getLock(ld.d.Instance, lockName, opts.LockWaitTimeout); err != nil {
		return nil, fmt.Errorf("Unable to obtain lock on %s: %s", ld.d.Instance, err)
	}
	// If this function errors, don't continue to hold the lock
	defer func() {
		if err != nil {
			ld.releaseLock()
			ld = nil
		}
	}()

	if has, err := ld.d.HasSchema(ld.schemaName); err != nil {
		return ld, fmt.Errorf("Unable to check for existence of temp schema on %s: %s", ld.d.Instance, err)
	} else if has {
		// Attempt to drop the schema, so we can recreate it below. (This is safer
		// than attempting to re-use the schema.) Fail if any tables actually have
		// 1 or more rows.
		dropOpts := tengo.BulkDropOptions{
			MaxConcurrency: 10,
			OnlyIfEmpty:    true,
			SkipBinlog:     true,
		}
		if err := ld.d.DropSchema(ld.schemaName, dropOpts); err != nil {
			return ld, fmt.Errorf("Cannot drop existing temporary schema on %s: %s", ld.d.Instance, err)
		}
	}

	createOpts := tengo.SchemaCreationOptions{
		DefaultCharSet:   opts.DefaultCharacterSet,
		DefaultCollation: opts.DefaultCollation,
		SkipBinlog:       true,
	}
	_, err = ld.d.CreateSchema(ld.schemaName, createOpts)
	if err != nil {
		return ld, fmt.Errorf("Cannot create temporary schema on %s: %s", ld.d.Instance, err)
	}
	return ld, nil
}

// ConnectionPool returns a connection pool (*sqlx.DB) to the temporary
// workspace schema, using the supplied connection params (which may be blank).
func (ld *LocalDocker) ConnectionPool(params string) (*sqlx.DB, error) {
	// User-configurable default connection params are stored in the LocalDocker
	// value, NOT in the tengo.DockerizedInstance. This permits re-use of the same
	// DockerizedInstance in multiple LocalDocker workspaces, even if the
	// workspaces have different connection params (e.g. due to being generated by
	// different sibling subdirectories with differing configurations).
	// So, here we must merge the params arg (callsite-dependent) over top of the
	// LocalDocker params (dir-dependent).
	var finalParams string
	if ld.defaultConnParams == "" && params == "" {
		// By default, disable TLS for connections to the DockerizedInstance, since
		// we know it's on the local machine
		finalParams = "tls=false"
	} else {
		v, err := url.ParseQuery(ld.defaultConnParams)
		if err != nil {
			return nil, err
		}

		// Forcibly disable TLS, regardless of what was in ld.defaultConnParams.
		// This is necessary since ld.defaultConnParams is typically populated using
		// Dir.InstanceDefaultParams() which sets tls=preferred by default.
		v.Set("tls", "false")

		// Apply overrides from params arg
		overrides, err := url.ParseQuery(params)
		if err != nil {
			return nil, err
		}
		for name := range overrides {
			v.Set(name, overrides.Get(name))
		}
		finalParams = v.Encode()
	}
	return ld.d.Connect(ld.schemaName, finalParams)
}

// IntrospectSchema introspects and returns the temporary workspace schema.
func (ld *LocalDocker) IntrospectSchema() (*tengo.Schema, error) {
	return ld.d.Schema(ld.schemaName)
}

// Cleanup drops the temporary schema from the Dockerized instance. If any
// tables have any rows in the temp schema, the cleanup aborts and an error is
// returned.
// Cleanup does not handle stopping or destroying the container. If requested,
// that is handled by Shutdown() instead, so that containers aren't needlessly
// created and stopped/destroyed multiple times during a program's execution.
func (ld *LocalDocker) Cleanup() error {
	if ld.releaseLock == nil {
		return errors.New("Cleanup() called multiple times on same LocalDocker")
	}
	defer func() {
		ld.releaseLock()
		ld.releaseLock = nil
	}()

	dropOpts := tengo.BulkDropOptions{
		MaxConcurrency: 10,
		OnlyIfEmpty:    true,
		SkipBinlog:     true,
	}
	if err := ld.d.DropSchema(ld.schemaName, dropOpts); err != nil {
		return fmt.Errorf("Cannot drop temporary schema on %s: %s", ld.d.Instance, err)
	}
	return nil
}

// shutdown handles shutdown logic for a specific LocalDocker instance. A single
// string arg may optionally be supplied as a container name prefix: if the
// container name does not begin with the prefix, no shutdown occurs.
func (ld *LocalDocker) shutdown(args ...interface{}) bool {
	if len(args) > 0 {
		if prefix, ok := args[0].(string); !ok || !strings.HasPrefix(ld.d.Name, prefix) {
			return false
		}
	}

	cstore.Lock()
	defer cstore.Unlock()

	if ld.cleanupAction == CleanupActionStop {
		log.Infof("Stopping container %s", ld.d.Name)
		ld.d.Stop()
	} else if ld.cleanupAction == CleanupActionDestroy {
		log.Infof("Destroying container %s", ld.d.Name)
		ld.d.Destroy()
	}
	delete(cstore.containers, ld.d.Name)
	return true
}
<|endoftext|>


<commit_before>// dsrt.go -- directoy sort in reverse date order.  IE, newest is first.

package main

import (
	"bufio"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/user"
	"path/filepath"
	"runtime"
	"sort"
	"strconv"
	"strings"
)

const LastAltered = "13 Dec 2017"

/*
Revision History
----------------
20 Apr 17 -- Started writing dsize rtn, based on dirlist.go
21 Apr 17 -- Now tweaking the output format.  And used flag package.  One as a pointer and one as a value, just to learn them.
22 Apr 17 -- Coded the use of the first non flag commandline param,  which is all I need.  Note that the flag must appear before the non-flag param, else the flag is ignored.
22 Apr 17 -- Now writing dsrt, to function similarly to dsort.
24 Apr 17 -- Now adding file matching, like "dir" or "ls" does.
25 Apr 17 -- Now adding sort by size as an option, like -s, and commas
26 Apr 17 -- Noticed that the match routine is case sensitive.  I don't like that.
27 Apr 17 -- commandline now allows a file spec.  I intend this for Windows.  I'll see how it goes.
19 May 17 -- Will now show the uid:gid for linux.
20 May 17 -- Turns out that (*syscall.Stat_t) only compiles on linux.  Time for platform specific code.
21 May 17 -- Cross compiling to GOARCH=386, and the uid and User routines won't work.
 2 Sep 17 -- Added timestamp detection code I first wrote for gastricgo.
18 Oct 17 -- Added filesize totals
22 Oct 17 -- Made default numlines of 40.
23 Oct 17 -- Broadened the defaults so that linux default is 40 and windows default is 50.
12 Dec 17 -- Added -d and -D flags to mean directory and nofilename output, respectively.
*/

// FIS is a FileInfo slice, as in os.FileInfo
type FISlice []os.FileInfo
type FISliceDate []os.FileInfo
type FISliceSize []os.FileInfo

func (f FISliceDate) Less(i, j int) bool {
	return f[i].ModTime().UnixNano() > f[j].ModTime().UnixNano() // I want a reverse sort, newest first
}

func (f FISliceDate) Swap(i, j int) {
	f[i], f[j] = f[j], f[i]
}

func (f FISliceDate) Len() int {
	return len(f)
}

func (f FISliceSize) Less(i, j int) bool {
	return f[i].Size() > f[j].Size() // I want a reverse sort, largest first
}

func (f FISliceSize) Swap(i, j int) {
	f[i], f[j] = f[j], f[i]
}

func (f FISliceSize) Len() int {
	return len(f)
}

func main() {
	const defaultlineswin = 50
	const defaultlineslinux = 40
	var numlines int
	var userptr *user.User
	var files FISlice
	var filesDate FISliceDate
	var filesSize FISliceSize
	var err error
	var count int
	var SizeTotal int64

	uid := 0
	gid := 0
	systemStr := ""
	linuxflag := runtime.GOOS == "linux"
	if linuxflag {
		systemStr = "Linux"
		numlines = defaultlineslinux
	} else if runtime.GOOS == "windows" {
		systemStr = "Windows"
		numlines = defaultlineswin
	} else {
		systemStr = "Mac, maybe"
		numlines = defaultlineslinux
	}

	if runtime.GOARCH == "amd64" {
		uid = os.Getuid() // int
		gid = os.Getgid() // int
		userptr, err = user.Current()
		if err != nil {
			fmt.Println(" user.Current error is ", err, "Exiting.")
			os.Exit(1)
		}
	}

	var revflag = flag.Bool("r", false, "reverse the sort, ie, oldest or smallest is first") // Ptr

	var RevFlag bool
	flag.BoolVar(&RevFlag, "R", false, "Reverse the sort, ie, oldest or smallest is first") // Value

	var nlines = flag.Int("n", numlines, "number of lines to display") // Ptr

	var NLines int
	flag.IntVar(&NLines, "N", numlines, "number of lines to display") // Value

	var helpflag = flag.Bool("h", false, "print help message") // pointer
	var HelpFlag bool
	flag.BoolVar(&HelpFlag, "H", false, "print help message")

	var sizeflag = flag.Bool("s", false, "sort by size instead of by date") // pointer
	var SizeFlag bool
	flag.BoolVar(&SizeFlag, "S", false, "sort by size instead of by date")

	var DirListFlag = flag.Bool("d", false, "include directories in the output listing")
	var FilenameListFlag bool
	flag.BoolVar(&FilenameListFlag, "D", false, "include filenames in the output listing")

	flag.Parse()

	fmt.Println(" dsrt will display sorted by date or size.  Written in Go.  LastAltered ", LastAltered)
	execname, _ := os.Executable()
	ExecFI, _ := os.Stat(execname)
	ExecTimeStamp := ExecFI.ModTime().Format("Mon Jan 2 2006 15:04:05 MST")
	fmt.Println(ExecFI.Name(), "timestamp is", ExecTimeStamp, ".  Full exec is", execname)
	fmt.Println()

	if *helpflag || HelpFlag {
		flag.PrintDefaults()
		if runtime.GOARCH == "amd64" {
			fmt.Printf("uid=%d, gid=%d, on a computer running %s for %s:%s Username %s, Name %s, HomeDir %s \n",
				uid, gid, systemStr, userptr.Uid, userptr.Gid, userptr.Username, userptr.Name, userptr.HomeDir)
		}

	}

	Reverse := *revflag || RevFlag
	SizeSort := *sizeflag || SizeFlag

	NumLines := numlines
	if *nlines != numlines {
		NumLines = *nlines
	} else if NLines != numlines {
		NumLines = NLines
	}

	Dirlist := *DirListFlag || FilenameListFlag // if -D entered then this expression also needs to be true.
	FilenameList := !FilenameListFlag           // need to reverse the flag.

	askforinput := true

	CleanDirName := "." + string(filepath.Separator)
	CleanFileName := ""
	commandline := flag.Arg(0) // this only gets the first non flag argument.  That's all I want
	if len(commandline) > 0 {
		//		CleanDirName = filepath.Clean(commandline)
		CleanDirName, CleanFileName = filepath.Split(commandline)
		CleanFileName = strings.ToUpper(CleanFileName)
		askforinput = false
	}

	if askforinput {
		// Asking for input so don't have to worry about command line globbing
		fmt.Print(" Enter input for globbing: ")
		scanner := bufio.NewScanner(os.Stdin)
		scanner.Scan()
		newtext := scanner.Text()
		if err = scanner.Err(); err != nil {
			fmt.Fprintln(os.Stderr, " reading std input: ", err)
			os.Exit(1)
		}
		if len(newtext) > 0 {
			// time to do the stuff I'm writing this pgm for
			CleanDirName, CleanFileName = filepath.Split(newtext)
			CleanFileName = strings.ToUpper(CleanFileName)
		}

	}

	if len(CleanDirName) == 0 {
		CleanDirName = "." + string(filepath.Separator)
	}

	if len(CleanFileName) == 0 {
		CleanFileName = "*"
	}

	if SizeSort {
		filesSize, err = ioutil.ReadDir(CleanDirName)
		if err != nil {
			log.Fatal(err)
		}
		if Reverse {
			sort.Sort(sort.Reverse(filesSize))
		} else {
			sort.Sort(filesSize)
		}
		files = FISlice(filesSize)
	} else {
		filesDate, err = ioutil.ReadDir(CleanDirName)
		if err != nil {
			log.Fatal(err)
		}
		if Reverse {
			sort.Sort(sort.Reverse(filesDate))
		} else {
			sort.Sort(filesDate)
		}
		files = FISlice(filesDate)
	}

	fmt.Println(" Dirname is", CleanDirName)

	for _, f := range files {
		NAME := strings.ToUpper(f.Name())
		//		if BOOL, _ := filepath.Match(CleanFileName, NAME); BOOL && f.Mode().IsRegular() {
		if BOOL, _ := filepath.Match(CleanFileName, NAME); BOOL {
			s := f.ModTime().Format("Jan-02-2006 15:04:05")
			sizeint := 0
			sizestr := ""
			if f.Mode().IsRegular() { // only sum regular files, not dir or symlink entries.
				SizeTotal += f.Size()
				sizeint = int(f.Size())
				sizestr = strconv.Itoa(sizeint)
				if sizeint > 100000 {
					sizestr = AddCommas(sizestr)
				}
			}

			usernameStr, groupnameStr := "", ""
			if runtime.GOARCH == "amd64" {
				usernameStr, groupnameStr = GetUserGroupStr(f)
			}

			if linuxflag {
				if Dirlist && f.IsDir() {
					fmt.Printf("%10v %s:%s %15s %s <%s>\n", f.Mode(), usernameStr, groupnameStr, sizestr, s, f.Name())
				} else if FilenameList && f.Mode().IsRegular() { // altered
					fmt.Printf("%10v %s:%s %15s %s %s\n", f.Mode(), usernameStr, groupnameStr, sizestr, s, f.Name())
				} else if Dirlist && !f.Mode().IsRegular() { // it's a symlink
					fmt.Printf("%10v %s:%s %15s %s (%s)\n", f.Mode(), usernameStr, groupnameStr, sizestr, s, f.Name())
				}
			} else { // must be windows because this won't compile on Mac.
				if Dirlist && f.IsDir() {
					fmt.Printf("%15s %s <%s>\n", sizestr, s, f.Name())
				} else if FilenameList {
					fmt.Printf("%15s %s %s\n", sizestr, s, f.Name())
				}
			}
			count++
			if count > NumLines {
				break
			}
		}
	}

	s := fmt.Sprintf("%d", SizeTotal)
	if SizeTotal > 100000 {
		s = AddCommas(s)
	}
	fmt.Println(" File Size total =", s)
} // end main dsrt

//-------------------------------------------------------------------- InsertByteSlice
func InsertIntoByteSlice(slice, insertion []byte, index int) []byte {
	return append(slice[:index], append(insertion, slice[index:]...)...)
}

//---------------------------------------------------------------------- AddCommas
func AddCommas(instr string) string {
	var Comma []byte = []byte{','}

	BS := make([]byte, 0, 15)
	BS = append(BS, instr...)

	i := len(BS)

	for NumberOfCommas := i / 3; (NumberOfCommas > 0) && (i > 3); NumberOfCommas-- {
		i -= 3
		BS = InsertIntoByteSlice(BS, Comma, i)
	}
	return string(BS)
} // AddCommas
//---------------------------------------------------------------------------------------------------

// ---------------------------- GetIDname -----------------------------------------------------------
func GetIDname(uidStr string) string {

	if len(uidStr) == 0 {
		return ""
	}
	ptrToUser, err := user.LookupId(uidStr)
	if err != nil {
		panic("uid not found")
	}

	idname := ptrToUser.Username
	return idname

} // GetIDname

/*
package path
func Match

func Match(pattern, name string) (matched bool, err error)

Match reports whether name matches the shell file name pattern.  The pattern syntax is:

pattern:
	{ term }
term:
	'*'         matches any sequence of non-/ characters
	'?'         matches any single non-/ character
	'[' [ '^' ] { character-range } ']'
	            character class (must be non-empty)
	c           matches character c (c != '*', '?', '\\', '[')
	'\\' c      matches character c

character-range:
	c           matches character c (c != '\\', '-', ']')
	'\\' c      matches character c
	lo '-' hi   matches character c for lo <= c <= hi

Match requires pattern to match all of name, not just a substring.  The only possible returned error is ErrBadPattern, when pattern is malformed.

package os
type FileInfo

type FileInfo interface {
        Name() string       // base name of the file
        Size() int64        // length in bytes for regular files; system-dependent for others
        Mode() FileMode     // file mode bits
        ModTime() time.Time // modification time
        IsDir() bool        // abbreviation for Mode().IsDir()
        Sys() interface{}   // underlying data source (can return nil)
}

A FileInfo describes a file and is returned by Stat and Lstat.

func Lstat

func Lstat(name string) (FileInfo, error)

Lstat returns a FileInfo describing the named file.  If the file is a symbolic link, the returned FileInfo describes the symbolic link.  Lstat makes no attempt to follow the link.
If there is an error, it will be of type *PathError.

func Stat

func Stat(name string) (FileInfo, error)

Stat returns a FileInfo describing the named file.  If there is an error, it will be of type *PathError.

The insight I had with my append troubles that the 1 slice entries were empty, is that when I used append, it would do just that to the end of the slice, and ignore the empty slices.
I needed to make the slice as empty for this to work.  So I am directly assigning the DirEntries slice, and appending the FileNames slice, to make sure that these both are doing what I want.
This code is now doing exactly what I want.  I guess there is no substitute for playing with myself.  Wait, that didn't come out right.  Or did it.

package os/user
type User struct {
  Uid string
  Gid string
  Username string // login name
  Name string     // full or display name.  It may be blank.
  HomeDir string
}
*/
<commit_msg>modified:   dsrt/dsrt.go -- now works on Windows, too.<commit_after>// dsrt.go -- directoy sort in reverse date order.  IE, newest is first.

package main

import (
	"bufio"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/user"
	"path/filepath"
	"runtime"
	"sort"
	"strconv"
	"strings"
)

const LastAltered = "13 Dec 2017"

/*
Revision History
----------------
20 Apr 17 -- Started writing dsize rtn, based on dirlist.go
21 Apr 17 -- Now tweaking the output format.  And used flag package.  One as a pointer and one as a value, just to learn them.
22 Apr 17 -- Coded the use of the first non flag commandline param,  which is all I need.  Note that the flag must appear before the non-flag param, else the flag is ignored.
22 Apr 17 -- Now writing dsrt, to function similarly to dsort.
24 Apr 17 -- Now adding file matching, like "dir" or "ls" does.
25 Apr 17 -- Now adding sort by size as an option, like -s, and commas
26 Apr 17 -- Noticed that the match routine is case sensitive.  I don't like that.
27 Apr 17 -- commandline now allows a file spec.  I intend this for Windows.  I'll see how it goes.
19 May 17 -- Will now show the uid:gid for linux.
20 May 17 -- Turns out that (*syscall.Stat_t) only compiles on linux.  Time for platform specific code.
21 May 17 -- Cross compiling to GOARCH=386, and the uid and User routines won't work.
 2 Sep 17 -- Added timestamp detection code I first wrote for gastricgo.
18 Oct 17 -- Added filesize totals
22 Oct 17 -- Made default numlines of 40.
23 Oct 17 -- Broadened the defaults so that linux default is 40 and windows default is 50.
12 Dec 17 -- Added -d and -D flags to mean directory and nofilename output, respectively.
*/

// FIS is a FileInfo slice, as in os.FileInfo
type FISlice []os.FileInfo
type FISliceDate []os.FileInfo
type FISliceSize []os.FileInfo

func (f FISliceDate) Less(i, j int) bool {
	return f[i].ModTime().UnixNano() > f[j].ModTime().UnixNano() // I want a reverse sort, newest first
}

func (f FISliceDate) Swap(i, j int) {
	f[i], f[j] = f[j], f[i]
}

func (f FISliceDate) Len() int {
	return len(f)
}

func (f FISliceSize) Less(i, j int) bool {
	return f[i].Size() > f[j].Size() // I want a reverse sort, largest first
}

func (f FISliceSize) Swap(i, j int) {
	f[i], f[j] = f[j], f[i]
}

func (f FISliceSize) Len() int {
	return len(f)
}

func main() {
	const defaultlineswin = 50
	const defaultlineslinux = 40
	var numlines int
	var userptr *user.User
	var files FISlice
	var filesDate FISliceDate
	var filesSize FISliceSize
	var err error
	var count int
	var SizeTotal int64

	uid := 0
	gid := 0
	systemStr := ""
	linuxflag := runtime.GOOS == "linux"
	if linuxflag {
		systemStr = "Linux"
		numlines = defaultlineslinux
	} else if runtime.GOOS == "windows" {
		systemStr = "Windows"
		numlines = defaultlineswin
	} else {
		systemStr = "Mac, maybe"
		numlines = defaultlineslinux
	}

	if runtime.GOARCH == "amd64" {
		uid = os.Getuid() // int
		gid = os.Getgid() // int
		userptr, err = user.Current()
		if err != nil {
			fmt.Println(" user.Current error is ", err, "Exiting.")
			os.Exit(1)
		}
	}

	var revflag = flag.Bool("r", false, "reverse the sort, ie, oldest or smallest is first") // Ptr

	var RevFlag bool
	flag.BoolVar(&RevFlag, "R", false, "Reverse the sort, ie, oldest or smallest is first") // Value

	var nlines = flag.Int("n", numlines, "number of lines to display") // Ptr

	var NLines int
	flag.IntVar(&NLines, "N", numlines, "number of lines to display") // Value

	var helpflag = flag.Bool("h", false, "print help message") // pointer
	var HelpFlag bool
	flag.BoolVar(&HelpFlag, "H", false, "print help message")

	var sizeflag = flag.Bool("s", false, "sort by size instead of by date") // pointer
	var SizeFlag bool
	flag.BoolVar(&SizeFlag, "S", false, "sort by size instead of by date")

	var DirListFlag = flag.Bool("d", false, "include directories in the output listing")
	var FilenameListFlag bool
	flag.BoolVar(&FilenameListFlag, "D", false, "include filenames in the output listing")

	flag.Parse()

	fmt.Println(" dsrt will display sorted by date or size.  Written in Go.  LastAltered ", LastAltered)
	execname, _ := os.Executable()
	ExecFI, _ := os.Stat(execname)
	ExecTimeStamp := ExecFI.ModTime().Format("Mon Jan 2 2006 15:04:05 MST")
	fmt.Println(ExecFI.Name(), "timestamp is", ExecTimeStamp, ".  Full exec is", execname)
	fmt.Println()

	if *helpflag || HelpFlag {
		flag.PrintDefaults()
		if runtime.GOARCH == "amd64" {
			fmt.Printf("uid=%d, gid=%d, on a computer running %s for %s:%s Username %s, Name %s, HomeDir %s \n",
				uid, gid, systemStr, userptr.Uid, userptr.Gid, userptr.Username, userptr.Name, userptr.HomeDir)
		}

	}

	Reverse := *revflag || RevFlag
	SizeSort := *sizeflag || SizeFlag

	NumLines := numlines
	if *nlines != numlines {
		NumLines = *nlines
	} else if NLines != numlines {
		NumLines = NLines
	}

	Dirlist := *DirListFlag || FilenameListFlag // if -D entered then this expression also needs to be true.
	FilenameList := !FilenameListFlag           // need to reverse the flag.

	askforinput := true

	CleanDirName := "." + string(filepath.Separator)
	CleanFileName := ""
	commandline := flag.Arg(0) // this only gets the first non flag argument.  That's all I want
	if len(commandline) > 0 {
		//		CleanDirName = filepath.Clean(commandline)
		CleanDirName, CleanFileName = filepath.Split(commandline)
		CleanFileName = strings.ToUpper(CleanFileName)
		askforinput = false
	}

	if askforinput {
		// Asking for input so don't have to worry about command line globbing
		fmt.Print(" Enter input for globbing: ")
		scanner := bufio.NewScanner(os.Stdin)
		scanner.Scan()
		newtext := scanner.Text()
		if err = scanner.Err(); err != nil {
			fmt.Fprintln(os.Stderr, " reading std input: ", err)
			os.Exit(1)
		}
		if len(newtext) > 0 {
			// time to do the stuff I'm writing this pgm for
			CleanDirName, CleanFileName = filepath.Split(newtext)
			CleanFileName = strings.ToUpper(CleanFileName)
		}

	}

	if len(CleanDirName) == 0 {
		CleanDirName = "." + string(filepath.Separator)
	}

	if len(CleanFileName) == 0 {
		CleanFileName = "*"
	}

	if SizeSort {
		filesSize, err = ioutil.ReadDir(CleanDirName)
		if err != nil {
			log.Fatal(err)
		}
		if Reverse {
			sort.Sort(sort.Reverse(filesSize))
		} else {
			sort.Sort(filesSize)
		}
		files = FISlice(filesSize)
	} else {
		filesDate, err = ioutil.ReadDir(CleanDirName)
		if err != nil {
			log.Fatal(err)
		}
		if Reverse {
			sort.Sort(sort.Reverse(filesDate))
		} else {
			sort.Sort(filesDate)
		}
		files = FISlice(filesDate)
	}

	fmt.Println(" Dirname is", CleanDirName)

	for _, f := range files {
		NAME := strings.ToUpper(f.Name())
		//		if BOOL, _ := filepath.Match(CleanFileName, NAME); BOOL && f.Mode().IsRegular() {
		if BOOL, _ := filepath.Match(CleanFileName, NAME); BOOL {
			s := f.ModTime().Format("Jan-02-2006 15:04:05")
			sizeint := 0
			sizestr := ""
			if f.Mode().IsRegular() { // only sum regular files, not dir or symlink entries.
				SizeTotal += f.Size()
				sizeint = int(f.Size())
				sizestr = strconv.Itoa(sizeint)
				if sizeint > 100000 {
					sizestr = AddCommas(sizestr)
				}
			}

			usernameStr, groupnameStr := "", ""
			if runtime.GOARCH == "amd64" {
				usernameStr, groupnameStr = GetUserGroupStr(f)
			}

			if linuxflag {
				if Dirlist && f.IsDir() {
					fmt.Printf("%10v %s:%s %15s %s <%s>\n", f.Mode(), usernameStr, groupnameStr, sizestr, s, f.Name())
				} else if FilenameList && f.Mode().IsRegular() { // altered
					fmt.Printf("%10v %s:%s %15s %s %s\n", f.Mode(), usernameStr, groupnameStr, sizestr, s, f.Name())
				} else if Dirlist && !f.Mode().IsRegular() { // it's a symlink
					fmt.Printf("%10v %s:%s %15s %s (%s)\n", f.Mode(), usernameStr, groupnameStr, sizestr, s, f.Name())
				}
			} else { // must be windows because I don't think this will compile on Mac.
				if Dirlist && f.IsDir() {
					fmt.Printf("%15s %s <%s>\n", sizestr, s, f.Name())
				} else if FilenameList && f.Mode().IsRegular() {
					fmt.Printf("%15s %s %s\n", sizestr, s, f.Name())
				}
			}
			count++
			if count > NumLines {
				break
			}
		}
	}

	s := fmt.Sprintf("%d", SizeTotal)
	if SizeTotal > 100000 {
		s = AddCommas(s)
	}
	fmt.Println(" File Size total =", s)
} // end main dsrt

//-------------------------------------------------------------------- InsertByteSlice
func InsertIntoByteSlice(slice, insertion []byte, index int) []byte {
	return append(slice[:index], append(insertion, slice[index:]...)...)
}

//---------------------------------------------------------------------- AddCommas
func AddCommas(instr string) string {
	var Comma []byte = []byte{','}

	BS := make([]byte, 0, 15)
	BS = append(BS, instr...)

	i := len(BS)

	for NumberOfCommas := i / 3; (NumberOfCommas > 0) && (i > 3); NumberOfCommas-- {
		i -= 3
		BS = InsertIntoByteSlice(BS, Comma, i)
	}
	return string(BS)
} // AddCommas
//---------------------------------------------------------------------------------------------------

// ---------------------------- GetIDname -----------------------------------------------------------
func GetIDname(uidStr string) string {

	if len(uidStr) == 0 {
		return ""
	}
	ptrToUser, err := user.LookupId(uidStr)
	if err != nil {
		panic("uid not found")
	}

	idname := ptrToUser.Username
	return idname

} // GetIDname

/*
package path
func Match

func Match(pattern, name string) (matched bool, err error)

Match reports whether name matches the shell file name pattern.  The pattern syntax is:

pattern:
	{ term }
term:
	'*'         matches any sequence of non-/ characters
	'?'         matches any single non-/ character
	'[' [ '^' ] { character-range } ']'
	            character class (must be non-empty)
	c           matches character c (c != '*', '?', '\\', '[')
	'\\' c      matches character c

character-range:
	c           matches character c (c != '\\', '-', ']')
	'\\' c      matches character c
	lo '-' hi   matches character c for lo <= c <= hi

Match requires pattern to match all of name, not just a substring.  The only possible returned error is ErrBadPattern, when pattern is malformed.

package os
type FileInfo

type FileInfo interface {
        Name() string       // base name of the file
        Size() int64        // length in bytes for regular files; system-dependent for others
        Mode() FileMode     // file mode bits
        ModTime() time.Time // modification time
        IsDir() bool        // abbreviation for Mode().IsDir()
        Sys() interface{}   // underlying data source (can return nil)
}

A FileInfo describes a file and is returned by Stat and Lstat.

func Lstat

func Lstat(name string) (FileInfo, error)

Lstat returns a FileInfo describing the named file.  If the file is a symbolic link, the returned FileInfo describes the symbolic link.  Lstat makes no attempt to follow the link.
If there is an error, it will be of type *PathError.

func Stat

func Stat(name string) (FileInfo, error)

Stat returns a FileInfo describing the named file.  If there is an error, it will be of type *PathError.

The insight I had with my append troubles that the 1 slice entries were empty, is that when I used append, it would do just that to the end of the slice, and ignore the empty slices.
I needed to make the slice as empty for this to work.  So I am directly assigning the DirEntries slice, and appending the FileNames slice, to make sure that these both are doing what I want.
This code is now doing exactly what I want.  I guess there is no substitute for playing with myself.  Wait, that didn't come out right.  Or did it.

package os/user
type User struct {
  Uid string
  Gid string
  Username string // login name
  Name string     // full or display name.  It may be blank.
  HomeDir string
}
*/
<|endoftext|>


<commit_before>// Copyright 2016 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package client

import (
	"golang.org/x/net/context"

	"github.com/keybase/client/go/libkb"
	"github.com/keybase/client/go/protocol/keybase1"
)

func CheckUserOrTeamName(ctx context.Context, g *libkb.GlobalContext, name string) (*keybase1.UserOrTeamResult, error) {
	cli, err := GetTeamsClient(g)
	if err != nil {
		return nil, err
	}
	_, err = cli.TeamGet(ctx, keybase1.TeamGetArg{Name: name, ForceRepoll: false})
	if err == nil {
		ret := keybase1.UserOrTeamResult_TEAM
		return &ret, nil
	}

	// Assume name is a User or a TLF
	ret := keybase1.UserOrTeamResult_USER
	return &ret, nil
}
<commit_msg>better errors on not found (#8074)<commit_after>// Copyright 2016 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package client

import (
	"fmt"

	"golang.org/x/net/context"

	"github.com/keybase/client/go/libkb"
	"github.com/keybase/client/go/protocol/keybase1"
)

func CheckUserOrTeamName(ctx context.Context, g *libkb.GlobalContext, name string) (*keybase1.UserOrTeamResult, error) {
	tlfCli, tlfError := GetTlfClient(g)
	if tlfError == nil {
		tlfQuery := keybase1.TLFQuery{
			TlfName:          name,
			IdentifyBehavior: keybase1.TLFIdentifyBehavior_CHAT_CLI,
		}
		_, tlfError = tlfCli.CompleteAndCanonicalizePrivateTlfName(ctx, tlfQuery)
		if tlfError == nil {
			ret := keybase1.UserOrTeamResult_USER
			return &ret, nil
		}
	}

	cli, teamError := GetTeamsClient(g)
	if teamError == nil {
		_, teamError = cli.TeamGet(ctx, keybase1.TeamGetArg{Name: name, ForceRepoll: false})
		if teamError == nil {
			ret := keybase1.UserOrTeamResult_TEAM
			return &ret, nil
		}
	}

	msg := `Unable to find conversation.
When considering %s as a username or a list of usernames, received error: %v.
When considering %s as a team name, received error: %v.`

	return nil, libkb.NotFoundError{Msg: fmt.Sprintf(msg, name, tlfError, name, teamError)}
}
<|endoftext|>


<commit_before>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package logger

import (
	"fmt"
	"runtime"
	"strings"
	"sync"
	"time"

	logging "github.com/keybase/go-logging"

	"golang.org/x/net/context"
)

// TestLogBackend is an interface for logging to a test object (i.e.,
// a *testing.T).  We define this in order to avoid pulling in the
// "testing" package in exported code.
type TestLogBackend interface {
	Error(args ...interface{})
	Errorf(format string, args ...interface{})
	Fatal(args ...interface{})
	Fatalf(format string, args ...interface{})
	Log(args ...interface{})
	Logf(format string, args ...interface{})
	Failed() bool
	Name() string
}

// TestLogger is a Logger that writes to a TestLogBackend.  All
// messages except Fatal are printed using Logf, to avoid failing a
// test that is trying to test an error condition.  No context tags
// are logged.
type TestLogger struct {
	log          TestLogBackend
	extraDepth   int
	failReported bool
	sync.Mutex
}

func NewTestLogger(log TestLogBackend) *TestLogger {
	return &TestLogger{log: log}
}

// Verify TestLogger fully implements the Logger interface.
var _ Logger = (*TestLogger)(nil)

// ctx can be `nil`
func (log *TestLogger) common(ctx context.Context, lvl logging.Level, useFatal bool, fmts string, arg ...interface{}) {
	log.Lock()
	defer log.Unlock()
	if log.log.Failed() {
		if !log.failReported {
			log.log.Logf("TEST FAILED: %s", log.log.Name())
		}
		log.failReported = true
	}

	if ctx != nil {
		if useFatal {
			log.log.Fatalf(prepareString(ctx,
				log.prefixCaller(log.extraDepth, lvl, fmts)), arg...)
		} else {
			log.log.Logf(prepareString(ctx,
				log.prefixCaller(log.extraDepth, lvl, fmts)), arg...)
		}
	} else {
		if useFatal {
			log.log.Fatalf(log.prefixCaller(log.extraDepth, lvl, fmts), arg...)
		} else {
			log.log.Logf(log.prefixCaller(log.extraDepth, lvl, fmts), arg...)
		}
	}
}

func (log *TestLogger) prefixCaller(extraDepth int, lvl logging.Level, fmts string) string {
	// The testing library doesn't let us control the stack depth,
	// and it always prints out its own prefix, so use \r to clear
	// it out (at least on a terminal) and do our own formatting.
	_, file, line, _ := runtime.Caller(3 + extraDepth)
	elements := strings.Split(file, "/")
	failed := ""
	if log.log.Failed() {
		failed = "[X] "
	}

	return fmt.Sprintf("\r%s %s%s:%d: [%.1s] %s", time.Now().Format("2006-01-02 15:04:05.00000"),
		failed, elements[len(elements)-1], line, lvl, fmts)
}

func (log *TestLogger) Debug(fmts string, arg ...interface{}) {
	log.common(nil, logging.INFO, false, fmts, arg...)
}

func (log *TestLogger) CDebugf(ctx context.Context, fmts string,
	arg ...interface{}) {
	log.common(ctx, logging.DEBUG, false, fmts, arg...)
}

func (log *TestLogger) Info(fmts string, arg ...interface{}) {
	log.common(nil, logging.INFO, false, fmts, arg...)
}

func (log *TestLogger) CInfof(ctx context.Context, fmts string,
	arg ...interface{}) {
	log.common(ctx, logging.INFO, false, fmts, arg...)
}

func (log *TestLogger) Notice(fmts string, arg ...interface{}) {
	log.common(nil, logging.NOTICE, false, fmts, arg...)
}

func (log *TestLogger) CNoticef(ctx context.Context, fmts string,
	arg ...interface{}) {
	log.common(ctx, logging.NOTICE, false, fmts, arg...)
}

func (log *TestLogger) Warning(fmts string, arg ...interface{}) {
	log.common(nil, logging.WARNING, false, fmts, arg...)
}

func (log *TestLogger) CWarningf(ctx context.Context, fmts string,
	arg ...interface{}) {
	log.common(ctx, logging.WARNING, false, fmts, arg...)
}

func (log *TestLogger) Error(fmts string, arg ...interface{}) {
	log.common(nil, logging.ERROR, false, fmts, arg...)
}

func (log *TestLogger) Errorf(fmts string, arg ...interface{}) {
	log.common(nil, logging.ERROR, false, fmts, arg...)
}

func (log *TestLogger) CErrorf(ctx context.Context, fmts string,
	arg ...interface{}) {
	log.common(ctx, logging.ERROR, false, fmts, arg...)
}

func (log *TestLogger) Critical(fmts string, arg ...interface{}) {
	log.common(nil, logging.CRITICAL, false, fmts, arg...)
}

func (log *TestLogger) CCriticalf(ctx context.Context, fmts string,
	arg ...interface{}) {
	log.common(ctx, logging.CRITICAL, false, fmts, arg...)
}

func (log *TestLogger) Fatalf(fmts string, arg ...interface{}) {
	log.common(nil, logging.CRITICAL, true, fmts, arg...)
}

func (log *TestLogger) CFatalf(ctx context.Context, fmts string,
	arg ...interface{}) {
	log.common(ctx, logging.CRITICAL, true, fmts, arg...)
}

func (log *TestLogger) Profile(fmts string, arg ...interface{}) {
	log.common(nil, logging.CRITICAL, false, fmts, arg...)
}

func (log *TestLogger) Configure(style string, debug bool, filename string) {
	// no-op
}

func (log *TestLogger) RotateLogFile() error {
	// no-op
	return nil
}

func (log *TestLogger) CloneWithAddedDepth(depth int) Logger {
	log.Lock()
	defer log.Unlock()
	var clone TestLogger
	clone.log = log.log
	clone.extraDepth = log.extraDepth + depth
	clone.failReported = log.failReported
	return &clone
}

// no-op stubs to fulfill the Logger interface
func (log *TestLogger) SetExternalHandler(_ ExternalHandler) {}
<commit_msg>Move lock back to where it started<commit_after>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package logger

import (
	"fmt"
	"runtime"
	"strings"
	"sync"
	"time"

	logging "github.com/keybase/go-logging"

	"golang.org/x/net/context"
)

// TestLogBackend is an interface for logging to a test object (i.e.,
// a *testing.T).  We define this in order to avoid pulling in the
// "testing" package in exported code.
type TestLogBackend interface {
	Error(args ...interface{})
	Errorf(format string, args ...interface{})
	Fatal(args ...interface{})
	Fatalf(format string, args ...interface{})
	Log(args ...interface{})
	Logf(format string, args ...interface{})
	Failed() bool
	Name() string
}

// TestLogger is a Logger that writes to a TestLogBackend.  All
// messages except Fatal are printed using Logf, to avoid failing a
// test that is trying to test an error condition.  No context tags
// are logged.
type TestLogger struct {
	log          TestLogBackend
	extraDepth   int
	failReported bool
	sync.Mutex
}

func NewTestLogger(log TestLogBackend) *TestLogger {
	return &TestLogger{log: log}
}

// Verify TestLogger fully implements the Logger interface.
var _ Logger = (*TestLogger)(nil)

// ctx can be `nil`
func (log *TestLogger) common(ctx context.Context, lvl logging.Level, useFatal bool, fmts string, arg ...interface{}) {
	if log.log.Failed() {
		log.Lock()
		if !log.failReported {
			log.log.Logf("TEST FAILED: %s", log.log.Name())
		}
		log.failReported = true
		log.Unlock()
	}

	if ctx != nil {
		if useFatal {
			log.log.Fatalf(prepareString(ctx,
				log.prefixCaller(log.extraDepth, lvl, fmts)), arg...)
		} else {
			log.log.Logf(prepareString(ctx,
				log.prefixCaller(log.extraDepth, lvl, fmts)), arg...)
		}
	} else {
		if useFatal {
			log.log.Fatalf(log.prefixCaller(log.extraDepth, lvl, fmts), arg...)
		} else {
			log.log.Logf(log.prefixCaller(log.extraDepth, lvl, fmts), arg...)
		}
	}
}

func (log *TestLogger) prefixCaller(extraDepth int, lvl logging.Level, fmts string) string {
	// The testing library doesn't let us control the stack depth,
	// and it always prints out its own prefix, so use \r to clear
	// it out (at least on a terminal) and do our own formatting.
	_, file, line, _ := runtime.Caller(3 + extraDepth)
	elements := strings.Split(file, "/")
	failed := ""
	if log.log.Failed() {
		failed = "[X] "
	}

	return fmt.Sprintf("\r%s %s%s:%d: [%.1s] %s", time.Now().Format("2006-01-02 15:04:05.00000"),
		failed, elements[len(elements)-1], line, lvl, fmts)
}

func (log *TestLogger) Debug(fmts string, arg ...interface{}) {
	log.common(nil, logging.INFO, false, fmts, arg...)
}

func (log *TestLogger) CDebugf(ctx context.Context, fmts string,
	arg ...interface{}) {
	log.common(ctx, logging.DEBUG, false, fmts, arg...)
}

func (log *TestLogger) Info(fmts string, arg ...interface{}) {
	log.common(nil, logging.INFO, false, fmts, arg...)
}

func (log *TestLogger) CInfof(ctx context.Context, fmts string,
	arg ...interface{}) {
	log.common(ctx, logging.INFO, false, fmts, arg...)
}

func (log *TestLogger) Notice(fmts string, arg ...interface{}) {
	log.common(nil, logging.NOTICE, false, fmts, arg...)
}

func (log *TestLogger) CNoticef(ctx context.Context, fmts string,
	arg ...interface{}) {
	log.common(ctx, logging.NOTICE, false, fmts, arg...)
}

func (log *TestLogger) Warning(fmts string, arg ...interface{}) {
	log.common(nil, logging.WARNING, false, fmts, arg...)
}

func (log *TestLogger) CWarningf(ctx context.Context, fmts string,
	arg ...interface{}) {
	log.common(ctx, logging.WARNING, false, fmts, arg...)
}

func (log *TestLogger) Error(fmts string, arg ...interface{}) {
	log.common(nil, logging.ERROR, false, fmts, arg...)
}

func (log *TestLogger) Errorf(fmts string, arg ...interface{}) {
	log.common(nil, logging.ERROR, false, fmts, arg...)
}

func (log *TestLogger) CErrorf(ctx context.Context, fmts string,
	arg ...interface{}) {
	log.common(ctx, logging.ERROR, false, fmts, arg...)
}

func (log *TestLogger) Critical(fmts string, arg ...interface{}) {
	log.common(nil, logging.CRITICAL, false, fmts, arg...)
}

func (log *TestLogger) CCriticalf(ctx context.Context, fmts string,
	arg ...interface{}) {
	log.common(ctx, logging.CRITICAL, false, fmts, arg...)
}

func (log *TestLogger) Fatalf(fmts string, arg ...interface{}) {
	log.common(nil, logging.CRITICAL, true, fmts, arg...)
}

func (log *TestLogger) CFatalf(ctx context.Context, fmts string,
	arg ...interface{}) {
	log.common(ctx, logging.CRITICAL, true, fmts, arg...)
}

func (log *TestLogger) Profile(fmts string, arg ...interface{}) {
	log.common(nil, logging.CRITICAL, false, fmts, arg...)
}

func (log *TestLogger) Configure(style string, debug bool, filename string) {
	// no-op
}

func (log *TestLogger) RotateLogFile() error {
	// no-op
	return nil
}

func (log *TestLogger) CloneWithAddedDepth(depth int) Logger {
	log.Lock()
	defer log.Unlock()
	var clone TestLogger
	clone.log = log.log
	clone.extraDepth = log.extraDepth + depth
	clone.failReported = log.failReported
	return &clone
}

// no-op stubs to fulfill the Logger interface
func (log *TestLogger) SetExternalHandler(_ ExternalHandler) {}
<|endoftext|>


<commit_before>package paddlecloud

import (
	"context"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"os"
	"path"
	"strings"

	"github.com/PaddlePaddle/cloud/go/utils/config"
	"github.com/PaddlePaddle/cloud/go/utils/restclient"
	"github.com/golang/glog"
	"github.com/google/subcommands"
)

// Config is global config object for paddlecloud commandline
var Config = config.ParseDefaultConfig()

// SubmitCmd define the subcommand of submitting paddle training jobs.
type SubmitCmd struct {
	Jobname     string `json:"name"`
	Jobpackage  string `json:"jobPackage"`
	Parallelism int    `json:"parallelism"`
	CPU         int    `json:"cpu"`
	GPU         int    `json:"gpu"`
	Memory      string `json:"memory"`
	Pservers    int    `json:"pservers"`
	PSCPU       int    `json:"pscpu"`
	PSMemory    string `json:"psmemory"`
	Entry       string `json:"entry"`
	Topology    string `json:"topology"`
	Datacenter  string `json:"datacenter"`
	Passes      int    `json:"passes"`
	Image       string `json:"image"`
	Registry    string `json:"registry"`
}

// Name is subcommands name.
func (*SubmitCmd) Name() string { return "submit" }

// Synopsis is subcommands synopsis.
func (*SubmitCmd) Synopsis() string { return "Submit job to PaddlePaddle Cloud." }

// Usage is subcommands Usage.
func (*SubmitCmd) Usage() string {
	return `submit [options] <package path>:
	Submit job to PaddlePaddle Cloud.
	Options:
`
}

// SetFlags registers subcommands flags.
func (p *SubmitCmd) SetFlags(f *flag.FlagSet) {
	f.StringVar(&p.Jobname, "jobname", "paddle-cluster-job", "Cluster job name.")
	f.IntVar(&p.Parallelism, "parallelism", 1, "Number of parrallel trainers. Defaults to 1.")
	f.IntVar(&p.CPU, "cpu", 1, "CPU resource each trainer will use. Defaults to 1.")
	f.IntVar(&p.GPU, "gpu", 0, "GPU resource each trainer will use. Defaults to 0.")
	f.StringVar(&p.Memory, "memory", "1Gi", " Memory resource each trainer will use. Defaults to 1Gi.")
	f.IntVar(&p.Pservers, "pservers", 0, "Number of parameter servers. Defaults equal to -p")
	f.IntVar(&p.PSCPU, "pscpu", 1, "Parameter server CPU resource. Defaults to 1.")
	f.StringVar(&p.PSMemory, "psmemory", "1Gi", "Parameter server momory resource. Defaults to 1Gi.")
	f.StringVar(&p.Entry, "entry", "", "Command of starting trainer process. Defaults to paddle train")
	f.StringVar(&p.Topology, "topology", "", "Will Be Deprecated .py file contains paddle v1 job configs")
	f.IntVar(&p.Passes, "passes", 1, "Pass count for training job")
	f.StringVar(&p.Image, "image", "", "Runtime Docker image for the job")
	f.StringVar(&p.Registry, "registry", "", "Registry secret name for the runtime Docker image")
}

// Execute submit command.
func (p *SubmitCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus {
	if f.NArg() != 1 {
		f.Usage()
		return subcommands.ExitFailure
	}
	// default pservers count equals to trainers count.
	if p.Pservers == 0 {
		p.Pservers = p.Parallelism
	}
	p.Jobpackage = f.Arg(0)
	p.Datacenter = Config.ActiveConfig.Name

	s := NewSubmitter(p)
	errS := s.Submit(f.Arg(0), p.Jobname)
	if errS != nil {
		fmt.Fprintf(os.Stderr, "error submiting job: %v\n", errS)
		return subcommands.ExitFailure
	}
	fmt.Printf("%s submited.\n", p.Jobname)
	return subcommands.ExitSuccess
}

// Submitter submit job to cloud.
type Submitter struct {
	args *SubmitCmd
}

// NewSubmitter returns a submitter object.
func NewSubmitter(cmd *SubmitCmd) *Submitter {
	s := Submitter{cmd}
	return &s
}

// Submit current job.
func (s *Submitter) Submit(jobPackage string, jobName string) error {
	// if jobPackage is not a local dir, skip uploading package.
	_, pkgerr := os.Stat(jobPackage)
	if pkgerr == nil {
		dest := path.Join("/pfs", Config.ActiveConfig.Name, "home", Config.ActiveConfig.Username, "jobs", jobName)
		if !strings.HasSuffix(jobPackage, "/") {
			jobPackage = jobPackage + "/"
		}
		err := putFiles(jobPackage, dest)
		if err != nil {
			return err
		}
	} else if os.IsNotExist(pkgerr) {
		glog.Warning("jobpackage not a local dir, skip upload.")
	}
	// 2. call paddlecloud server to create kubernetes job
	jsonString, err := json.Marshal(s.args)
	if err != nil {
		return err
	}
	glog.V(10).Infof("Submitting job: %s to %s\n", jsonString, Config.ActiveConfig.Endpoint+"/api/v1/jobs")
	respBody, err := restclient.PostCall(Config.ActiveConfig.Endpoint+"/api/v1/jobs/", jsonString)
	if err != nil {
		return err
	}
	var respObj interface{}
	if err = json.Unmarshal(respBody, &respObj); err != nil {
		return err
	}
	// FIXME: Return an error if error message is not empty. Use response code instead
	errMsg := respObj.(map[string]interface{})["msg"].(string)
	if len(errMsg) > 0 {
		return errors.New(errMsg)
	}
	return nil
}
<commit_msg>check job name in clint (#231)<commit_after>package paddlecloud

import (
	"context"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"os"
	"path"
	"strings"

	"github.com/PaddlePaddle/cloud/go/utils/config"
	"github.com/PaddlePaddle/cloud/go/utils/restclient"
	"github.com/golang/glog"
	"github.com/google/subcommands"
)

const (
	invalidJobName = "jobname can not contain '.' or '_'"
)

// Config is global config object for paddlecloud commandline
var Config = config.ParseDefaultConfig()

// SubmitCmd define the subcommand of submitting paddle training jobs.
type SubmitCmd struct {
	Jobname     string `json:"name"`
	Jobpackage  string `json:"jobPackage"`
	Parallelism int    `json:"parallelism"`
	CPU         int    `json:"cpu"`
	GPU         int    `json:"gpu"`
	Memory      string `json:"memory"`
	Pservers    int    `json:"pservers"`
	PSCPU       int    `json:"pscpu"`
	PSMemory    string `json:"psmemory"`
	Entry       string `json:"entry"`
	Topology    string `json:"topology"`
	Datacenter  string `json:"datacenter"`
	Passes      int    `json:"passes"`
	Image       string `json:"image"`
	Registry    string `json:"registry"`
}

// Name is subcommands name.
func (*SubmitCmd) Name() string { return "submit" }

// Synopsis is subcommands synopsis.
func (*SubmitCmd) Synopsis() string { return "Submit job to PaddlePaddle Cloud." }

// Usage is subcommands Usage.
func (*SubmitCmd) Usage() string {
	return `submit [options] <package path>:
	Submit job to PaddlePaddle Cloud.
	Options:
`
}

// SetFlags registers subcommands flags.
func (p *SubmitCmd) SetFlags(f *flag.FlagSet) {
	f.StringVar(&p.Jobname, "jobname", "paddle-cluster-job", "Cluster job name.")
	f.IntVar(&p.Parallelism, "parallelism", 1, "Number of parrallel trainers. Defaults to 1.")
	f.IntVar(&p.CPU, "cpu", 1, "CPU resource each trainer will use. Defaults to 1.")
	f.IntVar(&p.GPU, "gpu", 0, "GPU resource each trainer will use. Defaults to 0.")
	f.StringVar(&p.Memory, "memory", "1Gi", " Memory resource each trainer will use. Defaults to 1Gi.")
	f.IntVar(&p.Pservers, "pservers", 0, "Number of parameter servers. Defaults equal to -p")
	f.IntVar(&p.PSCPU, "pscpu", 1, "Parameter server CPU resource. Defaults to 1.")
	f.StringVar(&p.PSMemory, "psmemory", "1Gi", "Parameter server momory resource. Defaults to 1Gi.")
	f.StringVar(&p.Entry, "entry", "", "Command of starting trainer process. Defaults to paddle train")
	f.StringVar(&p.Topology, "topology", "", "Will Be Deprecated .py file contains paddle v1 job configs")
	f.IntVar(&p.Passes, "passes", 1, "Pass count for training job")
	f.StringVar(&p.Image, "image", "", "Runtime Docker image for the job")
	f.StringVar(&p.Registry, "registry", "", "Registry secret name for the runtime Docker image")
}

// Execute submit command.
func (p *SubmitCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus {
	if f.NArg() != 1 {
		f.Usage()
		return subcommands.ExitFailure
	}
	// default pservers count equals to trainers count.
	if p.Pservers == 0 {
		p.Pservers = p.Parallelism
	}
	p.Jobpackage = f.Arg(0)
	p.Datacenter = Config.ActiveConfig.Name

	s := NewSubmitter(p)
	errS := s.Submit(f.Arg(0), p.Jobname)
	if errS != nil {
		fmt.Fprintf(os.Stderr, "error submiting job: %v\n", errS)
		return subcommands.ExitFailure
	}
	fmt.Printf("%s submited.\n", p.Jobname)
	return subcommands.ExitSuccess
}

// Submitter submit job to cloud.
type Submitter struct {
	args *SubmitCmd
}

// NewSubmitter returns a submitter object.
func NewSubmitter(cmd *SubmitCmd) *Submitter {
	s := Submitter{cmd}
	return &s
}

// Submit current job.
func (s *Submitter) Submit(jobPackage string, jobName string) error {
	if err := checkJobName(jobName); err != nil {
		return err
	}
	// if jobPackage is not a local dir, skip uploading package.
	_, pkgerr := os.Stat(jobPackage)
	if pkgerr == nil {
		dest := path.Join("/pfs", Config.ActiveConfig.Name, "home", Config.ActiveConfig.Username, "jobs", jobName)
		if !strings.HasSuffix(jobPackage, "/") {
			jobPackage = jobPackage + "/"
		}
		err := putFiles(jobPackage, dest)
		if err != nil {
			return err
		}
	} else if os.IsNotExist(pkgerr) {
		glog.Warning("jobpackage not a local dir, skip upload.")
	}
	// 2. call paddlecloud server to create kubernetes job
	jsonString, err := json.Marshal(s.args)
	if err != nil {
		return err
	}
	glog.V(10).Infof("Submitting job: %s to %s\n", jsonString, Config.ActiveConfig.Endpoint+"/api/v1/jobs")
	respBody, err := restclient.PostCall(Config.ActiveConfig.Endpoint+"/api/v1/jobs/", jsonString)
	if err != nil {
		return err
	}
	var respObj interface{}
	if err = json.Unmarshal(respBody, &respObj); err != nil {
		return err
	}
	// FIXME: Return an error if error message is not empty. Use response code instead
	errMsg := respObj.(map[string]interface{})["msg"].(string)
	if len(errMsg) > 0 {
		return errors.New(errMsg)
	}
	return nil
}
func checkJobName(jobName string) error {
	if strings.Contains(jobName, "_") || strings.Contains(jobName, ".") {
		return errors.New(invalidJobName)
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"os/signal"
	"syscall"

	"github.com/cloudfoundry-incubator/garden"
	"github.com/cloudfoundry-incubator/garden-linux/container_daemon"
	"github.com/cloudfoundry-incubator/garden-linux/container_daemon/unix_socket"
	"github.com/cloudfoundry-incubator/garden-linux/containerizer/system"
	"golang.org/x/crypto/ssh/terminal"
)

func main() {
	socketPath := flag.String("socket", "./run/wshd.sock", "Path to socket")
	user := flag.String("user", "vcap", "User to change to")
	dir := flag.String("dir", "/home/vcap", "Working directory for the running process")

	var envVars container_daemon.StringList
	flag.Var(&envVars, "env", "Environment variables to set for the command.")

	pidfile := flag.String("pidfile", "", "File to save container-namespaced pid of spawned process to")
	flag.Bool("rsh", false, "RSH compatibility mode")

	flag.Parse()

	extraArgs := flag.Args()
	if len(extraArgs) == 0 {
		// Default is to run a shell.
		extraArgs = []string{"/bin/sh"}
	}

	var tty *garden.TTYSpec
	resize := make(chan os.Signal)
	if terminal.IsTerminal(syscall.Stdin) {
		tty = &garden.TTYSpec{}
		signal.Notify(resize, syscall.SIGWINCH)
	}

	var pidfileWriter container_daemon.PidfileWriter = container_daemon.NoPidfile{}
	if *pidfile != "" {
		pidfileWriter = container_daemon.Pidfile{
			Path: *pidfile,
		}
	}

	process := &container_daemon.Process{
		Connector: &unix_socket.Connector{
			SocketPath: *socketPath,
		},

		Term: system.TermPkg{},

		Pidfile: pidfileWriter,

		SigwinchCh: resize,

		Spec: &garden.ProcessSpec{
			Path: extraArgs[0],
			Args: extraArgs[1:],
			Env:  envVars.List,
			Dir:  *dir,
			User: *user,
			TTY:  tty, // used as a boolean -- non-nil = attach pty
		},

		IO: &garden.ProcessIO{
			Stdin:  os.Stdin,
			Stderr: os.Stderr,
			Stdout: os.Stdout,
		},
	}

	err := process.Start()
	if err != nil {
		fmt.Fprintf(os.Stderr, "start process: %s", err)
		os.Exit(container_daemon.UnknownExitStatus)
	}

	defer process.Cleanup()

	exitCode, err := process.Wait()
	if err != nil {
		fmt.Fprintf(os.Stderr, "wait for process: %s", err)
		os.Exit(container_daemon.UnknownExitStatus)
	}

	os.Exit(exitCode)
}
<commit_msg>Wsh extracts rlimits from env and feeds process spec<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"os/signal"
	"syscall"

	"github.com/cloudfoundry-incubator/garden"
	"github.com/cloudfoundry-incubator/garden-linux/container_daemon"
	"github.com/cloudfoundry-incubator/garden-linux/container_daemon/unix_socket"
	"github.com/cloudfoundry-incubator/garden-linux/containerizer/system"
	"golang.org/x/crypto/ssh/terminal"
)

func main() {
	socketPath := flag.String("socket", "./run/wshd.sock", "Path to socket")
	user := flag.String("user", "vcap", "User to change to")
	dir := flag.String("dir", "/home/vcap", "Working directory for the running process")

	var envVars container_daemon.StringList
	flag.Var(&envVars, "env", "Environment variables to set for the command.")

	pidfile := flag.String("pidfile", "", "File to save container-namespaced pid of spawned process to")
	flag.Bool("rsh", false, "RSH compatibility mode")

	flag.Parse()

	extraArgs := flag.Args()
	if len(extraArgs) == 0 {
		// Default is to run a shell.
		extraArgs = []string{"/bin/sh"}
	}

	var tty *garden.TTYSpec
	resize := make(chan os.Signal)
	if terminal.IsTerminal(syscall.Stdin) {
		tty = &garden.TTYSpec{}
		signal.Notify(resize, syscall.SIGWINCH)
	}

	var pidfileWriter container_daemon.PidfileWriter = container_daemon.NoPidfile{}
	if *pidfile != "" {
		pidfileWriter = container_daemon.Pidfile{
			Path: *pidfile,
		}
	}

	process := &container_daemon.Process{
		Connector: &unix_socket.Connector{
			SocketPath: *socketPath,
		},

		Term: system.TermPkg{},

		Pidfile: pidfileWriter,

		SigwinchCh: resize,

		Spec: &garden.ProcessSpec{
			Path:   extraArgs[0],
			Args:   extraArgs[1:],
			Env:    envVars.List,
			Dir:    *dir,
			User:   *user,
			TTY:    tty, // used as a boolean -- non-nil = attach pty
			Limits: getRlimits(),
		},

		IO: &garden.ProcessIO{
			Stdin:  os.Stdin,
			Stderr: os.Stderr,
			Stdout: os.Stdout,
		},
	}

	err := process.Start()
	if err != nil {
		fmt.Fprintf(os.Stderr, "start process: %s", err)
		os.Exit(container_daemon.UnknownExitStatus)
	}

	defer process.Cleanup()

	exitCode, err := process.Wait()
	if err != nil {
		fmt.Fprintf(os.Stderr, "wait for process: %s", err)
		os.Exit(container_daemon.UnknownExitStatus)
	}

	os.Exit(exitCode)
}

func getRLimitFromEnv(envVar string) *uint64 {
	strVal := os.Getenv(envVar)
	if strVal == "" {
		return nil
	}

	var val uint64
	fmt.Sscanf(strVal, "%d", &val)
	return &val
}

func getRlimits() garden.ResourceLimits {
	return garden.ResourceLimits{
		As:         getRLimitFromEnv("RLIMIT_AS"),
		Core:       getRLimitFromEnv("RLIMIT_CORE"),
		Cpu:        getRLimitFromEnv("RLIMIT_CPU"),
		Data:       getRLimitFromEnv("RLIMIT_DATA"),
		Fsize:      getRLimitFromEnv("RLIMIT_FSIZE"),
		Locks:      getRLimitFromEnv("RLIMIT_LOCKS"),
		Memlock:    getRLimitFromEnv("RLIMIT_MEMLOCK"),
		Msgqueue:   getRLimitFromEnv("RLIMIT_MSGQUEUE"),
		Nice:       getRLimitFromEnv("RLIMIT_NICE"),
		Nofile:     getRLimitFromEnv("RLIMIT_NOFILE"),
		Nproc:      getRLimitFromEnv("RLIMIT_NPROC"),
		Rss:        getRLimitFromEnv("RLIMIT_RSS"),
		Rtprio:     getRLimitFromEnv("RLIMIT_RTPRIO"),
		Sigpending: getRLimitFromEnv("RLIMIT_SIGPENDING"),
		Stack:      getRLimitFromEnv("RLIMIT_STACK"),
	}
}
<|endoftext|>


<commit_before>// +build test_integration
// +build test_local

/*
Copyright 2017 The Nuclio Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package test

import (
	"fmt"
	"path"
	"testing"
	"time"

	"github.com/nuclio/nuclio/pkg/common"
	"github.com/nuclio/nuclio/pkg/dockerclient"
	"github.com/nuclio/nuclio/pkg/functionconfig"
	"github.com/nuclio/nuclio/pkg/platform"
	"github.com/nuclio/nuclio/pkg/processor/trigger/mqtt"
	"github.com/nuclio/nuclio/pkg/processor/trigger/test"

	mqttclient "github.com/eclipse/paho.mqtt.golang"
	"github.com/stretchr/testify/suite"
)

type testSuite struct {
	*triggertest.AbstractBrokerSuite
	brokerPort    int
	brokerURL     string
	containerName string
	mqttClient    mqttclient.Client
}

func (suite *testSuite) SetupSuite() {
	suite.brokerPort = 1883
	suite.containerName = "mqtt-mosquitto"                     // nolint: misspell
	suite.BrokerContainerNetworkName = "nuclio-mosquitto-test" // nolint: misspell

	suite.brokerURL = fmt.Sprintf("tcp://%s:%d", suite.BrokerHost, suite.brokerPort)

	// create client
	suite.mqttClient = mqttclient.NewClient(mqttclient.NewClientOptions().AddBroker(suite.brokerURL))
	suite.AbstractBrokerSuite.SetupSuite()
}

// GetContainerRunInfo returns information about the broker container
func (suite *testSuite) GetContainerRunInfo() (string, *dockerclient.RunOptions) {
	return "eclipse-mosquitto:latest", &dockerclient.RunOptions{ // nolint: misspell
		ContainerName: suite.containerName,
		Network:       suite.BrokerContainerNetworkName,
		Remove:        true,
		Volumes: map[string]string{
			path.Join(suite.GetNuclioHostSourceDir(),
				"test",
				"mqtt",
				"artifacts",
				"mosquitto.conf"): "/mosquitto/config/mosquitto.conf", // nolint: misspell
		},
		Ports: map[int]int{
			suite.brokerPort: suite.brokerPort,
		},
	}
}

// WaitForBroker waits until the broker is ready
func (suite *testSuite) WaitForBroker() error {

	// retry to connect
	err := common.RetryUntilSuccessful(30*time.Second, 1*time.Second, func() bool {
		if token := suite.mqttClient.Connect(); token.Wait() && token.Error() != nil {
			return false
		}
		return true
	})

	suite.Require().NoError(err, "Failed to connect to MQTT broker in given timeframe")

	return nil
}

func (suite *testSuite) TestMultipleTopics() {
	triggerConfiguration := suite.getTriggerConfiguration([]mqtt.Subscription{
		{Topic: "a1/b1/c1", QOS: 0},
		{Topic: "a1/b1", QOS: 1},
		{Topic: "a2/b2/c3/c4", QOS: 2},
	})

	// invoke the event recorder
	triggertest.InvokeEventRecorder(&suite.AbstractBrokerSuite.TestSuite,
		suite.BrokerHost,
		suite.getCreateFunctionOptionsWithMQTTTrigger(triggerConfiguration),
		map[string]triggertest.TopicMessages{
			"a1/b1/c1":    {NumMessages: 3},
			"a1/b1":       {NumMessages: 3},
			"a2/b2/c3/c4": {NumMessages: 3},
		},
		nil,
		suite.publishMessageToTopic)
}

func (suite *testSuite) getCreateFunctionOptionsWithMQTTTrigger(triggerConfig functionconfig.Trigger) *platform.CreateFunctionOptions {
	createFunctionOptions := suite.GetDeployOptions("event_recorder", "")
	createFunctionOptions.FunctionConfig.Spec.Platform = functionconfig.Platform{
		Attributes: map[string]interface{}{
			"network": suite.BrokerContainerNetworkName,
		},
	}
	createFunctionOptions.FunctionConfig.Spec.Runtime = "python"
	createFunctionOptions.FunctionConfig.Meta.Name = "event-recorder"
	createFunctionOptions.FunctionConfig.Spec.Build.Path = suite.FunctionPaths["python"]
	createFunctionOptions.FunctionConfig.Spec.Triggers = map[string]functionconfig.Trigger{}
	createFunctionOptions.FunctionConfig.Spec.Triggers["test_mqtt"] = triggerConfig
	createFunctionOptions.FunctionConfig.Spec.ReadinessTimeoutSeconds = 10

	return createFunctionOptions
}

func (suite *testSuite) getTriggerConfiguration(subscriptions []mqtt.Subscription) functionconfig.Trigger {
	return functionconfig.Trigger{
		Kind: "mqtt",
		URL:  fmt.Sprintf("tcp://%s:%d", suite.containerName, suite.brokerPort),
		Attributes: map[string]interface{}{
			"subscriptions": subscriptions,
		},
	}
}

func (suite *testSuite) publishMessageToTopic(topic string, body string) error {
	token := suite.mqttClient.Publish(topic,
		byte(0),
		false,
		body)

	token.Wait()

	return token.Error()
}

func TestIntegrationSuite(t *testing.T) {
	if testing.Short() {
		return
	}

	newTestSuite := &testSuite{}
	newTestSuite.AbstractBrokerSuite = triggertest.NewAbstractBrokerSuite(newTestSuite)
	suite.Run(t, newTestSuite)
}
<commit_msg>[CI] - Fix broken test (#2318)<commit_after>// +build test_integration
// +build test_local
// +build test_broken

// NOTE: Currently broken
// It seems that the mqtt eclipse container refuses to take incoming requests when running
// from GitHub Action worker, while working just fine when running locally - macOS.
// Container logs:
/*
	1630833404: mosquitto version 2.0.12 starting
	1630833404: Config loaded from /mosquitto/config/mosquitto.conf.
	1630833404: Opening ipv4 listen socket on port 1883.
	1630833404: Opening ipv6 listen socket on port 1883.
	1630833404: mosquitto version 1.6.15 running
	1630833405: New connection from 172.20.0.1 on port 1883.
	1630833405: Sending CONNACK to 172.20.0.1 (0, 2)
	1630833405: Client <unknown> disconnected due to protocol error.
	1630833406: New connection from 172.20.0.1 on port 1883.
	1630833406: Sending CONNACK to 172.20.0.1 (0, 2)
	... and so on
*/ // nolint: misspell

/*
Copyright 2017 The Nuclio Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package test

import (
	"fmt"
	"path"
	"testing"
	"time"

	"github.com/nuclio/nuclio/pkg/common"
	"github.com/nuclio/nuclio/pkg/dockerclient"
	"github.com/nuclio/nuclio/pkg/functionconfig"
	"github.com/nuclio/nuclio/pkg/platform"
	"github.com/nuclio/nuclio/pkg/processor/trigger/mqtt"
	"github.com/nuclio/nuclio/pkg/processor/trigger/test"

	mqttclient "github.com/eclipse/paho.mqtt.golang"
	"github.com/stretchr/testify/suite"
)

type testSuite struct {
	*triggertest.AbstractBrokerSuite
	brokerPort    int
	brokerURL     string
	containerName string
	mqttClient    mqttclient.Client
}

func (suite *testSuite) SetupSuite() {
	suite.brokerPort = 1883
	suite.containerName = "mqtt-mosquitto"                     // nolint: misspell
	suite.BrokerContainerNetworkName = "nuclio-mosquitto-test" // nolint: misspell

	suite.brokerURL = fmt.Sprintf("tcp://%s:%d", suite.BrokerHost, suite.brokerPort)

	// create client
	suite.mqttClient = mqttclient.NewClient(mqttclient.NewClientOptions().AddBroker(suite.brokerURL))
	suite.AbstractBrokerSuite.SetupSuite()
}

// GetContainerRunInfo returns information about the broker container
func (suite *testSuite) GetContainerRunInfo() (string, *dockerclient.RunOptions) {
	return "eclipse-mosquitto", &dockerclient.RunOptions{ // nolint: misspell
		ContainerName: suite.containerName,
		Network:       suite.BrokerContainerNetworkName,
		Remove:        true,
		Volumes: map[string]string{
			path.Join(suite.GetNuclioHostSourceDir(),
				"test",
				"mqtt",
				"artifacts",
				"mosquitto.conf"): "/mosquitto/config/mosquitto.conf", // nolint: misspell
		},
		Ports: map[int]int{
			suite.brokerPort: suite.brokerPort,
		},
	}
}

// WaitForBroker waits until the broker is ready
func (suite *testSuite) WaitForBroker() error {

	// retry to connect
	err := common.RetryUntilSuccessful(30*time.Second, 1*time.Second, func() bool {
		if token := suite.mqttClient.Connect(); token.Wait() && token.Error() != nil {
			return false
		}
		return true
	})

	// get broker logs in case connect has failed, we want the logs to be logged
	containerLogs, containerLogsErr := suite.DockerClient.GetContainerLogs(suite.containerName)
	suite.Logger.DebugWith("Fetched broker container logs", "logs", containerLogs)
	suite.Require().NoError(containerLogsErr, "Failed to get broker container logs")

	suite.Require().NoError(err, "Failed to connect to MQTT broker in given timeframe")
	return nil
}

func (suite *testSuite) TestMultipleTopics() {
	triggerConfiguration := suite.getTriggerConfiguration([]mqtt.Subscription{
		{Topic: "a1/b1/c1", QOS: 0},
		{Topic: "a1/b1", QOS: 1},
		{Topic: "a2/b2/c3/c4", QOS: 2},
	})

	// invoke the event recorder
	triggertest.InvokeEventRecorder(&suite.AbstractBrokerSuite.TestSuite,
		suite.BrokerHost,
		suite.getCreateFunctionOptionsWithMQTTTrigger(triggerConfiguration),
		map[string]triggertest.TopicMessages{
			"a1/b1/c1":    {NumMessages: 3},
			"a1/b1":       {NumMessages: 3},
			"a2/b2/c3/c4": {NumMessages: 3},
		},
		nil,
		suite.publishMessageToTopic)
}

func (suite *testSuite) getCreateFunctionOptionsWithMQTTTrigger(triggerConfig functionconfig.Trigger) *platform.CreateFunctionOptions {
	createFunctionOptions := suite.GetDeployOptions("event_recorder", "")
	createFunctionOptions.FunctionConfig.Spec.Platform = functionconfig.Platform{
		Attributes: map[string]interface{}{
			"network": suite.BrokerContainerNetworkName,
		},
	}
	createFunctionOptions.FunctionConfig.Spec.Runtime = "python"
	createFunctionOptions.FunctionConfig.Meta.Name = "event-recorder"
	createFunctionOptions.FunctionConfig.Spec.Build.Path = suite.FunctionPaths["python"]
	createFunctionOptions.FunctionConfig.Spec.Triggers = map[string]functionconfig.Trigger{}
	createFunctionOptions.FunctionConfig.Spec.Triggers["test_mqtt"] = triggerConfig
	createFunctionOptions.FunctionConfig.Spec.ReadinessTimeoutSeconds = 10

	return createFunctionOptions
}

func (suite *testSuite) getTriggerConfiguration(subscriptions []mqtt.Subscription) functionconfig.Trigger {
	return functionconfig.Trigger{
		Kind: "mqtt",
		URL:  fmt.Sprintf("tcp://%s:%d", suite.containerName, suite.brokerPort),
		Attributes: map[string]interface{}{
			"subscriptions": subscriptions,
		},
	}
}

func (suite *testSuite) publishMessageToTopic(topic string, body string) error {
	token := suite.mqttClient.Publish(topic,
		byte(0),
		false,
		body)

	token.Wait()

	return token.Error()
}

func TestIntegrationSuite(t *testing.T) {
	if testing.Short() {
		return
	}

	newTestSuite := &testSuite{}
	newTestSuite.AbstractBrokerSuite = triggertest.NewAbstractBrokerSuite(newTestSuite)
	suite.Run(t, newTestSuite)
}
<|endoftext|>


<commit_before>package device_manager

import (
	"os"
	"strings"

	"github.com/golang/mock/gomock"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"k8s.io/client-go/kubernetes/fake"

	v1 "kubevirt.io/api/core/v1"

	"k8s.io/apimachinery/pkg/util/yaml"

	"kubevirt.io/kubevirt/pkg/testutils"
	virtconfig "kubevirt.io/kubevirt/pkg/virt-config"
)

const (
	fakeName       = "example.org/deadbeef"
	fakeID         = "dead:beef"
	fakeDriver     = "vfio-pci"
	fakeAddress    = "0000:00:00.0"
	fakeIommuGroup = "0"
	fakeNumaNode   = 0
)

var _ = Describe("PCI Device", func() {
	var mockPCI *MockDeviceHandler
	var fakePermittedHostDevicesConfig string
	var fakePermittedHostDevices v1.PermittedHostDevices
	var ctrl *gomock.Controller
	var clientTest *fake.Clientset

	BeforeEach(func() {
		clientTest = fake.NewSimpleClientset()
		By("making sure the environment has a PCI device at " + fakeAddress)
		_, err := os.Stat("/sys/bus/pci/devices/" + fakeAddress)
		if os.IsNotExist(err) {
			Skip("No PCI device found at " + fakeAddress + ", can't run PCI tests")
		}

		By("mocking PCI functions to simulate a vfio-pci device at " + fakeAddress)
		ctrl = gomock.NewController(GinkgoT())
		mockPCI = NewMockDeviceHandler(ctrl)
		Handler = mockPCI
		// Force pre-defined returned values and ensure the function only get called exacly once each on 0000:00:00.0
		mockPCI.EXPECT().GetDeviceIOMMUGroup(pciBasePath, fakeAddress).Return(fakeIommuGroup, nil).Times(1)
		mockPCI.EXPECT().GetDeviceDriver(pciBasePath, fakeAddress).Return(fakeDriver, nil).Times(1)
		mockPCI.EXPECT().GetDeviceNumaNode(pciBasePath, fakeAddress).Return(fakeNumaNode).Times(1)
		mockPCI.EXPECT().GetDevicePCIID(pciBasePath, fakeAddress).Return(fakeID, nil).Times(1)
		// Allow the regular functions to be called for all the other devices, they're harmless.
		// Just force the driver to NOT vfio-pci to ensure they all get ignored.
		mockPCI.EXPECT().GetDeviceIOMMUGroup(pciBasePath, gomock.Any()).AnyTimes()
		mockPCI.EXPECT().GetDeviceDriver(pciBasePath, gomock.Any()).Return("definitely-not-vfio-pci", nil).AnyTimes()
		mockPCI.EXPECT().GetDeviceNumaNode(pciBasePath, gomock.Any()).AnyTimes()
		mockPCI.EXPECT().GetDevicePCIID(pciBasePath, gomock.Any()).AnyTimes()

		By("creating a list of fake device using the yaml decoder")
		fakePermittedHostDevicesConfig = `
pciHostDevices:
- pciVendorSelector: "` + fakeID + `"
  resourceName: "` + fakeName + `"
`
		err = yaml.NewYAMLOrJSONDecoder(strings.NewReader(fakePermittedHostDevicesConfig), 1024).Decode(&fakePermittedHostDevices)
		Expect(err).ToNot(HaveOccurred())
		Expect(len(fakePermittedHostDevices.PciHostDevices)).To(Equal(1))
		Expect(fakePermittedHostDevices.PciHostDevices[0].PCIVendorSelector).To(Equal(fakeID))
		Expect(fakePermittedHostDevices.PciHostDevices[0].ResourceName).To(Equal(fakeName))
	})

	AfterEach(func() {
		ctrl.Finish()
	})

	It("Should parse the permitted devices and find 1 matching PCI device", func() {
		supportedPCIDeviceMap := make(map[string]string)
		for _, pciDev := range fakePermittedHostDevices.PciHostDevices {
			// do not add a device plugin for this resource if it's being provided via an external device plugin
			if !pciDev.ExternalResourceProvider {
				supportedPCIDeviceMap[pciDev.PCIVendorSelector] = pciDev.ResourceName
			}
		}
		// discoverPermittedHostPCIDevices() will walk real PCI devices wherever the tests are running
		// It's assumed here that it will find a PCI device at 0000:00:00.0
		devices := discoverPermittedHostPCIDevices(supportedPCIDeviceMap)
		Expect(len(devices)).To(Equal(1))
		Expect(len(devices[fakeID])).To(Equal(1))
		Expect(devices[fakeID][0].pciID).To(Equal(fakeID))
		Expect(devices[fakeID][0].driver).To(Equal(fakeDriver))
		Expect(devices[fakeID][0].pciAddress).To(Equal(fakeAddress))
		Expect(devices[fakeID][0].iommuGroup).To(Equal(fakeIommuGroup))
		Expect(devices[fakeID][0].numaNode).To(Equal(fakeNumaNode))
	})

	It("Should validate DPI devices", func() {
		iommuToPCIMap := make(map[string]string)
		supportedPCIDeviceMap := make(map[string]string)
		for _, pciDev := range fakePermittedHostDevices.PciHostDevices {
			// do not add a device plugin for this resource if it's being provided via an external device plugin
			if !pciDev.ExternalResourceProvider {
				supportedPCIDeviceMap[pciDev.PCIVendorSelector] = pciDev.ResourceName
			}
		}
		// discoverPermittedHostPCIDevices() will walk real PCI devices wherever the tests are running
		// It's assumed here that it will find a PCI device at 0000:00:00.0
		pciDevices := discoverPermittedHostPCIDevices(supportedPCIDeviceMap)
		devs := constructDPIdevices(pciDevices[fakeID], iommuToPCIMap)
		Expect(devs[0].ID).To(Equal(fakeIommuGroup))
		Expect(devs[0].Topology.Nodes[0].ID).To(Equal(int64(fakeNumaNode)))
	})
	It("Should update the device list according to the configmap", func() {
		By("creating a cluster config")
		kv := &v1.KubeVirt{
			ObjectMeta: metav1.ObjectMeta{
				Name:      "kubevirt",
				Namespace: "kubevirt",
			},
			Spec: v1.KubeVirtSpec{
				Configuration: v1.KubeVirtConfiguration{
					DeveloperConfiguration: &v1.DeveloperConfiguration{},
				},
			},
			Status: v1.KubeVirtStatus{
				Phase: v1.KubeVirtPhaseDeploying,
			},
		}
		fakeClusterConfig, _, kvInformer := testutils.NewFakeClusterConfigUsingKV(kv)

		By("creating an empty device controller")
		deviceController := NewDeviceController("master", 10, "rw", fakeClusterConfig, clientTest.CoreV1())
		deviceController.devicePlugins = make(map[string]ControlledDevice)

		By("adding a host device to the cluster config")
		kvConfig := kv.DeepCopy()
		kvConfig.Spec.Configuration.DeveloperConfiguration.FeatureGates = []string{virtconfig.HostDevicesGate}
		kvConfig.Spec.Configuration.PermittedHostDevices = &v1.PermittedHostDevices{
			PciHostDevices: []v1.PciHostDevice{
				{
					PCIVendorSelector: fakeID,
					ResourceName:      fakeName,
				},
			},
		}
		testutils.UpdateFakeKubeVirtClusterConfig(kvInformer, kvConfig)
		permittedDevices := fakeClusterConfig.GetPermittedHostDevices()
		Expect(permittedDevices).ToNot(BeNil(), "something went wrong while parsing the configmap(s)")
		Expect(len(permittedDevices.PciHostDevices)).To(Equal(1), "the fake device was not found")

		By("ensuring a device plugin gets created for our fake device")
		enabledDevicePlugins, disabledDevicePlugins := deviceController.updatePermittedHostDevicePlugins()
		Expect(len(enabledDevicePlugins)).To(Equal(1), "a device plugin wasn't created for the fake device")
		Expect(len(disabledDevicePlugins)).To(Equal(0))
		Ω(enabledDevicePlugins).Should(HaveKey(fakeName))
		// Manually adding the enabled plugin, since the device controller is not actually running
		deviceController.devicePlugins[fakeName] = enabledDevicePlugins[fakeName]

		By("deletting the device from the configmap")
		kvConfig.Spec.Configuration.PermittedHostDevices = &v1.PermittedHostDevices{}
		testutils.UpdateFakeKubeVirtClusterConfig(kvInformer, kvConfig)
		permittedDevices = fakeClusterConfig.GetPermittedHostDevices()
		Expect(permittedDevices).ToNot(BeNil(), "something went wrong while parsing the configmap(s)")
		Expect(len(permittedDevices.PciHostDevices)).To(Equal(0), "the fake device was not deleted")

		By("ensuring the device plugin gets stopped")
		enabledDevicePlugins, disabledDevicePlugins = deviceController.updatePermittedHostDevicePlugins()
		Expect(len(enabledDevicePlugins)).To(Equal(0))
		Expect(len(disabledDevicePlugins)).To(Equal(1), "the fake device plugin did not get disabled")
		Ω(disabledDevicePlugins).Should(HaveKey(fakeName))
	})
})
<commit_msg>adjust unit test for pci devices discovery<commit_after>package device_manager

import (
	"os"
	"strings"

	"github.com/golang/mock/gomock"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"k8s.io/client-go/kubernetes/fake"

	v1 "kubevirt.io/api/core/v1"

	"k8s.io/apimachinery/pkg/util/yaml"

	"kubevirt.io/kubevirt/pkg/testutils"
	virtconfig "kubevirt.io/kubevirt/pkg/virt-config"
)

const (
	fakeName       = "example.org/deadbeef"
	fakeID         = "dead:beef"
	fakeDriver     = "vfio-pci"
	fakeAddress    = "0000:00:00.0"
	fakeIommuGroup = "0"
	fakeNumaNode   = 0
)

var _ = Describe("PCI Device", func() {
	var mockPCI *MockDeviceHandler
	var fakePermittedHostDevicesConfig string
	var fakePermittedHostDevices v1.PermittedHostDevices
	var ctrl *gomock.Controller
	var clientTest *fake.Clientset

	BeforeEach(func() {
		clientTest = fake.NewSimpleClientset()
		By("making sure the environment has a PCI device at " + fakeAddress)
		_, err := os.Stat("/sys/bus/pci/devices/" + fakeAddress)
		if os.IsNotExist(err) {
			Skip("No PCI device found at " + fakeAddress + ", can't run PCI tests")
		}

		By("mocking PCI functions to simulate a vfio-pci device at " + fakeAddress)
		ctrl = gomock.NewController(GinkgoT())
		mockPCI = NewMockDeviceHandler(ctrl)
		Handler = mockPCI
		// Force pre-defined returned values and ensure the function only get called exacly once each on 0000:00:00.0
		mockPCI.EXPECT().GetDeviceIOMMUGroup(pciBasePath, fakeAddress).Return(fakeIommuGroup, nil).Times(1)
		mockPCI.EXPECT().GetDeviceDriver(pciBasePath, fakeAddress).Return(fakeDriver, nil).Times(1)
		mockPCI.EXPECT().GetDeviceNumaNode(pciBasePath, fakeAddress).Return(fakeNumaNode).Times(1)
		mockPCI.EXPECT().GetDevicePCIID(pciBasePath, fakeAddress).Return(fakeID, nil).Times(1)
		// Allow the regular functions to be called for all the other devices, they're harmless.
		// Just force the driver to NOT vfio-pci to ensure they all get ignored.
		mockPCI.EXPECT().GetDeviceIOMMUGroup(pciBasePath, gomock.Any()).AnyTimes()
		mockPCI.EXPECT().GetDeviceDriver(pciBasePath, gomock.Any()).Return("definitely-not-vfio-pci", nil).AnyTimes()
		mockPCI.EXPECT().GetDeviceNumaNode(pciBasePath, gomock.Any()).AnyTimes()
		mockPCI.EXPECT().GetDevicePCIID(pciBasePath, gomock.Any()).AnyTimes()

		By("creating a list of fake device using the yaml decoder")
		fakePermittedHostDevicesConfig = `
pciHostDevices:
- pciVendorSelector: "` + fakeID + `"
  resourceName: "` + fakeName + `"
`
		err = yaml.NewYAMLOrJSONDecoder(strings.NewReader(fakePermittedHostDevicesConfig), 1024).Decode(&fakePermittedHostDevices)
		Expect(err).ToNot(HaveOccurred())
		Expect(len(fakePermittedHostDevices.PciHostDevices)).To(Equal(1))
		Expect(fakePermittedHostDevices.PciHostDevices[0].PCIVendorSelector).To(Equal(fakeID))
		Expect(fakePermittedHostDevices.PciHostDevices[0].ResourceName).To(Equal(fakeName))
	})

	AfterEach(func() {
		ctrl.Finish()
	})

	It("Should parse the permitted devices and find 1 matching PCI device", func() {
		supportedPCIDeviceMap := make(map[string]string)
		for _, pciDev := range fakePermittedHostDevices.PciHostDevices {
			// do not add a device plugin for this resource if it's being provided via an external device plugin
			if !pciDev.ExternalResourceProvider {
				supportedPCIDeviceMap[pciDev.PCIVendorSelector] = pciDev.ResourceName
			}
		}
		// discoverPermittedHostPCIDevices() will walk real PCI devices wherever the tests are running
		// It's assumed here that it will find a PCI device at 0000:00:00.0
		devices := discoverPermittedHostPCIDevices(supportedPCIDeviceMap)
		Expect(len(devices)).To(Equal(1))
		Expect(len(devices[fakeName])).To(Equal(1))
		Expect(devices[fakeName][0].pciID).To(Equal(fakeID))
		Expect(devices[fakeName][0].driver).To(Equal(fakeDriver))
		Expect(devices[fakeName][0].pciAddress).To(Equal(fakeAddress))
		Expect(devices[fakeName][0].iommuGroup).To(Equal(fakeIommuGroup))
		Expect(devices[fakeName][0].numaNode).To(Equal(fakeNumaNode))
	})

	It("Should validate DPI devices", func() {
		iommuToPCIMap := make(map[string]string)
		supportedPCIDeviceMap := make(map[string]string)
		for _, pciDev := range fakePermittedHostDevices.PciHostDevices {
			// do not add a device plugin for this resource if it's being provided via an external device plugin
			if !pciDev.ExternalResourceProvider {
				supportedPCIDeviceMap[pciDev.PCIVendorSelector] = pciDev.ResourceName
			}
		}
		// discoverPermittedHostPCIDevices() will walk real PCI devices wherever the tests are running
		// It's assumed here that it will find a PCI device at 0000:00:00.0
		pciDevices := discoverPermittedHostPCIDevices(supportedPCIDeviceMap)
		devs := constructDPIdevices(pciDevices[fakeName], iommuToPCIMap)
		Expect(devs[0].ID).To(Equal(fakeIommuGroup))
		Expect(devs[0].Topology.Nodes[0].ID).To(Equal(int64(fakeNumaNode)))
	})
	It("Should update the device list according to the configmap", func() {
		By("creating a cluster config")
		kv := &v1.KubeVirt{
			ObjectMeta: metav1.ObjectMeta{
				Name:      "kubevirt",
				Namespace: "kubevirt",
			},
			Spec: v1.KubeVirtSpec{
				Configuration: v1.KubeVirtConfiguration{
					DeveloperConfiguration: &v1.DeveloperConfiguration{},
				},
			},
			Status: v1.KubeVirtStatus{
				Phase: v1.KubeVirtPhaseDeploying,
			},
		}
		fakeClusterConfig, _, kvInformer := testutils.NewFakeClusterConfigUsingKV(kv)

		By("creating an empty device controller")
		deviceController := NewDeviceController("master", 10, "rw", fakeClusterConfig, clientTest.CoreV1())
		deviceController.devicePlugins = make(map[string]ControlledDevice)

		By("adding a host device to the cluster config")
		kvConfig := kv.DeepCopy()
		kvConfig.Spec.Configuration.DeveloperConfiguration.FeatureGates = []string{virtconfig.HostDevicesGate}
		kvConfig.Spec.Configuration.PermittedHostDevices = &v1.PermittedHostDevices{
			PciHostDevices: []v1.PciHostDevice{
				{
					PCIVendorSelector: fakeID,
					ResourceName:      fakeName,
				},
			},
		}
		testutils.UpdateFakeKubeVirtClusterConfig(kvInformer, kvConfig)
		permittedDevices := fakeClusterConfig.GetPermittedHostDevices()
		Expect(permittedDevices).ToNot(BeNil(), "something went wrong while parsing the configmap(s)")
		Expect(len(permittedDevices.PciHostDevices)).To(Equal(1), "the fake device was not found")

		By("ensuring a device plugin gets created for our fake device")
		enabledDevicePlugins, disabledDevicePlugins := deviceController.updatePermittedHostDevicePlugins()
		Expect(len(enabledDevicePlugins)).To(Equal(1), "a device plugin wasn't created for the fake device")
		Expect(len(disabledDevicePlugins)).To(Equal(0))
		Ω(enabledDevicePlugins).Should(HaveKey(fakeName))
		// Manually adding the enabled plugin, since the device controller is not actually running
		deviceController.devicePlugins[fakeName] = enabledDevicePlugins[fakeName]

		By("deletting the device from the configmap")
		kvConfig.Spec.Configuration.PermittedHostDevices = &v1.PermittedHostDevices{}
		testutils.UpdateFakeKubeVirtClusterConfig(kvInformer, kvConfig)
		permittedDevices = fakeClusterConfig.GetPermittedHostDevices()
		Expect(permittedDevices).ToNot(BeNil(), "something went wrong while parsing the configmap(s)")
		Expect(len(permittedDevices.PciHostDevices)).To(Equal(0), "the fake device was not deleted")

		By("ensuring the device plugin gets stopped")
		enabledDevicePlugins, disabledDevicePlugins = deviceController.updatePermittedHostDevicePlugins()
		Expect(len(enabledDevicePlugins)).To(Equal(0))
		Expect(len(disabledDevicePlugins)).To(Equal(1), "the fake device plugin did not get disabled")
		Ω(disabledDevicePlugins).Should(HaveKey(fakeName))
	})
})
<|endoftext|>


<commit_before>package grouper

import (
	"errors"
	"fmt"
	"os"

	"github.com/tedsuo/ifrit"
)

type ProcessGroup interface {
	ifrit.Process
	Exits() <-chan Member
}

func EnvokeGroup(rGroup RunGroup) ProcessGroup {
	count := len(rGroup)
	p := make(processGroup, count)
	mChan := make(MemberChan, count)

	for name, runner := range rGroup {
		go mChan.envokeMember(name, runner)
	}
	for i := 0; i < count; i++ {
		p[i] = <-mChan
	}
	return p
}

type Member struct {
	Name    string
	Process ifrit.Process
	Error   error
}

type MemberChan chan Member

func (mChan MemberChan) envokeMember(name string, runner ifrit.Runner) {
	mChan <- Member{Name: name, Process: ifrit.Envoke(runner)}
}

type processGroup []Member

func (group processGroup) Signal(signal os.Signal) {
	for _, m := range group {
		m.Process.Signal(signal)
	}
}

func (group processGroup) Wait() <-chan error {
	errChan := make(chan error, 1)

	go func() {
		errChan <- group.waitForGroup()
	}()

	return errChan
}

func (group processGroup) Exits() <-chan Member {
	memChan := make(MemberChan, len(group))
	for _, m := range group {
		go group.waitForMember(memChan, m)
	}
	return memChan
}

func (group processGroup) waitForMember(memChan MemberChan, m Member) {
	err := <-m.Process.Wait()
	m.Error = err
	memChan <- m
}

func (group processGroup) waitForGroup() error {
	var errMsg string
	for _, m := range group {
		err := <-m.Process.Wait()
		if err != nil {
			errMsg += fmt.Sprintf("%s: %s/n", m.Name, err)
		}
	}

	var err error
	if errMsg != "" {
		err = errors.New(errMsg)
	}
	return err
}
<commit_msg>/ -> \<commit_after>package grouper

import (
	"errors"
	"fmt"
	"os"

	"github.com/tedsuo/ifrit"
)

type ProcessGroup interface {
	ifrit.Process
	Exits() <-chan Member
}

func EnvokeGroup(rGroup RunGroup) ProcessGroup {
	count := len(rGroup)
	p := make(processGroup, count)
	mChan := make(MemberChan, count)

	for name, runner := range rGroup {
		go mChan.envokeMember(name, runner)
	}
	for i := 0; i < count; i++ {
		p[i] = <-mChan
	}
	return p
}

type Member struct {
	Name    string
	Process ifrit.Process
	Error   error
}

type MemberChan chan Member

func (mChan MemberChan) envokeMember(name string, runner ifrit.Runner) {
	mChan <- Member{Name: name, Process: ifrit.Envoke(runner)}
}

type processGroup []Member

func (group processGroup) Signal(signal os.Signal) {
	for _, m := range group {
		m.Process.Signal(signal)
	}
}

func (group processGroup) Wait() <-chan error {
	errChan := make(chan error, 1)

	go func() {
		errChan <- group.waitForGroup()
	}()

	return errChan
}

func (group processGroup) Exits() <-chan Member {
	memChan := make(MemberChan, len(group))
	for _, m := range group {
		go group.waitForMember(memChan, m)
	}
	return memChan
}

func (group processGroup) waitForMember(memChan MemberChan, m Member) {
	err := <-m.Process.Wait()
	m.Error = err
	memChan <- m
}

func (group processGroup) waitForGroup() error {
	var errMsg string
	for _, m := range group {
		err := <-m.Process.Wait()
		if err != nil {
			errMsg += fmt.Sprintf("%s: %s\n", m.Name, err)
		}
	}

	var err error
	if errMsg != "" {
		err = errors.New(errMsg)
	}
	return err
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Johan Brandhorst

// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

// Package status provides a gRPC Status struct compatible
// with the Improbable gRPC-web trailers and errors.
package status

import (
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/metadata"

	// Include gRPC-web JS objects
	_ "github.com/johanbrandhorst/protobuf/grpcweb/grpcwebjs"
)

// Status is a gRPC-web Status.
type Status struct {
	Code     codes.Code
	Message  string
	Trailers metadata.MD
}

// Error returns a string representation of the status
func (s Status) Error() string {
	return "rpc error: code = " + s.Code.String() + " desc = " + s.Message
}

// FromError constructs a Status from an error.
func FromError(err error) *Status {
	s, ok := err.(*Status)
	if !ok {
		s = &Status{
			Code:    codes.Unknown,
			Message: err.Error(),
		}
	}

	return s
}
<commit_msg>Nil errors should create nil status'<commit_after>// Copyright (c) 2017 Johan Brandhorst

// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

// Package status provides a gRPC Status struct compatible
// with the Improbable gRPC-web trailers and errors.
package status

import (
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/metadata"

	// Include gRPC-web JS objects
	_ "github.com/johanbrandhorst/protobuf/grpcweb/grpcwebjs"
)

// Status is a gRPC-web Status.
type Status struct {
	Code     codes.Code
	Message  string
	Trailers metadata.MD
}

// Error returns a string representation of the status
func (s Status) Error() string {
	return "rpc error: code = " + s.Code.String() + " desc = " + s.Message
}

// FromError constructs a Status from an error.
func FromError(err error) *Status {
	if err == nil {
		return nil
	}
	s, ok := err.(*Status)
	if !ok {
		s = &Status{
			Code:    codes.Unknown,
			Message: err.Error(),
		}
	}

	return s
}
<|endoftext|>