<commit_before>// +build experimental

package daemon

import (
	"fmt"
	"os"
	"path/filepath"
	"runtime"

	"github.com/Sirupsen/logrus"
	"github.com/docker/docker/pkg/directory"
	"github.com/docker/docker/pkg/idtools"
	"github.com/docker/docker/runconfig"
)

func setupRemappedRoot(config *Config) ([]idtools.IDMap, []idtools.IDMap, error) {
	if runtime.GOOS != "linux" && config.RemappedRoot != "" {
		return nil, nil, fmt.Errorf("User namespaces are not supported on Linux")
	}

	// if the daemon was started with remapped root option, parse
	// the config option to the int uid,gid values
	var (
		uidMaps, gidMaps []idtools.IDMap
	)
	if config.RemappedRoot != "" {
		username, groupname, err := parseRemappedRoot(config.RemappedRoot)
		if err != nil {
			return nil, nil, err
		}
		if username == "root" {
			// Cannot setup user namespaces with a 1-to-1 mapping; "--root=0:0" is a no-op
			// effectively
			logrus.Warnf("User namespaces: root cannot be remapped with itself; user namespaces are OFF")
			return uidMaps, gidMaps, nil
		}
		logrus.Infof("User namespaces: ID ranges will be mapped to subuid/subgid ranges of: %s:%s", username, groupname)
		// update remapped root setting now that we have resolved them to actual names
		config.RemappedRoot = fmt.Sprintf("%s:%s", username, groupname)

		uidMaps, gidMaps, err = idtools.CreateIDMappings(username, groupname)
		if err != nil {
			return nil, nil, fmt.Errorf("Can't create ID mappings: %v", err)
		}
	}
	return uidMaps, gidMaps, nil
}

func setupDaemonRoot(config *Config, rootDir string, rootUID, rootGID int) error {
	// the main docker root needs to be accessible by all users, as user namespace support
	// will create subdirectories owned by either a) the real system root (when no remapping
	// is setup) or b) the remapped root host ID (when --root=uid:gid is used)
	// for "first time" users of user namespaces, we need to migrate the current directory
	// contents to the "0.0" (root == root "namespace" daemon root)
	nsRoot := "0.0"
	if _, err := os.Stat(rootDir); err == nil {
		// root current exists; we need to check for a prior migration
		if _, err := os.Stat(filepath.Join(rootDir, nsRoot)); err != nil && os.IsNotExist(err) {
			// need to migrate current root to "0.0" subroot
			// 1. create non-usernamespaced root as "0.0"
			if err := os.Mkdir(filepath.Join(rootDir, nsRoot), 0700); err != nil {
				return fmt.Errorf("Cannot create daemon root %q: %v", filepath.Join(rootDir, nsRoot), err)
			}
			// 2. move current root content to "0.0" new subroot
			if err := directory.MoveToSubdir(rootDir, nsRoot); err != nil {
				return fmt.Errorf("Cannot migrate current daemon root %q for user namespaces: %v", rootDir, err)
			}
			// 3. chmod outer root to 755
			if chmodErr := os.Chmod(rootDir, 0755); chmodErr != nil {
				return chmodErr
			}
		}
	} else if os.IsNotExist(err) {
		// no root exists yet, create it 0755 with root:root ownership
		if err := os.MkdirAll(rootDir, 0755); err != nil {
			return err
		}
		// create the "0.0" subroot (so no future "migration" happens of the root)
		if err := os.Mkdir(filepath.Join(rootDir, nsRoot), 0700); err != nil {
			return err
		}
	}

	// for user namespaces we will create a subtree underneath the specified root
	// with any/all specified remapped root uid/gid options on the daemon creating
	// a new subdirectory with ownership set to the remapped uid/gid (so as to allow
	// `chdir()` to work for containers namespaced to that uid/gid)
	if config.RemappedRoot != "" {
		nsRoot = fmt.Sprintf("%d.%d", rootUID, rootGID)
	}
	config.Root = filepath.Join(rootDir, nsRoot)
	logrus.Debugf("Creating actual daemon root: %s", config.Root)

	// Create the root directory if it doesn't exists
	if err := idtools.MkdirAllAs(config.Root, 0700, rootUID, rootGID); err != nil {
		return fmt.Errorf("Cannot create daemon root: %s: %v", config.Root, err)
	}
	return nil
}

func (daemon *Daemon) verifyExperimentalContainerSettings(hostConfig *runconfig.HostConfig, config *runconfig.Config) ([]string, error) {
	if hostConfig.Privileged && daemon.config().RemappedRoot != "" {
		return nil, fmt.Errorf("Privileged mode is incompatible with user namespace mappings")
	}
	return nil, nil
}
<commit_msg>daemon: daemon_experimental.go: fix typo<commit_after>// +build experimental

package daemon

import (
	"fmt"
	"os"
	"path/filepath"
	"runtime"

	"github.com/Sirupsen/logrus"
	"github.com/docker/docker/pkg/directory"
	"github.com/docker/docker/pkg/idtools"
	"github.com/docker/docker/runconfig"
)

func setupRemappedRoot(config *Config) ([]idtools.IDMap, []idtools.IDMap, error) {
	if runtime.GOOS != "linux" && config.RemappedRoot != "" {
		return nil, nil, fmt.Errorf("User namespaces are only supported on Linux")
	}

	// if the daemon was started with remapped root option, parse
	// the config option to the int uid,gid values
	var (
		uidMaps, gidMaps []idtools.IDMap
	)
	if config.RemappedRoot != "" {
		username, groupname, err := parseRemappedRoot(config.RemappedRoot)
		if err != nil {
			return nil, nil, err
		}
		if username == "root" {
			// Cannot setup user namespaces with a 1-to-1 mapping; "--root=0:0" is a no-op
			// effectively
			logrus.Warnf("User namespaces: root cannot be remapped with itself; user namespaces are OFF")
			return uidMaps, gidMaps, nil
		}
		logrus.Infof("User namespaces: ID ranges will be mapped to subuid/subgid ranges of: %s:%s", username, groupname)
		// update remapped root setting now that we have resolved them to actual names
		config.RemappedRoot = fmt.Sprintf("%s:%s", username, groupname)

		uidMaps, gidMaps, err = idtools.CreateIDMappings(username, groupname)
		if err != nil {
			return nil, nil, fmt.Errorf("Can't create ID mappings: %v", err)
		}
	}
	return uidMaps, gidMaps, nil
}

func setupDaemonRoot(config *Config, rootDir string, rootUID, rootGID int) error {
	// the main docker root needs to be accessible by all users, as user namespace support
	// will create subdirectories owned by either a) the real system root (when no remapping
	// is setup) or b) the remapped root host ID (when --root=uid:gid is used)
	// for "first time" users of user namespaces, we need to migrate the current directory
	// contents to the "0.0" (root == root "namespace" daemon root)
	nsRoot := "0.0"
	if _, err := os.Stat(rootDir); err == nil {
		// root current exists; we need to check for a prior migration
		if _, err := os.Stat(filepath.Join(rootDir, nsRoot)); err != nil && os.IsNotExist(err) {
			// need to migrate current root to "0.0" subroot
			// 1. create non-usernamespaced root as "0.0"
			if err := os.Mkdir(filepath.Join(rootDir, nsRoot), 0700); err != nil {
				return fmt.Errorf("Cannot create daemon root %q: %v", filepath.Join(rootDir, nsRoot), err)
			}
			// 2. move current root content to "0.0" new subroot
			if err := directory.MoveToSubdir(rootDir, nsRoot); err != nil {
				return fmt.Errorf("Cannot migrate current daemon root %q for user namespaces: %v", rootDir, err)
			}
			// 3. chmod outer root to 755
			if chmodErr := os.Chmod(rootDir, 0755); chmodErr != nil {
				return chmodErr
			}
		}
	} else if os.IsNotExist(err) {
		// no root exists yet, create it 0755 with root:root ownership
		if err := os.MkdirAll(rootDir, 0755); err != nil {
			return err
		}
		// create the "0.0" subroot (so no future "migration" happens of the root)
		if err := os.Mkdir(filepath.Join(rootDir, nsRoot), 0700); err != nil {
			return err
		}
	}

	// for user namespaces we will create a subtree underneath the specified root
	// with any/all specified remapped root uid/gid options on the daemon creating
	// a new subdirectory with ownership set to the remapped uid/gid (so as to allow
	// `chdir()` to work for containers namespaced to that uid/gid)
	if config.RemappedRoot != "" {
		nsRoot = fmt.Sprintf("%d.%d", rootUID, rootGID)
	}
	config.Root = filepath.Join(rootDir, nsRoot)
	logrus.Debugf("Creating actual daemon root: %s", config.Root)

	// Create the root directory if it doesn't exists
	if err := idtools.MkdirAllAs(config.Root, 0700, rootUID, rootGID); err != nil {
		return fmt.Errorf("Cannot create daemon root: %s: %v", config.Root, err)
	}
	return nil
}

func (daemon *Daemon) verifyExperimentalContainerSettings(hostConfig *runconfig.HostConfig, config *runconfig.Config) ([]string, error) {
	if hostConfig.Privileged && daemon.config().RemappedRoot != "" {
		return nil, fmt.Errorf("Privileged mode is incompatible with user namespace mappings")
	}
	return nil, nil
}
<|endoftext|>


<commit_before>// Copyright 2017 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package git

import (
	"bufio"
	"context"
	"fmt"
	"os/exec"
	"path"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"
)

const (
	// parameters for searching for commit infos. If the untargeted search has
	// not found any entries in the past 5 commits, and 12 or fewer entries
	// remain, then we'll just let the targeted-searching threads finish off,
	// and stop the untargeted search to not interfere.
	deferToTargetedSearchColdStreak          = 5
	deferToTargetedSearchNumRemainingEntries = 12
)

// getCommitsInfoState shared state while getting commit info for entries
type getCommitsInfoState struct {
	lock sync.Mutex
	/* read-only fields, can be read without the mutex */
	// entries and entryPaths are read-only after initialization, so they can
	// safely be read without the mutex
	entries []*TreeEntry
	// set of filepaths to get info for
	entryPaths map[string]struct{}
	treePath   string
	headCommit *Commit

	/* mutable fields, must hold mutex to read or write */
	// map from filepath to commit
	commits map[string]*Commit
	// set of filepaths that have been or are being searched for in a target search
	targetedPaths map[string]struct{}
}

func (state *getCommitsInfoState) numRemainingEntries() int {
	state.lock.Lock()
	defer state.lock.Unlock()
	return len(state.entries) - len(state.commits)
}

// getTargetEntryPath Returns the next path for a targeted-searching thread to
// search for, or returns the empty string if nothing left to search for
func (state *getCommitsInfoState) getTargetedEntryPath() string {
	var targetedEntryPath string
	state.lock.Lock()
	defer state.lock.Unlock()
	for _, entry := range state.entries {
		entryPath := path.Join(state.treePath, entry.Name())
		if _, ok := state.commits[entryPath]; ok {
			continue
		} else if _, ok = state.targetedPaths[entryPath]; ok {
			continue
		}
		targetedEntryPath = entryPath
		state.targetedPaths[entryPath] = struct{}{}
		break
	}
	return targetedEntryPath
}

// repeatedly perform targeted searches for unpopulated entries
func targetedSearch(state *getCommitsInfoState, done chan error) {
	for {
		entryPath := state.getTargetedEntryPath()
		if len(entryPath) == 0 {
			done <- nil
			return
		}
		command := NewCommand("rev-list", "-1", state.headCommit.ID.String(), "--", entryPath)
		output, err := command.RunInDir(state.headCommit.repo.Path)
		if err != nil {
			done <- err
			return
		}
		id, err := NewIDFromString(strings.TrimSpace(output))
		if err != nil {
			done <- err
			return
		}
		commit, err := state.headCommit.repo.getCommit(id)
		if err != nil {
			done <- err
			return
		}
		state.update(entryPath, commit)
	}
}

func initGetCommitInfoState(entries Entries, headCommit *Commit, treePath string) *getCommitsInfoState {
	entryPaths := make(map[string]struct{}, len(entries))
	for _, entry := range entries {
		entryPaths[path.Join(treePath, entry.Name())] = struct{}{}
	}
	if treePath = path.Clean(treePath); treePath == "." {
		treePath = ""
	}
	return &getCommitsInfoState{
		entries:       entries,
		entryPaths:    entryPaths,
		commits:       make(map[string]*Commit, len(entries)),
		targetedPaths: make(map[string]struct{}, len(entries)),
		treePath:      treePath,
		headCommit:    headCommit,
	}
}

// GetCommitsInfo gets information of all commits that are corresponding to these entries
func (tes Entries) GetCommitsInfo(commit *Commit, treePath string) ([][]interface{}, error) {
	state := initGetCommitInfoState(tes, commit, treePath)
	if err := getCommitsInfo(state); err != nil {
		return nil, err
	}
	if len(state.commits) < len(state.entryPaths) {
		return nil, fmt.Errorf("could not find commits for all entries")
	}

	commitsInfo := make([][]interface{}, len(tes))
	for i, entry := range tes {
		commit, ok := state.commits[path.Join(treePath, entry.Name())]
		if !ok {
			return nil, fmt.Errorf("could not find commit for %s", entry.Name())
		}
		switch entry.Type {
		case ObjectCommit:
			subModuleURL := ""
			if subModule, err := state.headCommit.GetSubModule(entry.Name()); err != nil {
				return nil, err
			} else if subModule != nil {
				subModuleURL = subModule.URL
			}
			subModuleFile := NewSubModuleFile(commit, subModuleURL, entry.ID.String())
			commitsInfo[i] = []interface{}{entry, subModuleFile}
		default:
			commitsInfo[i] = []interface{}{entry, commit}
		}
	}
	return commitsInfo, nil
}

func (state *getCommitsInfoState) cleanEntryPath(rawEntryPath string) (string, error) {
	if rawEntryPath[0] == '"' {
		var err error
		rawEntryPath, err = strconv.Unquote(rawEntryPath)
		if err != nil {
			return rawEntryPath, err
		}
	}
	var entryNameStartIndex int
	if len(state.treePath) > 0 {
		entryNameStartIndex = len(state.treePath) + 1
	}

	if index := strings.IndexByte(rawEntryPath[entryNameStartIndex:], '/'); index >= 0 {
		return rawEntryPath[:entryNameStartIndex+index], nil
	}
	return rawEntryPath, nil
}

// update report that the given path was last modified by the given commit.
// Returns whether state.commits was updated
func (state *getCommitsInfoState) update(entryPath string, commit *Commit) bool {
	if _, ok := state.entryPaths[entryPath]; !ok {
		return false
	}

	var updated bool
	state.lock.Lock()
	defer state.lock.Unlock()
	if _, ok := state.commits[entryPath]; !ok {
		state.commits[entryPath] = commit
		updated = true
	}
	return updated
}

const getCommitsInfoPretty = "--pretty=format:%H %ct %s"

func getCommitsInfo(state *getCommitsInfoState) error {
	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
	defer cancel()

	args := []string{"log", state.headCommit.ID.String(), getCommitsInfoPretty, "--name-status", "-c"}
	if len(state.treePath) > 0 {
		args = append(args, "--", state.treePath)
	}
	cmd := exec.CommandContext(ctx, "git", args...)
	cmd.Dir = state.headCommit.repo.Path

	readCloser, err := cmd.StdoutPipe()
	if err != nil {
		return err
	}

	if err := cmd.Start(); err != nil {
		return err
	}

	numThreads := runtime.NumCPU()
	done := make(chan error, numThreads)
	for i := 0; i < numThreads; i++ {
		go targetedSearch(state, done)
	}

	scanner := bufio.NewScanner(readCloser)
	err = state.processGitLogOutput(scanner)
	for i := 0; i < numThreads; i++ {
		doneErr := <-done
		if doneErr != nil && err == nil {
			err = doneErr
		}
	}
	return err
}

func (state *getCommitsInfoState) processGitLogOutput(scanner *bufio.Scanner) error {
	// keep a local cache of seen paths to avoid acquiring a lock for paths
	// we've already seen
	seenPaths := make(map[string]struct{}, len(state.entryPaths))
	// number of consecutive commits without any finds
	coldStreak := 0
	var commit *Commit
	var err error
	for scanner.Scan() {
		line := scanner.Text()
		if len(line) == 0 { // in-between commits
			numRemainingEntries := state.numRemainingEntries()
			if numRemainingEntries == 0 {
				break
			}
			if coldStreak >= deferToTargetedSearchColdStreak &&
				numRemainingEntries <= deferToTargetedSearchNumRemainingEntries {
				// stop this untargeted search, and let the targeted-search threads
				// finish the work
				break
			}
			continue
		}
		if line[0] >= 'A' && line[0] <= 'X' { // a file was changed by the current commit
			// look for the last tab, since for copies (C) and renames (R) two
			// filenames are printed: src, then dest
			tabIndex := strings.LastIndexByte(line, '\t')
			if tabIndex < 1 {
				return fmt.Errorf("misformatted line: %s", line)
			}
			entryPath, err := state.cleanEntryPath(line[tabIndex+1:])
			if err != nil {
				return err
			}
			if _, ok := seenPaths[entryPath]; !ok {
				if state.update(entryPath, commit) {
					coldStreak = 0
				}
				seenPaths[entryPath] = struct{}{}
			}
			continue
		}

		// a new commit
		commit, err = parseCommitInfo(line)
		if err != nil {
			return err
		}
		coldStreak++
	}
	return scanner.Err()
}

// parseCommitInfo parse a commit from a line of `git log` output. Expects the
// line to be formatted according to getCommitsInfoPretty.
func parseCommitInfo(line string) (*Commit, error) {
	if len(line) < 43 {
		return nil, fmt.Errorf("invalid git output: %s", line)
	}
	ref, err := NewIDFromString(line[:40])
	if err != nil {
		return nil, err
	}
	spaceIndex := strings.IndexByte(line[41:], ' ')
	if spaceIndex < 0 {
		return nil, fmt.Errorf("invalid git output: %s", line)
	}
	unixSeconds, err := strconv.Atoi(line[41 : 41+spaceIndex])
	if err != nil {
		return nil, err
	}
	message := line[spaceIndex+42:]
	return &Commit{
		ID:            ref,
		CommitMessage: message,
		Committer: &Signature{
			When: time.Unix(int64(unixSeconds), 0),
		},
	}, nil
}
<commit_msg>Kill zombie subprocesses in GetCommitsInfo (#101)<commit_after>// Copyright 2017 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package git

import (
	"bufio"
	"context"
	"fmt"
	"os/exec"
	"path"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"
)

const (
	// parameters for searching for commit infos. If the untargeted search has
	// not found any entries in the past 5 commits, and 12 or fewer entries
	// remain, then we'll just let the targeted-searching threads finish off,
	// and stop the untargeted search to not interfere.
	deferToTargetedSearchColdStreak          = 5
	deferToTargetedSearchNumRemainingEntries = 12
)

// getCommitsInfoState shared state while getting commit info for entries
type getCommitsInfoState struct {
	lock sync.Mutex
	/* read-only fields, can be read without the mutex */
	// entries and entryPaths are read-only after initialization, so they can
	// safely be read without the mutex
	entries []*TreeEntry
	// set of filepaths to get info for
	entryPaths map[string]struct{}
	treePath   string
	headCommit *Commit

	/* mutable fields, must hold mutex to read or write */
	// map from filepath to commit
	commits map[string]*Commit
	// set of filepaths that have been or are being searched for in a target search
	targetedPaths map[string]struct{}
}

func (state *getCommitsInfoState) numRemainingEntries() int {
	state.lock.Lock()
	defer state.lock.Unlock()
	return len(state.entries) - len(state.commits)
}

// getTargetEntryPath Returns the next path for a targeted-searching thread to
// search for, or returns the empty string if nothing left to search for
func (state *getCommitsInfoState) getTargetedEntryPath() string {
	var targetedEntryPath string
	state.lock.Lock()
	defer state.lock.Unlock()
	for _, entry := range state.entries {
		entryPath := path.Join(state.treePath, entry.Name())
		if _, ok := state.commits[entryPath]; ok {
			continue
		} else if _, ok = state.targetedPaths[entryPath]; ok {
			continue
		}
		targetedEntryPath = entryPath
		state.targetedPaths[entryPath] = struct{}{}
		break
	}
	return targetedEntryPath
}

// repeatedly perform targeted searches for unpopulated entries
func targetedSearch(state *getCommitsInfoState, done chan error) {
	for {
		entryPath := state.getTargetedEntryPath()
		if len(entryPath) == 0 {
			done <- nil
			return
		}
		command := NewCommand("rev-list", "-1", state.headCommit.ID.String(), "--", entryPath)
		output, err := command.RunInDir(state.headCommit.repo.Path)
		if err != nil {
			done <- err
			return
		}
		id, err := NewIDFromString(strings.TrimSpace(output))
		if err != nil {
			done <- err
			return
		}
		commit, err := state.headCommit.repo.getCommit(id)
		if err != nil {
			done <- err
			return
		}
		state.update(entryPath, commit)
	}
}

func initGetCommitInfoState(entries Entries, headCommit *Commit, treePath string) *getCommitsInfoState {
	entryPaths := make(map[string]struct{}, len(entries))
	for _, entry := range entries {
		entryPaths[path.Join(treePath, entry.Name())] = struct{}{}
	}
	if treePath = path.Clean(treePath); treePath == "." {
		treePath = ""
	}
	return &getCommitsInfoState{
		entries:       entries,
		entryPaths:    entryPaths,
		commits:       make(map[string]*Commit, len(entries)),
		targetedPaths: make(map[string]struct{}, len(entries)),
		treePath:      treePath,
		headCommit:    headCommit,
	}
}

// GetCommitsInfo gets information of all commits that are corresponding to these entries
func (tes Entries) GetCommitsInfo(commit *Commit, treePath string) ([][]interface{}, error) {
	state := initGetCommitInfoState(tes, commit, treePath)
	if err := getCommitsInfo(state); err != nil {
		return nil, err
	}
	if len(state.commits) < len(state.entryPaths) {
		return nil, fmt.Errorf("could not find commits for all entries")
	}

	commitsInfo := make([][]interface{}, len(tes))
	for i, entry := range tes {
		commit, ok := state.commits[path.Join(treePath, entry.Name())]
		if !ok {
			return nil, fmt.Errorf("could not find commit for %s", entry.Name())
		}
		switch entry.Type {
		case ObjectCommit:
			subModuleURL := ""
			if subModule, err := state.headCommit.GetSubModule(entry.Name()); err != nil {
				return nil, err
			} else if subModule != nil {
				subModuleURL = subModule.URL
			}
			subModuleFile := NewSubModuleFile(commit, subModuleURL, entry.ID.String())
			commitsInfo[i] = []interface{}{entry, subModuleFile}
		default:
			commitsInfo[i] = []interface{}{entry, commit}
		}
	}
	return commitsInfo, nil
}

func (state *getCommitsInfoState) cleanEntryPath(rawEntryPath string) (string, error) {
	if rawEntryPath[0] == '"' {
		var err error
		rawEntryPath, err = strconv.Unquote(rawEntryPath)
		if err != nil {
			return rawEntryPath, err
		}
	}
	var entryNameStartIndex int
	if len(state.treePath) > 0 {
		entryNameStartIndex = len(state.treePath) + 1
	}

	if index := strings.IndexByte(rawEntryPath[entryNameStartIndex:], '/'); index >= 0 {
		return rawEntryPath[:entryNameStartIndex+index], nil
	}
	return rawEntryPath, nil
}

// update report that the given path was last modified by the given commit.
// Returns whether state.commits was updated
func (state *getCommitsInfoState) update(entryPath string, commit *Commit) bool {
	if _, ok := state.entryPaths[entryPath]; !ok {
		return false
	}

	var updated bool
	state.lock.Lock()
	defer state.lock.Unlock()
	if _, ok := state.commits[entryPath]; !ok {
		state.commits[entryPath] = commit
		updated = true
	}
	return updated
}

const getCommitsInfoPretty = "--pretty=format:%H %ct %s"

func getCommitsInfo(state *getCommitsInfoState) error {
	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
	defer cancel()

	args := []string{"log", state.headCommit.ID.String(), getCommitsInfoPretty, "--name-status", "-c"}
	if len(state.treePath) > 0 {
		args = append(args, "--", state.treePath)
	}
	cmd := exec.CommandContext(ctx, "git", args...)
	cmd.Dir = state.headCommit.repo.Path

	readCloser, err := cmd.StdoutPipe()
	if err != nil {
		return err
	}

	if err := cmd.Start(); err != nil {
		return err
	}
	// it's okay to ignore the error returned by cmd.Wait(); we expect the
	// subprocess to sometimes have a non-zero exit status, since we may
	// prematurely close stdout, resulting in a broken pipe.
	defer cmd.Wait()

	numThreads := runtime.NumCPU()
	done := make(chan error, numThreads)
	for i := 0; i < numThreads; i++ {
		go targetedSearch(state, done)
	}

	scanner := bufio.NewScanner(readCloser)
	err = state.processGitLogOutput(scanner)

	// it is important that we close stdout here; if we do not close
	// stdout, the subprocess will keep running, and the deffered call
	// cmd.Wait() may block for a long time.
	if closeErr := readCloser.Close(); closeErr != nil && err == nil {
		err = closeErr
	}

	for i := 0; i < numThreads; i++ {
		doneErr := <-done
		if doneErr != nil && err == nil {
			err = doneErr
		}
	}
	return err
}

func (state *getCommitsInfoState) processGitLogOutput(scanner *bufio.Scanner) error {
	// keep a local cache of seen paths to avoid acquiring a lock for paths
	// we've already seen
	seenPaths := make(map[string]struct{}, len(state.entryPaths))
	// number of consecutive commits without any finds
	coldStreak := 0
	var commit *Commit
	var err error
	for scanner.Scan() {
		line := scanner.Text()
		if len(line) == 0 { // in-between commits
			numRemainingEntries := state.numRemainingEntries()
			if numRemainingEntries == 0 {
				break
			}
			if coldStreak >= deferToTargetedSearchColdStreak &&
				numRemainingEntries <= deferToTargetedSearchNumRemainingEntries {
				// stop this untargeted search, and let the targeted-search threads
				// finish the work
				break
			}
			continue
		}
		if line[0] >= 'A' && line[0] <= 'X' { // a file was changed by the current commit
			// look for the last tab, since for copies (C) and renames (R) two
			// filenames are printed: src, then dest
			tabIndex := strings.LastIndexByte(line, '\t')
			if tabIndex < 1 {
				return fmt.Errorf("misformatted line: %s", line)
			}
			entryPath, err := state.cleanEntryPath(line[tabIndex+1:])
			if err != nil {
				return err
			}
			if _, ok := seenPaths[entryPath]; !ok {
				if state.update(entryPath, commit) {
					coldStreak = 0
				}
				seenPaths[entryPath] = struct{}{}
			}
			continue
		}

		// a new commit
		commit, err = parseCommitInfo(line)
		if err != nil {
			return err
		}
		coldStreak++
	}
	return scanner.Err()
}

// parseCommitInfo parse a commit from a line of `git log` output. Expects the
// line to be formatted according to getCommitsInfoPretty.
func parseCommitInfo(line string) (*Commit, error) {
	if len(line) < 43 {
		return nil, fmt.Errorf("invalid git output: %s", line)
	}
	ref, err := NewIDFromString(line[:40])
	if err != nil {
		return nil, err
	}
	spaceIndex := strings.IndexByte(line[41:], ' ')
	if spaceIndex < 0 {
		return nil, fmt.Errorf("invalid git output: %s", line)
	}
	unixSeconds, err := strconv.Atoi(line[41 : 41+spaceIndex])
	if err != nil {
		return nil, err
	}
	message := line[spaceIndex+42:]
	return &Commit{
		ID:            ref,
		CommitMessage: message,
		Committer: &Signature{
			When: time.Unix(int64(unixSeconds), 0),
		},
	}, nil
}
<|endoftext|>


<commit_before>package common

import (
	"bufio"
	"net"
	"io"
	"os"
)

// A connection state.
// See RFC 3501 section 3.
type ConnState int

const (
	// In the not authenticated state, the client MUST supply
	// authentication credentials before most commands will be
	// permitted.  This state is entered when a connection starts
	// unless the connection has been pre-authenticated.
	NotAuthenticatedState ConnState = 1

	// In the authenticated state, the client is authenticated and MUST
	// select a mailbox to access before commands that affect messages
	// will be permitted.  This state is entered when a
	// pre-authenticated connection starts, when acceptable
	// authentication credentials have been provided, after an error in
	// selecting a mailbox, or after a successful CLOSE command.
	AuthenticatedState = 1 << 1

	// In a selected state, a mailbox has been selected to access.
	// This state is entered when a mailbox has been successfully
	// selected.
	SelectedState = AuthenticatedState + 1 << 2

	// In the logout state, the connection is being terminated. This
	// state can be entered as a result of a client request (via the
	// LOGOUT command) or by unilateral action on the part of either
	// the client or server.
	LogoutState = 0
)

// A function that upgrades a connection.
//
// This should only be used by libraries implementing an IMAP extension (e.g.
// COMPRESS).
type ConnUpgrader func(conn net.Conn) (net.Conn, error)

// An IMAP connection.
type Conn struct {
	net.Conn
	*Reader
	*Writer

	waits chan struct{}

	// Set to true to print all commands and responses to STDOUT.
	debug bool
}

func (c *Conn) init() {
	r := io.Reader(c.Conn)
	w := io.Writer(c.Conn)

	if c.debug {
		r = io.TeeReader(c.Conn, os.Stdout)
		w = io.MultiWriter(c.Conn, os.Stdout)
	}

	c.Reader.reader = bufio.NewReader(r)
	c.Writer.writer = bufio.NewWriter(w)
}

// Write any buffered data to the underlying connection.
func (c *Conn) Flush() (err error) {
	if err = c.Writer.Flush(); err != nil {
		return
	}

	if f, ok := c.Conn.(Flusher); ok {
		if err = f.Flush(); err != nil {
			return
		}
	}

	return
}

// Upgrade a connection, e.g. wrap an unencrypted connection with an encrypted
// tunnel.
func (c *Conn) Upgrade(upgrader ConnUpgrader) error {
	// Block reads and writes during the upgrading process
	c.waits = make(chan struct{})
	defer close(c.waits)

	upgraded, err := upgrader(c.Conn)
	if err != nil {
		return err
	}

	c.Conn = upgraded
	c.init()
	return nil
}

// Wait for the connection to be ready for reads and writes.
func (c *Conn) Wait() {
	if c.waits != nil {
		<-c.waits
	}
}

// Enable or disable debugging.
func (c *Conn) SetDebug(debug bool) {
	c.debug = debug
	c.init()
}

// Create a new IMAP connection.
func NewConn(conn net.Conn, r *Reader, w *Writer) *Conn {
	c := &Conn{Conn: conn, Reader: r, Writer: w}

	c.init()
	return c
}
<commit_msg>Adds common.ConnectedState<commit_after>package common

import (
	"bufio"
	"net"
	"io"
	"os"
)

// A connection state.
// See RFC 3501 section 3.
type ConnState int

const (
	// In the not authenticated state, the client MUST supply
	// authentication credentials before most commands will be
	// permitted.  This state is entered when a connection starts
	// unless the connection has been pre-authenticated.
	NotAuthenticatedState ConnState = 1

	// In the authenticated state, the client is authenticated and MUST
	// select a mailbox to access before commands that affect messages
	// will be permitted.  This state is entered when a
	// pre-authenticated connection starts, when acceptable
	// authentication credentials have been provided, after an error in
	// selecting a mailbox, or after a successful CLOSE command.
	AuthenticatedState = 1 << 1

	// In a selected state, a mailbox has been selected to access.
	// This state is entered when a mailbox has been successfully
	// selected.
	SelectedState = AuthenticatedState + 1 << 2

	// In the logout state, the connection is being terminated. This
	// state can be entered as a result of a client request (via the
	// LOGOUT command) or by unilateral action on the part of either
	// the client or server.
	LogoutState = 0

	// ConnectedState is any state except LogoutState.
	ConnectedState = NotAuthenticatedState | AuthenticatedState | SelectedState
)

// A function that upgrades a connection.
//
// This should only be used by libraries implementing an IMAP extension (e.g.
// COMPRESS).
type ConnUpgrader func(conn net.Conn) (net.Conn, error)

// An IMAP connection.
type Conn struct {
	net.Conn
	*Reader
	*Writer

	waits chan struct{}

	// Set to true to print all commands and responses to STDOUT.
	debug bool
}

func (c *Conn) init() {
	r := io.Reader(c.Conn)
	w := io.Writer(c.Conn)

	if c.debug {
		r = io.TeeReader(c.Conn, os.Stdout)
		w = io.MultiWriter(c.Conn, os.Stdout)
	}

	c.Reader.reader = bufio.NewReader(r)
	c.Writer.writer = bufio.NewWriter(w)
}

// Write any buffered data to the underlying connection.
func (c *Conn) Flush() (err error) {
	if err = c.Writer.Flush(); err != nil {
		return
	}

	if f, ok := c.Conn.(Flusher); ok {
		if err = f.Flush(); err != nil {
			return
		}
	}

	return
}

// Upgrade a connection, e.g. wrap an unencrypted connection with an encrypted
// tunnel.
func (c *Conn) Upgrade(upgrader ConnUpgrader) error {
	// Block reads and writes during the upgrading process
	c.waits = make(chan struct{})
	defer close(c.waits)

	upgraded, err := upgrader(c.Conn)
	if err != nil {
		return err
	}

	c.Conn = upgraded
	c.init()
	return nil
}

// Wait for the connection to be ready for reads and writes.
func (c *Conn) Wait() {
	if c.waits != nil {
		<-c.waits
	}
}

// Enable or disable debugging.
func (c *Conn) SetDebug(debug bool) {
	c.debug = debug
	c.init()
}

// Create a new IMAP connection.
func NewConn(conn net.Conn, r *Reader, w *Writer) *Conn {
	c := &Conn{Conn: conn, Reader: r, Writer: w}

	c.init()
	return c
}
<|endoftext|>


<commit_before>package camoproxy

import (
	"fmt"
	"github.com/cactus/go-camo/camoproxy/encoding"
	"github.com/gorilla/mux"
	"net/http"
	"net/http/httptest"
	"testing"
	"time"
)

type testStruct struct {
	Desc   string
	Url    string
	Status int
}

var camoConfig = Config{
	HmacKey:        "0x24FEEDFACEDEADBEEFCAFE",
	MaxSize:        5120 * 1024,
	RequestTimeout: time.Duration(4) * time.Second}

func makeReq(testUrl string) (*http.Request, error) {
	k := []byte(camoConfig.HmacKey)
	hexurl := encoding.EncodeUrl(&k, testUrl)
	out := "http://example.com" + hexurl
	req, err := http.NewRequest("GET", out, nil)
	if err != nil {
		return nil, fmt.Errorf("Error building req url '%s': %s", testUrl, err.Error())
	}
	return req, nil
}

func processRequest(req *http.Request, status int) (*httptest.ResponseRecorder, error) {
	camoServer, err := New(camoConfig)
	if err != nil {
		return nil, fmt.Errorf("Error building CamoProxy: %s", err.Error())
	}

	router := mux.NewRouter()
	router.Handle("/{sigHash}/{encodedUrl}", camoServer).Methods("GET")

	record := httptest.NewRecorder()
	router.ServeHTTP(record, req)
	if got, want := record.Code, status; got != want {
		return nil, fmt.Errorf("response code = %d, wanted %d", got, want)
	}
	return record, nil
}

func makeTestReq(testUrl string, status int) (*httptest.ResponseRecorder, error) {
	req, err := makeReq(testUrl)
	if err != nil {
		return nil, err
	}
	record, err := processRequest(req, status)
	if err != nil {
		return nil, err
	}
	return record, nil
}

func TestSimpleValidImageUrl(t *testing.T) {
	t.Parallel()
	testUrl := "http://media.ebaumsworld.com/picture/Mincemeat/Pimp.jpg"
	_, err := makeTestReq(testUrl, 200)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func TestGoogleChartUrl(t *testing.T) {
	t.Parallel()
	testUrl := "http://chart.apis.google.com/chart?chs=920x200&chxl=0:%7C2010-08-13%7C2010-09-12%7C2010-10-12%7C2010-11-11%7C1:%7C0%7C0%7C0%7C0%7C0%7C0&chm=B,EBF5FB,0,0,0&chco=008Cd6&chls=3,1,0&chg=8.3,20,1,4&chd=s:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA&chxt=x,y&cht=lc"
	_, err := makeTestReq(testUrl, 200)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func TestChunkedImageFile(t *testing.T) {
	t.Parallel()
	testUrl := "http://www.igvita.com/posts/12/spdyproxy-diagram.png"
	_, err := makeTestReq(testUrl, 200)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func TestFollowRedirects(t *testing.T) {
	t.Parallel()
	testUrl := "http://cl.ly/1K0X2Y2F1P0o3z140p0d/boom-headshot.gif"
	_, err := makeTestReq(testUrl, 200)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func TestStrangeFormatRedirects(t *testing.T) {
	t.Parallel()
	testUrl := "http://cl.ly/DPcp/Screen%20Shot%202012-01-17%20at%203.42.32%20PM.png"
	_, err := makeTestReq(testUrl, 200)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func TestRedirectsWithPathOnly(t *testing.T) {
	t.Parallel()
	testUrl := "http://blogs.msdn.com/photos/noahric/images/9948044/425x286.aspx"
	_, err := makeTestReq(testUrl, 200)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func TestFollowTempRedirects(t *testing.T) {
	t.Parallel()
	testUrl := "http://d.pr/i/rr7F+"
	_, err := makeTestReq(testUrl, 200)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func Test404InfiniRedirect(t *testing.T) {
	t.Parallel()
	testUrl := "http://modeselektor.herokuapp.com/"
	_, err := makeTestReq(testUrl, 404)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func Test404UrlWithoutHTTPHost(t *testing.T) {
	t.Parallel()
	testUrl := "/picture/Mincemeat/Pimp.jpg"
	_, err := makeTestReq(testUrl, 404)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func Test404ImageLargerThan5MB(t *testing.T) {
	t.Parallel()
	testUrl := "http://apod.nasa.gov/apod/image/0505/larryslookout_spirit_big.jpg"
	_, err := makeTestReq(testUrl, 404)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func Test404HostNotFound(t *testing.T) {
	t.Parallel()
	testUrl := "http://flabergasted.cx"
	_, err := makeTestReq(testUrl, 404)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func Test404OnExcludes(t *testing.T) {
	t.Parallel()
	testUrl := "http://iphone.internal.example.org/foo.cgi"
	_, err := makeTestReq(testUrl, 404)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func Test404OnNonImageContent(t *testing.T) {
	t.Parallel()
	testUrl := "https://github.com/atmos/cinderella/raw/master/bootstrap.sh"
	_, err := makeTestReq(testUrl, 404)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func Test404On10xIpRange(t *testing.T) {
	t.Parallel()
	testUrl := "http://10.0.0.1/foo.cgi"
	_, err := makeTestReq(testUrl, 404)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func Test404On169Dot254Net(t *testing.T) {
	t.Parallel()
	testUrl := "http://169.254.0.1/foo.cgi"
	_, err := makeTestReq(testUrl, 404)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func Test404On172Dot16Net(t *testing.T) {
	t.Parallel()
	for i := 16; i < 32; i++ {
		testUrl := "http://172.%d.0.1/foo.cgi"
		_, err := makeTestReq(fmt.Sprintf(testUrl, i), 404)
		if err != nil {
			t.Errorf(err.Error())
		}
	}
}

func Test404On192Dot168Net(t *testing.T) {
	t.Parallel()
	testUrl := "http://192.168.0.1/foo.cgi"
	_, err := makeTestReq(testUrl, 404)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func TestSupplyAcceptIfNoneGiven(t *testing.T) {
	t.Parallel()
	testUrl := "http://images.anandtech.com/doci/6673/OpenMoboAMD30_575px.png"
	req, err := makeReq(testUrl)
	req.Header.Del("Accept")
	if err != nil {
		t.Errorf(err.Error())
	}
	_, err = processRequest(req, 200)
	if err != nil {
		t.Errorf(err.Error())
	}
}
<commit_msg>bump timeout to keep finicky external test from failing so often<commit_after>package camoproxy

import (
	"fmt"
	"github.com/cactus/go-camo/camoproxy/encoding"
	"github.com/gorilla/mux"
	"net/http"
	"net/http/httptest"
	"testing"
	"time"
)

type testStruct struct {
	Desc   string
	Url    string
	Status int
}

var camoConfig = Config{
	HmacKey:        "0x24FEEDFACEDEADBEEFCAFE",
	MaxSize:        5120 * 1024,
	RequestTimeout: time.Duration(6) * time.Second}

func makeReq(testUrl string) (*http.Request, error) {
	k := []byte(camoConfig.HmacKey)
	hexurl := encoding.EncodeUrl(&k, testUrl)
	out := "http://example.com" + hexurl
	req, err := http.NewRequest("GET", out, nil)
	if err != nil {
		return nil, fmt.Errorf("Error building req url '%s': %s", testUrl, err.Error())
	}
	return req, nil
}

func processRequest(req *http.Request, status int) (*httptest.ResponseRecorder, error) {
	camoServer, err := New(camoConfig)
	if err != nil {
		return nil, fmt.Errorf("Error building CamoProxy: %s", err.Error())
	}

	router := mux.NewRouter()
	router.Handle("/{sigHash}/{encodedUrl}", camoServer).Methods("GET")

	record := httptest.NewRecorder()
	router.ServeHTTP(record, req)
	if got, want := record.Code, status; got != want {
		return nil, fmt.Errorf("response code = %d, wanted %d", got, want)
	}
	return record, nil
}

func makeTestReq(testUrl string, status int) (*httptest.ResponseRecorder, error) {
	req, err := makeReq(testUrl)
	if err != nil {
		return nil, err
	}
	record, err := processRequest(req, status)
	if err != nil {
		return nil, err
	}
	return record, nil
}

func TestSimpleValidImageUrl(t *testing.T) {
	t.Parallel()
	testUrl := "http://media.ebaumsworld.com/picture/Mincemeat/Pimp.jpg"
	_, err := makeTestReq(testUrl, 200)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func TestGoogleChartUrl(t *testing.T) {
	t.Parallel()
	testUrl := "http://chart.apis.google.com/chart?chs=920x200&chxl=0:%7C2010-08-13%7C2010-09-12%7C2010-10-12%7C2010-11-11%7C1:%7C0%7C0%7C0%7C0%7C0%7C0&chm=B,EBF5FB,0,0,0&chco=008Cd6&chls=3,1,0&chg=8.3,20,1,4&chd=s:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA&chxt=x,y&cht=lc"
	_, err := makeTestReq(testUrl, 200)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func TestChunkedImageFile(t *testing.T) {
	t.Parallel()
	testUrl := "http://www.igvita.com/posts/12/spdyproxy-diagram.png"
	_, err := makeTestReq(testUrl, 200)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func TestFollowRedirects(t *testing.T) {
	t.Parallel()
	testUrl := "http://cl.ly/1K0X2Y2F1P0o3z140p0d/boom-headshot.gif"
	_, err := makeTestReq(testUrl, 200)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func TestStrangeFormatRedirects(t *testing.T) {
	t.Parallel()
	testUrl := "http://cl.ly/DPcp/Screen%20Shot%202012-01-17%20at%203.42.32%20PM.png"
	_, err := makeTestReq(testUrl, 200)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func TestRedirectsWithPathOnly(t *testing.T) {
	t.Parallel()
	testUrl := "http://blogs.msdn.com/photos/noahric/images/9948044/425x286.aspx"
	_, err := makeTestReq(testUrl, 200)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func TestFollowTempRedirects(t *testing.T) {
	t.Parallel()
	testUrl := "http://d.pr/i/rr7F+"
	_, err := makeTestReq(testUrl, 200)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func Test404InfiniRedirect(t *testing.T) {
	t.Parallel()
	testUrl := "http://modeselektor.herokuapp.com/"
	_, err := makeTestReq(testUrl, 404)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func Test404UrlWithoutHTTPHost(t *testing.T) {
	t.Parallel()
	testUrl := "/picture/Mincemeat/Pimp.jpg"
	_, err := makeTestReq(testUrl, 404)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func Test404ImageLargerThan5MB(t *testing.T) {
	t.Parallel()
	testUrl := "http://apod.nasa.gov/apod/image/0505/larryslookout_spirit_big.jpg"
	_, err := makeTestReq(testUrl, 404)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func Test404HostNotFound(t *testing.T) {
	t.Parallel()
	testUrl := "http://flabergasted.cx"
	_, err := makeTestReq(testUrl, 404)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func Test404OnExcludes(t *testing.T) {
	t.Parallel()
	testUrl := "http://iphone.internal.example.org/foo.cgi"
	_, err := makeTestReq(testUrl, 404)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func Test404OnNonImageContent(t *testing.T) {
	t.Parallel()
	testUrl := "https://github.com/atmos/cinderella/raw/master/bootstrap.sh"
	_, err := makeTestReq(testUrl, 404)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func Test404On10xIpRange(t *testing.T) {
	t.Parallel()
	testUrl := "http://10.0.0.1/foo.cgi"
	_, err := makeTestReq(testUrl, 404)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func Test404On169Dot254Net(t *testing.T) {
	t.Parallel()
	testUrl := "http://169.254.0.1/foo.cgi"
	_, err := makeTestReq(testUrl, 404)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func Test404On172Dot16Net(t *testing.T) {
	t.Parallel()
	for i := 16; i < 32; i++ {
		testUrl := "http://172.%d.0.1/foo.cgi"
		_, err := makeTestReq(fmt.Sprintf(testUrl, i), 404)
		if err != nil {
			t.Errorf(err.Error())
		}
	}
}

func Test404On192Dot168Net(t *testing.T) {
	t.Parallel()
	testUrl := "http://192.168.0.1/foo.cgi"
	_, err := makeTestReq(testUrl, 404)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func TestSupplyAcceptIfNoneGiven(t *testing.T) {
	t.Parallel()
	testUrl := "http://images.anandtech.com/doci/6673/OpenMoboAMD30_575px.png"
	req, err := makeReq(testUrl)
	req.Header.Del("Accept")
	if err != nil {
		t.Errorf(err.Error())
	}
	_, err = processRequest(req, 200)
	if err != nil {
		t.Errorf(err.Error())
	}
}
<|endoftext|>


<commit_before>package common

import (
	"context"
	"encoding/hex"
	"fmt"
	"math"
	"math/big"
	"math/rand"
	"mime"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"testing"
	"time"

	"github.com/ethereum/go-ethereum/crypto"
	"github.com/golang/glog"
	"github.com/livepeer/go-livepeer/net"
	ffmpeg "github.com/livepeer/lpms/ffmpeg"
	"github.com/pkg/errors"
	"google.golang.org/grpc/peer"
)

// HTTPTimeout timeout used in HTTP connections between nodes
const HTTPTimeout = 8 * time.Second

const maxInt64 = int64(math.MaxInt64)

// using a scaleFactor of 1000 for orchestrator prices
// resulting in max decimal places of 3
const priceScalingFactor = int64(1000)

var (
	ErrParseBigInt = fmt.Errorf("failed to parse big integer")
	ErrProfile     = fmt.Errorf("failed to parse profile")

	ErrFormatProto = fmt.Errorf("unknown VideoProfile format for protobufs")
	ErrFormatMime  = fmt.Errorf("unknown VideoProfile format for mime type")
	ErrFormatExt   = fmt.Errorf("unknown VideoProfile format for extension")
)

func init() {
	rand.Seed(time.Now().UnixNano())
}

func ParseBigInt(num string) (*big.Int, error) {
	bigNum := new(big.Int)
	_, ok := bigNum.SetString(num, 10)

	if !ok {
		return nil, ErrParseBigInt
	} else {
		return bigNum, nil
	}
}

func WaitUntil(waitTime time.Duration, condition func() bool) {
	start := time.Now()
	for time.Since(start) < waitTime {
		if condition() == false {
			time.Sleep(100 * time.Millisecond)
			continue
		}
		break
	}
}

func WaitAssert(t *testing.T, waitTime time.Duration, condition func() bool, msg string) {
	start := time.Now()
	for time.Since(start) < waitTime {
		if condition() == false {
			time.Sleep(100 * time.Millisecond)
			continue
		}
		break
	}

	if condition() == false {
		t.Errorf(msg)
	}
}

func Retry(attempts int, sleep time.Duration, fn func() error) error {
	if err := fn(); err != nil {
		if attempts--; attempts > 0 {
			time.Sleep(sleep)
			return Retry(attempts, 2*sleep, fn)
		}
		return err
	}

	return nil
}

func TxDataToVideoProfile(txData string) ([]ffmpeg.VideoProfile, error) {
	profiles := make([]ffmpeg.VideoProfile, 0)

	if len(txData) == 0 {
		return profiles, nil
	}
	if len(txData) < VideoProfileIDSize {
		return nil, ErrProfile
	}

	for i := 0; i+VideoProfileIDSize <= len(txData); i += VideoProfileIDSize {
		txp := txData[i : i+VideoProfileIDSize]

		p, ok := ffmpeg.VideoProfileLookup[VideoProfileNameLookup[txp]]
		if !ok {
			glog.Errorf("Cannot find video profile for job: %v", txp)
			return nil, ErrProfile // monitor to see if this is too aggressive
		}
		profiles = append(profiles, p)
	}

	return profiles, nil
}

func BytesToVideoProfile(txData []byte) ([]ffmpeg.VideoProfile, error) {
	profiles := make([]ffmpeg.VideoProfile, 0)

	if len(txData) == 0 {
		return profiles, nil
	}
	if len(txData) < VideoProfileIDBytes {
		return nil, ErrProfile
	}

	for i := 0; i+VideoProfileIDBytes <= len(txData); i += VideoProfileIDBytes {
		var txp [VideoProfileIDBytes]byte
		copy(txp[:], txData[i:i+VideoProfileIDBytes])

		p, ok := ffmpeg.VideoProfileLookup[VideoProfileByteLookup[txp]]
		if !ok {
			glog.Errorf("Cannot find video profile for job: %v", txp)
			return nil, ErrProfile // monitor to see if this is too aggressive
		}
		profiles = append(profiles, p)
	}

	return profiles, nil
}

func DefaultProfileName(width int, height int, bitrate int) string {
	return fmt.Sprintf("%dx%d_%d", width, height, bitrate)
}

func FFmpegProfiletoNetProfile(ffmpegProfiles []ffmpeg.VideoProfile) ([]*net.VideoProfile, error) {
	profiles := make([]*net.VideoProfile, 0, len(ffmpegProfiles))
	for _, profile := range ffmpegProfiles {
		width, height, err := ffmpeg.VideoProfileResolution(profile)
		if err != nil {
			return nil, err
		}
		br := strings.Replace(profile.Bitrate, "k", "000", 1)
		bitrate, err := strconv.Atoi(br)
		if err != nil {
			return nil, err
		}
		name := profile.Name
		if name == "" {
			name = "ffmpeg_" + DefaultProfileName(width, height, bitrate)
		}
		format := net.VideoProfile_MPEGTS
		switch profile.Format {
		case ffmpeg.FormatNone:
		case ffmpeg.FormatMPEGTS:
		case ffmpeg.FormatMP4:
			format = net.VideoProfile_MP4
		default:
			return nil, ErrFormatProto
		}
		fullProfile := net.VideoProfile{
			Name:    name,
			Width:   int32(width),
			Height:  int32(height),
			Bitrate: int32(bitrate),
			Fps:     uint32(profile.Framerate),
			Format:  format,
		}
		profiles = append(profiles, &fullProfile)
	}
	return profiles, nil
}

func ProfilesToTranscodeOpts(profiles []ffmpeg.VideoProfile) []byte {
	transOpts := []byte{}
	for _, prof := range profiles {
		transOpts = append(transOpts, crypto.Keccak256([]byte(prof.Name))[0:4]...)
	}
	return transOpts
}

func ProfilesToHex(profiles []ffmpeg.VideoProfile) string {
	return hex.EncodeToString(ProfilesToTranscodeOpts(profiles))
}

func ProfilesNames(profiles []ffmpeg.VideoProfile) string {
	names := make(sort.StringSlice, 0, len(profiles))
	for _, p := range profiles {
		names = append(names, p.Name)
	}
	names.Sort()
	return strings.Join(names, ",")
}

func ProfileExtensionFormat(ext string) ffmpeg.Format {
	p, ok := ffmpeg.ExtensionFormats[ext]
	if !ok {
		return ffmpeg.FormatNone
	}
	return p
}

func ProfileFormatExtension(f ffmpeg.Format) (string, error) {
	ext, ok := ffmpeg.FormatExtensions[f]
	if !ok {
		return "", ErrFormatExt
	}
	return ext, nil
}

func ProfileFormatMimeType(f ffmpeg.Format) (string, error) {
	ext, err := ProfileFormatExtension(f)
	if err != nil {
		return "", err
	}
	m := mime.TypeByExtension(ext)
	if m == "" {
		return "", ErrFormatMime
	}
	return m, nil
}

func GetConnectionAddr(ctx context.Context) string {
	from := "unknown"
	if p, ok := peer.FromContext(ctx); ok {
		from = p.Addr.String()
	}
	return from
}

// GenErrRegex generates a regexp `(err1)|(err2)|(err3)` given a list of
// error strings [err1, err2, err3]
func GenErrRegex(errStrings []string) *regexp.Regexp {
	groups := []string{}
	for _, v := range errStrings {
		groups = append(groups, fmt.Sprintf("(%v)", v))
	}
	return regexp.MustCompile(strings.Join(groups, "|"))
}

// PriceToFixed converts a big.Rat into a fixed point number represented as int64
// using a scaleFactor of 1000 resulting in max decimal places of 3
func PriceToFixed(price *big.Rat) (int64, error) {
	return ratToFixed(price, priceScalingFactor)
}

// FixedToPrice converts an fixed point number with 3 decimal places represented as in int64 into a big.Rat
func FixedToPrice(price int64) *big.Rat {
	return big.NewRat(price, priceScalingFactor)
}

// BaseTokenAmountToFixed converts the base amount of a token (i.e. ETH/LPT) represented as a big.Int into a fixed point number represented
// as a int64 using a scalingFactor of 100000 resulting in max decimal places of 5
func BaseTokenAmountToFixed(baseAmount *big.Int) (int64, error) {
	// The base token amount is denominated in base units of a token
	// Assume that there are 10 ** 18 base units for each token
	// Convert the base token amount to a whole token amount by representing the base token amount
	// as a fraction with the denominator set to 10 ** 18
	var rat *big.Rat
	if baseAmount != nil {
		maxDecimals := new(big.Int).Exp(big.NewInt(10), big.NewInt(18), nil)
		rat = new(big.Rat).SetFrac(baseAmount, maxDecimals)
	}

	return ratToFixed(rat, 100000)
}

func ratToFixed(rat *big.Rat, scalingFactor int64) (int64, error) {
	if rat == nil {
		return 0, fmt.Errorf("reference to rat is nil")
	}
	scaled := new(big.Rat).Mul(rat, big.NewRat(scalingFactor, 1))
	fp, _ := new(big.Float).SetRat(scaled).Int64()
	return fp, nil
}

// RandomIDGenerator generates random hexadecimal string of specified length
// defined as variable for unit tests
var RandomIDGenerator = func(length uint) string {
	x := make([]byte, length, length)
	for i := 0; i < len(x); i++ {
		x[i] = byte(rand.Uint32())
	}
	return hex.EncodeToString(x)
}

// RandName generates random hexadecimal string
func RandName() string {
	return RandomIDGenerator(10)
}

func ToInt64(val *big.Int) int64 {
	if val.Cmp(big.NewInt(maxInt64)) > 0 {
		return maxInt64
	}

	return val.Int64()
}

func RatPriceInfo(priceInfo *net.PriceInfo) (*big.Rat, error) {
	if priceInfo == nil {
		return nil, nil
	}

	pixelsPerUnit := priceInfo.PixelsPerUnit
	if pixelsPerUnit == 0 {
		return nil, errors.New("pixels per unit is 0")
	}

	return big.NewRat(priceInfo.PricePerUnit, pixelsPerUnit), nil
}
<commit_msg>[Embed mime types for video formats defined in LPMS Needed because golang not always loads mime types from host machine<commit_after>package common

import (
	"context"
	"encoding/hex"
	"fmt"
	"math"
	"math/big"
	"math/rand"
	"mime"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"testing"
	"time"

	"github.com/ethereum/go-ethereum/crypto"
	"github.com/golang/glog"
	"github.com/livepeer/go-livepeer/net"
	ffmpeg "github.com/livepeer/lpms/ffmpeg"
	"github.com/pkg/errors"
	"google.golang.org/grpc/peer"
)

// HTTPTimeout timeout used in HTTP connections between nodes
const HTTPTimeout = 8 * time.Second

const maxInt64 = int64(math.MaxInt64)

// using a scaleFactor of 1000 for orchestrator prices
// resulting in max decimal places of 3
const priceScalingFactor = int64(1000)

var (
	ErrParseBigInt = fmt.Errorf("failed to parse big integer")
	ErrProfile     = fmt.Errorf("failed to parse profile")

	ErrFormatProto = fmt.Errorf("unknown VideoProfile format for protobufs")
	ErrFormatMime  = fmt.Errorf("unknown VideoProfile format for mime type")
	ErrFormatExt   = fmt.Errorf("unknown VideoProfile format for extension")

	ext2mime = map[string]string{
		".ts":  "video/mp2t",
		".mp4": "video/mp4",
	}
)

func init() {
	rand.Seed(time.Now().UnixNano())
}

func ParseBigInt(num string) (*big.Int, error) {
	bigNum := new(big.Int)
	_, ok := bigNum.SetString(num, 10)

	if !ok {
		return nil, ErrParseBigInt
	} else {
		return bigNum, nil
	}
}

func WaitUntil(waitTime time.Duration, condition func() bool) {
	start := time.Now()
	for time.Since(start) < waitTime {
		if condition() == false {
			time.Sleep(100 * time.Millisecond)
			continue
		}
		break
	}
}

func WaitAssert(t *testing.T, waitTime time.Duration, condition func() bool, msg string) {
	start := time.Now()
	for time.Since(start) < waitTime {
		if condition() == false {
			time.Sleep(100 * time.Millisecond)
			continue
		}
		break
	}

	if condition() == false {
		t.Errorf(msg)
	}
}

func Retry(attempts int, sleep time.Duration, fn func() error) error {
	if err := fn(); err != nil {
		if attempts--; attempts > 0 {
			time.Sleep(sleep)
			return Retry(attempts, 2*sleep, fn)
		}
		return err
	}

	return nil
}

func TxDataToVideoProfile(txData string) ([]ffmpeg.VideoProfile, error) {
	profiles := make([]ffmpeg.VideoProfile, 0)

	if len(txData) == 0 {
		return profiles, nil
	}
	if len(txData) < VideoProfileIDSize {
		return nil, ErrProfile
	}

	for i := 0; i+VideoProfileIDSize <= len(txData); i += VideoProfileIDSize {
		txp := txData[i : i+VideoProfileIDSize]

		p, ok := ffmpeg.VideoProfileLookup[VideoProfileNameLookup[txp]]
		if !ok {
			glog.Errorf("Cannot find video profile for job: %v", txp)
			return nil, ErrProfile // monitor to see if this is too aggressive
		}
		profiles = append(profiles, p)
	}

	return profiles, nil
}

func BytesToVideoProfile(txData []byte) ([]ffmpeg.VideoProfile, error) {
	profiles := make([]ffmpeg.VideoProfile, 0)

	if len(txData) == 0 {
		return profiles, nil
	}
	if len(txData) < VideoProfileIDBytes {
		return nil, ErrProfile
	}

	for i := 0; i+VideoProfileIDBytes <= len(txData); i += VideoProfileIDBytes {
		var txp [VideoProfileIDBytes]byte
		copy(txp[:], txData[i:i+VideoProfileIDBytes])

		p, ok := ffmpeg.VideoProfileLookup[VideoProfileByteLookup[txp]]
		if !ok {
			glog.Errorf("Cannot find video profile for job: %v", txp)
			return nil, ErrProfile // monitor to see if this is too aggressive
		}
		profiles = append(profiles, p)
	}

	return profiles, nil
}

func DefaultProfileName(width int, height int, bitrate int) string {
	return fmt.Sprintf("%dx%d_%d", width, height, bitrate)
}

func FFmpegProfiletoNetProfile(ffmpegProfiles []ffmpeg.VideoProfile) ([]*net.VideoProfile, error) {
	profiles := make([]*net.VideoProfile, 0, len(ffmpegProfiles))
	for _, profile := range ffmpegProfiles {
		width, height, err := ffmpeg.VideoProfileResolution(profile)
		if err != nil {
			return nil, err
		}
		br := strings.Replace(profile.Bitrate, "k", "000", 1)
		bitrate, err := strconv.Atoi(br)
		if err != nil {
			return nil, err
		}
		name := profile.Name
		if name == "" {
			name = "ffmpeg_" + DefaultProfileName(width, height, bitrate)
		}
		format := net.VideoProfile_MPEGTS
		switch profile.Format {
		case ffmpeg.FormatNone:
		case ffmpeg.FormatMPEGTS:
		case ffmpeg.FormatMP4:
			format = net.VideoProfile_MP4
		default:
			return nil, ErrFormatProto
		}
		fullProfile := net.VideoProfile{
			Name:    name,
			Width:   int32(width),
			Height:  int32(height),
			Bitrate: int32(bitrate),
			Fps:     uint32(profile.Framerate),
			Format:  format,
		}
		profiles = append(profiles, &fullProfile)
	}
	return profiles, nil
}

func ProfilesToTranscodeOpts(profiles []ffmpeg.VideoProfile) []byte {
	transOpts := []byte{}
	for _, prof := range profiles {
		transOpts = append(transOpts, crypto.Keccak256([]byte(prof.Name))[0:4]...)
	}
	return transOpts
}

func ProfilesToHex(profiles []ffmpeg.VideoProfile) string {
	return hex.EncodeToString(ProfilesToTranscodeOpts(profiles))
}

func ProfilesNames(profiles []ffmpeg.VideoProfile) string {
	names := make(sort.StringSlice, 0, len(profiles))
	for _, p := range profiles {
		names = append(names, p.Name)
	}
	names.Sort()
	return strings.Join(names, ",")
}

func ProfileExtensionFormat(ext string) ffmpeg.Format {
	p, ok := ffmpeg.ExtensionFormats[ext]
	if !ok {
		return ffmpeg.FormatNone
	}
	return p
}

func ProfileFormatExtension(f ffmpeg.Format) (string, error) {
	ext, ok := ffmpeg.FormatExtensions[f]
	if !ok {
		return "", ErrFormatExt
	}
	return ext, nil
}

func ProfileFormatMimeType(f ffmpeg.Format) (string, error) {
	ext, err := ProfileFormatExtension(f)
	if err != nil {
		return "", err
	}
	if m, ok := ext2mime[ext]; ok && m != "" {
		return m, nil
	}
	m := mime.TypeByExtension(ext)
	if m == "" {
		return "", ErrFormatMime
	}
	return m, nil
}

func GetConnectionAddr(ctx context.Context) string {
	from := "unknown"
	if p, ok := peer.FromContext(ctx); ok {
		from = p.Addr.String()
	}
	return from
}

// GenErrRegex generates a regexp `(err1)|(err2)|(err3)` given a list of
// error strings [err1, err2, err3]
func GenErrRegex(errStrings []string) *regexp.Regexp {
	groups := []string{}
	for _, v := range errStrings {
		groups = append(groups, fmt.Sprintf("(%v)", v))
	}
	return regexp.MustCompile(strings.Join(groups, "|"))
}

// PriceToFixed converts a big.Rat into a fixed point number represented as int64
// using a scaleFactor of 1000 resulting in max decimal places of 3
func PriceToFixed(price *big.Rat) (int64, error) {
	return ratToFixed(price, priceScalingFactor)
}

// FixedToPrice converts an fixed point number with 3 decimal places represented as in int64 into a big.Rat
func FixedToPrice(price int64) *big.Rat {
	return big.NewRat(price, priceScalingFactor)
}

// BaseTokenAmountToFixed converts the base amount of a token (i.e. ETH/LPT) represented as a big.Int into a fixed point number represented
// as a int64 using a scalingFactor of 100000 resulting in max decimal places of 5
func BaseTokenAmountToFixed(baseAmount *big.Int) (int64, error) {
	// The base token amount is denominated in base units of a token
	// Assume that there are 10 ** 18 base units for each token
	// Convert the base token amount to a whole token amount by representing the base token amount
	// as a fraction with the denominator set to 10 ** 18
	var rat *big.Rat
	if baseAmount != nil {
		maxDecimals := new(big.Int).Exp(big.NewInt(10), big.NewInt(18), nil)
		rat = new(big.Rat).SetFrac(baseAmount, maxDecimals)
	}

	return ratToFixed(rat, 100000)
}

func ratToFixed(rat *big.Rat, scalingFactor int64) (int64, error) {
	if rat == nil {
		return 0, fmt.Errorf("reference to rat is nil")
	}
	scaled := new(big.Rat).Mul(rat, big.NewRat(scalingFactor, 1))
	fp, _ := new(big.Float).SetRat(scaled).Int64()
	return fp, nil
}

// RandomIDGenerator generates random hexadecimal string of specified length
// defined as variable for unit tests
var RandomIDGenerator = func(length uint) string {
	x := make([]byte, length, length)
	for i := 0; i < len(x); i++ {
		x[i] = byte(rand.Uint32())
	}
	return hex.EncodeToString(x)
}

// RandName generates random hexadecimal string
func RandName() string {
	return RandomIDGenerator(10)
}

func ToInt64(val *big.Int) int64 {
	if val.Cmp(big.NewInt(maxInt64)) > 0 {
		return maxInt64
	}

	return val.Int64()
}

func RatPriceInfo(priceInfo *net.PriceInfo) (*big.Rat, error) {
	if priceInfo == nil {
		return nil, nil
	}

	pixelsPerUnit := priceInfo.PixelsPerUnit
	if pixelsPerUnit == 0 {
		return nil, errors.New("pixels per unit is 0")
	}

	return big.NewRat(priceInfo.PricePerUnit, pixelsPerUnit), nil
}
<|endoftext|>


<commit_before>package common

import (
	"context"
	"encoding/hex"
	"fmt"
	"math"
	"math/big"
	"math/rand"
	"mime"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"testing"
	"time"

	"github.com/ethereum/go-ethereum/crypto"
	"github.com/golang/glog"
	"github.com/livepeer/go-livepeer/net"
	ffmpeg "github.com/livepeer/lpms/ffmpeg"
	"github.com/pkg/errors"
	"google.golang.org/grpc/peer"
)

// HTTPTimeout timeout used in HTTP connections between nodes
var HTTPTimeout = 8 * time.Second

const maxInt64 = int64(math.MaxInt64)

// using a scaleFactor of 1000 for orchestrator prices
// resulting in max decimal places of 3
const priceScalingFactor = int64(1000)

var (
	ErrParseBigInt = fmt.Errorf("failed to parse big integer")
	ErrProfile     = fmt.Errorf("failed to parse profile")

	ErrFormatProto = fmt.Errorf("unknown VideoProfile format for protobufs")
	ErrFormatMime  = fmt.Errorf("unknown VideoProfile format for mime type")
	ErrFormatExt   = fmt.Errorf("unknown VideoProfile format for extension")
	ErrProfProto   = fmt.Errorf("unknown VideoProfile profile for protobufs")
	ErrProfName    = fmt.Errorf("unknown VideoProfile profile name")

	ext2mime = map[string]string{
		".ts":  "video/mp2t",
		".mp4": "video/mp4",
	}
)

func init() {
	rand.Seed(time.Now().UnixNano())
}

func ParseBigInt(num string) (*big.Int, error) {
	bigNum := new(big.Int)
	_, ok := bigNum.SetString(num, 10)

	if !ok {
		return nil, ErrParseBigInt
	} else {
		return bigNum, nil
	}
}

func WaitUntil(waitTime time.Duration, condition func() bool) {
	start := time.Now()
	for time.Since(start) < waitTime {
		if condition() == false {
			time.Sleep(100 * time.Millisecond)
			continue
		}
		break
	}
}

func WaitAssert(t *testing.T, waitTime time.Duration, condition func() bool, msg string) {
	start := time.Now()
	for time.Since(start) < waitTime {
		if condition() == false {
			time.Sleep(100 * time.Millisecond)
			continue
		}
		break
	}

	if condition() == false {
		t.Errorf(msg)
	}
}

func Retry(attempts int, sleep time.Duration, fn func() error) error {
	if err := fn(); err != nil {
		if attempts--; attempts > 0 {
			time.Sleep(sleep)
			return Retry(attempts, 2*sleep, fn)
		}
		return err
	}

	return nil
}

func TxDataToVideoProfile(txData string) ([]ffmpeg.VideoProfile, error) {
	profiles := make([]ffmpeg.VideoProfile, 0)

	if len(txData) == 0 {
		return profiles, nil
	}
	if len(txData) < VideoProfileIDSize {
		return nil, ErrProfile
	}

	for i := 0; i+VideoProfileIDSize <= len(txData); i += VideoProfileIDSize {
		txp := txData[i : i+VideoProfileIDSize]

		p, ok := ffmpeg.VideoProfileLookup[VideoProfileNameLookup[txp]]
		if !ok {
			glog.Errorf("Cannot find video profile for job: %v", txp)
			return nil, ErrProfile // monitor to see if this is too aggressive
		}
		profiles = append(profiles, p)
	}

	return profiles, nil
}

func BytesToVideoProfile(txData []byte) ([]ffmpeg.VideoProfile, error) {
	profiles := make([]ffmpeg.VideoProfile, 0)

	if len(txData) == 0 {
		return profiles, nil
	}
	if len(txData) < VideoProfileIDBytes {
		return nil, ErrProfile
	}

	for i := 0; i+VideoProfileIDBytes <= len(txData); i += VideoProfileIDBytes {
		var txp [VideoProfileIDBytes]byte
		copy(txp[:], txData[i:i+VideoProfileIDBytes])

		p, ok := ffmpeg.VideoProfileLookup[VideoProfileByteLookup[txp]]
		if !ok {
			glog.Errorf("Cannot find video profile for job: %v", txp)
			return nil, ErrProfile // monitor to see if this is too aggressive
		}
		profiles = append(profiles, p)
	}

	return profiles, nil
}

func DefaultProfileName(width int, height int, bitrate int) string {
	return fmt.Sprintf("%dx%d_%d", width, height, bitrate)
}

func FFmpegProfiletoNetProfile(ffmpegProfiles []ffmpeg.VideoProfile) ([]*net.VideoProfile, error) {
	profiles := make([]*net.VideoProfile, 0, len(ffmpegProfiles))
	for _, profile := range ffmpegProfiles {
		width, height, err := ffmpeg.VideoProfileResolution(profile)
		if err != nil {
			return nil, err
		}
		br := strings.Replace(profile.Bitrate, "k", "000", 1)
		bitrate, err := strconv.Atoi(br)
		if err != nil {
			return nil, err
		}
		name := profile.Name
		if name == "" {
			name = "ffmpeg_" + DefaultProfileName(width, height, bitrate)
		}
		format := net.VideoProfile_MPEGTS
		switch profile.Format {
		case ffmpeg.FormatNone:
		case ffmpeg.FormatMPEGTS:
		case ffmpeg.FormatMP4:
			format = net.VideoProfile_MP4
		default:
			return nil, ErrFormatProto
		}
		encoderProf := net.VideoProfile_ENCODER_DEFAULT
		switch profile.Profile {
		case ffmpeg.ProfileNone:
		case ffmpeg.ProfileH264Baseline:
			encoderProf = net.VideoProfile_H264_BASELINE
		case ffmpeg.ProfileH264Main:
			encoderProf = net.VideoProfile_H264_MAIN
		case ffmpeg.ProfileH264High:
			encoderProf = net.VideoProfile_H264_HIGH
		case ffmpeg.ProfileH264ConstrainedHigh:
			encoderProf = net.VideoProfile_H264_CONSTRAINED_HIGH
		default:
			return nil, ErrProfProto
		}
		gop := int32(0)
		if profile.GOP < 0 {
			gop = int32(profile.GOP)
		} else {
			gop = int32(profile.GOP.Milliseconds())
		}
		fullProfile := net.VideoProfile{
			Name:    name,
			Width:   int32(width),
			Height:  int32(height),
			Bitrate: int32(bitrate),
			Fps:     uint32(profile.Framerate),
			FpsDen:  uint32(profile.FramerateDen),
			Format:  format,
			Profile: encoderProf,
			Gop:     gop,
		}
		profiles = append(profiles, &fullProfile)
	}
	return profiles, nil
}

func ProfilesToTranscodeOpts(profiles []ffmpeg.VideoProfile) []byte {
	transOpts := []byte{}
	for _, prof := range profiles {
		transOpts = append(transOpts, crypto.Keccak256([]byte(prof.Name))[0:4]...)
	}
	return transOpts
}

func ProfilesToHex(profiles []ffmpeg.VideoProfile) string {
	return hex.EncodeToString(ProfilesToTranscodeOpts(profiles))
}

func ProfilesNames(profiles []ffmpeg.VideoProfile) string {
	names := make(sort.StringSlice, 0, len(profiles))
	for _, p := range profiles {
		names = append(names, p.Name)
	}
	names.Sort()
	return strings.Join(names, ",")
}

func EncoderProfileNameToValue(profile string) (ffmpeg.Profile, error) {
	var EncoderProfileLookup = map[string]ffmpeg.Profile{
		"":                    ffmpeg.ProfileNone,
		"none":                ffmpeg.ProfileNone,
		"h264baseline":        ffmpeg.ProfileH264Baseline,
		"h264main":            ffmpeg.ProfileH264Main,
		"h264high":            ffmpeg.ProfileH264High,
		"h264constrainedhigh": ffmpeg.ProfileH264ConstrainedHigh,
	}
	p, ok := EncoderProfileLookup[strings.ToLower(profile)]
	if !ok {
		return -1, ErrProfName
	}
	return p, nil
}

func ProfileExtensionFormat(ext string) ffmpeg.Format {
	p, ok := ffmpeg.ExtensionFormats[ext]
	if !ok {
		return ffmpeg.FormatNone
	}
	return p
}

func ProfileFormatExtension(f ffmpeg.Format) (string, error) {
	ext, ok := ffmpeg.FormatExtensions[f]
	if !ok {
		return "", ErrFormatExt
	}
	return ext, nil
}

func ProfileFormatMimeType(f ffmpeg.Format) (string, error) {
	ext, err := ProfileFormatExtension(f)
	if err != nil {
		return "", err
	}
	if m, ok := ext2mime[ext]; ok && m != "" {
		return m, nil
	}
	m := mime.TypeByExtension(ext)
	if m == "" {
		return "", ErrFormatMime
	}
	return m, nil
}

func GetConnectionAddr(ctx context.Context) string {
	from := "unknown"
	if p, ok := peer.FromContext(ctx); ok {
		from = p.Addr.String()
	}
	return from
}

// GenErrRegex generates a regexp `(err1)|(err2)|(err3)` given a list of
// error strings [err1, err2, err3]
func GenErrRegex(errStrings []string) *regexp.Regexp {
	groups := []string{}
	for _, v := range errStrings {
		groups = append(groups, fmt.Sprintf("(%v)", v))
	}
	return regexp.MustCompile(strings.Join(groups, "|"))
}

// PriceToFixed converts a big.Rat into a fixed point number represented as int64
// using a scaleFactor of 1000 resulting in max decimal places of 3
func PriceToFixed(price *big.Rat) (int64, error) {
	return ratToFixed(price, priceScalingFactor)
}

// FixedToPrice converts an fixed point number with 3 decimal places represented as in int64 into a big.Rat
func FixedToPrice(price int64) *big.Rat {
	return big.NewRat(price, priceScalingFactor)
}

// BaseTokenAmountToFixed converts the base amount of a token (i.e. ETH/LPT) represented as a big.Int into a fixed point number represented
// as a int64 using a scalingFactor of 100000 resulting in max decimal places of 5
func BaseTokenAmountToFixed(baseAmount *big.Int) (int64, error) {
	// The base token amount is denominated in base units of a token
	// Assume that there are 10 ** 18 base units for each token
	// Convert the base token amount to a whole token amount by representing the base token amount
	// as a fraction with the denominator set to 10 ** 18
	var rat *big.Rat
	if baseAmount != nil {
		maxDecimals := new(big.Int).Exp(big.NewInt(10), big.NewInt(18), nil)
		rat = new(big.Rat).SetFrac(baseAmount, maxDecimals)
	}

	return ratToFixed(rat, 100000)
}

func ratToFixed(rat *big.Rat, scalingFactor int64) (int64, error) {
	if rat == nil {
		return 0, fmt.Errorf("reference to rat is nil")
	}
	scaled := new(big.Rat).Mul(rat, big.NewRat(scalingFactor, 1))
	fp, _ := new(big.Float).SetRat(scaled).Int64()
	return fp, nil
}

// RandomIDGenerator generates random hexadecimal string of specified length
// defined as variable for unit tests
var RandomIDGenerator = func(length uint) string {
	x := make([]byte, length, length)
	for i := 0; i < len(x); i++ {
		x[i] = byte(rand.Uint32())
	}
	return hex.EncodeToString(x)
}

// RandName generates random hexadecimal string
func RandName() string {
	return RandomIDGenerator(10)
}

func ToInt64(val *big.Int) int64 {
	if val.Cmp(big.NewInt(maxInt64)) > 0 {
		return maxInt64
	}

	return val.Int64()
}

func RatPriceInfo(priceInfo *net.PriceInfo) (*big.Rat, error) {
	if priceInfo == nil {
		return nil, nil
	}

	pixelsPerUnit := priceInfo.PixelsPerUnit
	if pixelsPerUnit == 0 {
		return nil, errors.New("pixels per unit is 0")
	}

	return big.NewRat(priceInfo.PricePerUnit, pixelsPerUnit), nil
}
<commit_msg>common: Add RandomBytesGenerator<commit_after>package common

import (
	"context"
	"encoding/hex"
	"fmt"
	"math"
	"math/big"
	"math/rand"
	"mime"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"testing"
	"time"

	"github.com/ethereum/go-ethereum/crypto"
	"github.com/golang/glog"
	"github.com/livepeer/go-livepeer/net"
	ffmpeg "github.com/livepeer/lpms/ffmpeg"
	"github.com/pkg/errors"
	"google.golang.org/grpc/peer"
)

// HTTPTimeout timeout used in HTTP connections between nodes
var HTTPTimeout = 8 * time.Second

const maxInt64 = int64(math.MaxInt64)

// using a scaleFactor of 1000 for orchestrator prices
// resulting in max decimal places of 3
const priceScalingFactor = int64(1000)

var (
	ErrParseBigInt = fmt.Errorf("failed to parse big integer")
	ErrProfile     = fmt.Errorf("failed to parse profile")

	ErrFormatProto = fmt.Errorf("unknown VideoProfile format for protobufs")
	ErrFormatMime  = fmt.Errorf("unknown VideoProfile format for mime type")
	ErrFormatExt   = fmt.Errorf("unknown VideoProfile format for extension")
	ErrProfProto   = fmt.Errorf("unknown VideoProfile profile for protobufs")
	ErrProfName    = fmt.Errorf("unknown VideoProfile profile name")

	ext2mime = map[string]string{
		".ts":  "video/mp2t",
		".mp4": "video/mp4",
	}
)

func init() {
	rand.Seed(time.Now().UnixNano())
}

func ParseBigInt(num string) (*big.Int, error) {
	bigNum := new(big.Int)
	_, ok := bigNum.SetString(num, 10)

	if !ok {
		return nil, ErrParseBigInt
	} else {
		return bigNum, nil
	}
}

func WaitUntil(waitTime time.Duration, condition func() bool) {
	start := time.Now()
	for time.Since(start) < waitTime {
		if condition() == false {
			time.Sleep(100 * time.Millisecond)
			continue
		}
		break
	}
}

func WaitAssert(t *testing.T, waitTime time.Duration, condition func() bool, msg string) {
	start := time.Now()
	for time.Since(start) < waitTime {
		if condition() == false {
			time.Sleep(100 * time.Millisecond)
			continue
		}
		break
	}

	if condition() == false {
		t.Errorf(msg)
	}
}

func Retry(attempts int, sleep time.Duration, fn func() error) error {
	if err := fn(); err != nil {
		if attempts--; attempts > 0 {
			time.Sleep(sleep)
			return Retry(attempts, 2*sleep, fn)
		}
		return err
	}

	return nil
}

func TxDataToVideoProfile(txData string) ([]ffmpeg.VideoProfile, error) {
	profiles := make([]ffmpeg.VideoProfile, 0)

	if len(txData) == 0 {
		return profiles, nil
	}
	if len(txData) < VideoProfileIDSize {
		return nil, ErrProfile
	}

	for i := 0; i+VideoProfileIDSize <= len(txData); i += VideoProfileIDSize {
		txp := txData[i : i+VideoProfileIDSize]

		p, ok := ffmpeg.VideoProfileLookup[VideoProfileNameLookup[txp]]
		if !ok {
			glog.Errorf("Cannot find video profile for job: %v", txp)
			return nil, ErrProfile // monitor to see if this is too aggressive
		}
		profiles = append(profiles, p)
	}

	return profiles, nil
}

func BytesToVideoProfile(txData []byte) ([]ffmpeg.VideoProfile, error) {
	profiles := make([]ffmpeg.VideoProfile, 0)

	if len(txData) == 0 {
		return profiles, nil
	}
	if len(txData) < VideoProfileIDBytes {
		return nil, ErrProfile
	}

	for i := 0; i+VideoProfileIDBytes <= len(txData); i += VideoProfileIDBytes {
		var txp [VideoProfileIDBytes]byte
		copy(txp[:], txData[i:i+VideoProfileIDBytes])

		p, ok := ffmpeg.VideoProfileLookup[VideoProfileByteLookup[txp]]
		if !ok {
			glog.Errorf("Cannot find video profile for job: %v", txp)
			return nil, ErrProfile // monitor to see if this is too aggressive
		}
		profiles = append(profiles, p)
	}

	return profiles, nil
}

func DefaultProfileName(width int, height int, bitrate int) string {
	return fmt.Sprintf("%dx%d_%d", width, height, bitrate)
}

func FFmpegProfiletoNetProfile(ffmpegProfiles []ffmpeg.VideoProfile) ([]*net.VideoProfile, error) {
	profiles := make([]*net.VideoProfile, 0, len(ffmpegProfiles))
	for _, profile := range ffmpegProfiles {
		width, height, err := ffmpeg.VideoProfileResolution(profile)
		if err != nil {
			return nil, err
		}
		br := strings.Replace(profile.Bitrate, "k", "000", 1)
		bitrate, err := strconv.Atoi(br)
		if err != nil {
			return nil, err
		}
		name := profile.Name
		if name == "" {
			name = "ffmpeg_" + DefaultProfileName(width, height, bitrate)
		}
		format := net.VideoProfile_MPEGTS
		switch profile.Format {
		case ffmpeg.FormatNone:
		case ffmpeg.FormatMPEGTS:
		case ffmpeg.FormatMP4:
			format = net.VideoProfile_MP4
		default:
			return nil, ErrFormatProto
		}
		encoderProf := net.VideoProfile_ENCODER_DEFAULT
		switch profile.Profile {
		case ffmpeg.ProfileNone:
		case ffmpeg.ProfileH264Baseline:
			encoderProf = net.VideoProfile_H264_BASELINE
		case ffmpeg.ProfileH264Main:
			encoderProf = net.VideoProfile_H264_MAIN
		case ffmpeg.ProfileH264High:
			encoderProf = net.VideoProfile_H264_HIGH
		case ffmpeg.ProfileH264ConstrainedHigh:
			encoderProf = net.VideoProfile_H264_CONSTRAINED_HIGH
		default:
			return nil, ErrProfProto
		}
		gop := int32(0)
		if profile.GOP < 0 {
			gop = int32(profile.GOP)
		} else {
			gop = int32(profile.GOP.Milliseconds())
		}
		fullProfile := net.VideoProfile{
			Name:    name,
			Width:   int32(width),
			Height:  int32(height),
			Bitrate: int32(bitrate),
			Fps:     uint32(profile.Framerate),
			FpsDen:  uint32(profile.FramerateDen),
			Format:  format,
			Profile: encoderProf,
			Gop:     gop,
		}
		profiles = append(profiles, &fullProfile)
	}
	return profiles, nil
}

func ProfilesToTranscodeOpts(profiles []ffmpeg.VideoProfile) []byte {
	transOpts := []byte{}
	for _, prof := range profiles {
		transOpts = append(transOpts, crypto.Keccak256([]byte(prof.Name))[0:4]...)
	}
	return transOpts
}

func ProfilesToHex(profiles []ffmpeg.VideoProfile) string {
	return hex.EncodeToString(ProfilesToTranscodeOpts(profiles))
}

func ProfilesNames(profiles []ffmpeg.VideoProfile) string {
	names := make(sort.StringSlice, 0, len(profiles))
	for _, p := range profiles {
		names = append(names, p.Name)
	}
	names.Sort()
	return strings.Join(names, ",")
}

func EncoderProfileNameToValue(profile string) (ffmpeg.Profile, error) {
	var EncoderProfileLookup = map[string]ffmpeg.Profile{
		"":                    ffmpeg.ProfileNone,
		"none":                ffmpeg.ProfileNone,
		"h264baseline":        ffmpeg.ProfileH264Baseline,
		"h264main":            ffmpeg.ProfileH264Main,
		"h264high":            ffmpeg.ProfileH264High,
		"h264constrainedhigh": ffmpeg.ProfileH264ConstrainedHigh,
	}
	p, ok := EncoderProfileLookup[strings.ToLower(profile)]
	if !ok {
		return -1, ErrProfName
	}
	return p, nil
}

func ProfileExtensionFormat(ext string) ffmpeg.Format {
	p, ok := ffmpeg.ExtensionFormats[ext]
	if !ok {
		return ffmpeg.FormatNone
	}
	return p
}

func ProfileFormatExtension(f ffmpeg.Format) (string, error) {
	ext, ok := ffmpeg.FormatExtensions[f]
	if !ok {
		return "", ErrFormatExt
	}
	return ext, nil
}

func ProfileFormatMimeType(f ffmpeg.Format) (string, error) {
	ext, err := ProfileFormatExtension(f)
	if err != nil {
		return "", err
	}
	if m, ok := ext2mime[ext]; ok && m != "" {
		return m, nil
	}
	m := mime.TypeByExtension(ext)
	if m == "" {
		return "", ErrFormatMime
	}
	return m, nil
}

func GetConnectionAddr(ctx context.Context) string {
	from := "unknown"
	if p, ok := peer.FromContext(ctx); ok {
		from = p.Addr.String()
	}
	return from
}

// GenErrRegex generates a regexp `(err1)|(err2)|(err3)` given a list of
// error strings [err1, err2, err3]
func GenErrRegex(errStrings []string) *regexp.Regexp {
	groups := []string{}
	for _, v := range errStrings {
		groups = append(groups, fmt.Sprintf("(%v)", v))
	}
	return regexp.MustCompile(strings.Join(groups, "|"))
}

// PriceToFixed converts a big.Rat into a fixed point number represented as int64
// using a scaleFactor of 1000 resulting in max decimal places of 3
func PriceToFixed(price *big.Rat) (int64, error) {
	return ratToFixed(price, priceScalingFactor)
}

// FixedToPrice converts an fixed point number with 3 decimal places represented as in int64 into a big.Rat
func FixedToPrice(price int64) *big.Rat {
	return big.NewRat(price, priceScalingFactor)
}

// BaseTokenAmountToFixed converts the base amount of a token (i.e. ETH/LPT) represented as a big.Int into a fixed point number represented
// as a int64 using a scalingFactor of 100000 resulting in max decimal places of 5
func BaseTokenAmountToFixed(baseAmount *big.Int) (int64, error) {
	// The base token amount is denominated in base units of a token
	// Assume that there are 10 ** 18 base units for each token
	// Convert the base token amount to a whole token amount by representing the base token amount
	// as a fraction with the denominator set to 10 ** 18
	var rat *big.Rat
	if baseAmount != nil {
		maxDecimals := new(big.Int).Exp(big.NewInt(10), big.NewInt(18), nil)
		rat = new(big.Rat).SetFrac(baseAmount, maxDecimals)
	}

	return ratToFixed(rat, 100000)
}

func ratToFixed(rat *big.Rat, scalingFactor int64) (int64, error) {
	if rat == nil {
		return 0, fmt.Errorf("reference to rat is nil")
	}
	scaled := new(big.Rat).Mul(rat, big.NewRat(scalingFactor, 1))
	fp, _ := new(big.Float).SetRat(scaled).Int64()
	return fp, nil
}

// RandomIDGenerator generates random hexadecimal string of specified length
// defined as variable for unit tests
var RandomIDGenerator = func(length uint) string {
	return hex.EncodeToString(RandomBytesGenerator(length))
}

var RandomBytesGenerator = func(length uint) []byte {
	x := make([]byte, length, length)
	for i := 0; i < len(x); i++ {
		x[i] = byte(rand.Uint32())
	}
	return x
}

// RandName generates random hexadecimal string
func RandName() string {
	return RandomIDGenerator(10)
}

func ToInt64(val *big.Int) int64 {
	if val.Cmp(big.NewInt(maxInt64)) > 0 {
		return maxInt64
	}

	return val.Int64()
}

func RatPriceInfo(priceInfo *net.PriceInfo) (*big.Rat, error) {
	if priceInfo == nil {
		return nil, nil
	}

	pixelsPerUnit := priceInfo.PixelsPerUnit
	if pixelsPerUnit == 0 {
		return nil, errors.New("pixels per unit is 0")
	}

	return big.NewRat(priceInfo.PricePerUnit, pixelsPerUnit), nil
}
<|endoftext|>


<commit_before>package main

import (
	"testing"

	cpb "github.com/google/kne/proto/controller"
	"github.com/h-fam/errdiff"
)

func TestNewDeployment(t *testing.T) {

	tests := []struct {
		desc    string
		req     *cpb.CreateClusterRequest
		wantErr string
	}{{
		desc: "full request spec",
		req: &cpb.CreateClusterRequest{
			ClusterSpec: &cpb.CreateClusterRequest_Kind{
				&cpb.KindSpec{
					Name:    "kne",
					Recycle: true,
					Version: "0.11.1",
					Image:   "kindest/node:v1.22.1",
				},
			},
			IngressSpec: &cpb.CreateClusterRequest_Metallb{
				&cpb.MetallbSpec{
					IpCount: 100,
				},
			},
			CniSpec: &cpb.CreateClusterRequest_Meshnet{
				&cpb.MeshnetSpec{},
			},
		},
	}, {
		desc: "bad metallb manifest dir",
		req: &cpb.CreateClusterRequest{
			ClusterSpec: &cpb.CreateClusterRequest_Kind{
				&cpb.KindSpec{
					Name:    "kne",
					Recycle: true,
					Version: "0.11.1",
					Image:   "kindest/node:v1.22.1",
				},
			},
			IngressSpec: &cpb.CreateClusterRequest_Metallb{
				&cpb.MetallbSpec{
					ManifestDir: "/foo",
					IpCount:     100,
				},
			},
			CniSpec: &cpb.CreateClusterRequest_Meshnet{
				&cpb.MeshnetSpec{},
			},
		},
		wantErr: "failed to validate path \"/foo\"",
	}, {
		desc:    "empty kind spec",
		req:     &cpb.CreateClusterRequest{},
		wantErr: "cluster type not supported:",
	}, {
		desc: "empty ingress spec",
		req: &cpb.CreateClusterRequest{
			ClusterSpec: &cpb.CreateClusterRequest_Kind{
				&cpb.KindSpec{
					Name:    "kne",
					Recycle: true,
					Version: "0.11.1",
					Image:   "kindest/node:v1.22.1",
				},
			},
			CniSpec: &cpb.CreateClusterRequest_Meshnet{
				&cpb.MeshnetSpec{
					ManifestDir: "../../bar",
				},
			},
		},
		wantErr: "ingress spec not supported:",
	}, {
		desc: "bad meshnet manifest dir",
		req: &cpb.CreateClusterRequest{
			ClusterSpec: &cpb.CreateClusterRequest_Kind{
				&cpb.KindSpec{
					Name:    "kne",
					Recycle: true,
					Version: "0.11.1",
					Image:   "kindest/node:v1.22.1",
				},
			},
			IngressSpec: &cpb.CreateClusterRequest_Metallb{
				&cpb.MetallbSpec{
					IpCount: 100,
				},
			},
			CniSpec: &cpb.CreateClusterRequest_Meshnet{
				&cpb.MeshnetSpec{
					ManifestDir: "/foo",
				},
			},
		},
		wantErr: "failed to validate path \"/foo\"",
	}}
	for _, tt := range tests {
		t.Run(tt.desc, func(t *testing.T) {
			d, err := newDeployment(tt.req)
			if s := errdiff.Substring(err, tt.wantErr); s != "" {
				t.Fatalf("unexpected error: %s", s)
			}
			if err != nil {
				return
			}
			t.Log(d)
		})
	}

}
<commit_msg>remove empty line<commit_after>package main

import (
	"testing"

	cpb "github.com/google/kne/proto/controller"
	"github.com/h-fam/errdiff"
)

func TestNewDeployment(t *testing.T) {
	tests := []struct {
		desc    string
		req     *cpb.CreateClusterRequest
		wantErr string
	}{{
		desc: "full request spec",
		req: &cpb.CreateClusterRequest{
			ClusterSpec: &cpb.CreateClusterRequest_Kind{
				&cpb.KindSpec{
					Name:    "kne",
					Recycle: true,
					Version: "0.11.1",
					Image:   "kindest/node:v1.22.1",
				},
			},
			IngressSpec: &cpb.CreateClusterRequest_Metallb{
				&cpb.MetallbSpec{
					IpCount: 100,
				},
			},
			CniSpec: &cpb.CreateClusterRequest_Meshnet{
				&cpb.MeshnetSpec{},
			},
		},
	}, {
		desc: "bad metallb manifest dir",
		req: &cpb.CreateClusterRequest{
			ClusterSpec: &cpb.CreateClusterRequest_Kind{
				&cpb.KindSpec{
					Name:    "kne",
					Recycle: true,
					Version: "0.11.1",
					Image:   "kindest/node:v1.22.1",
				},
			},
			IngressSpec: &cpb.CreateClusterRequest_Metallb{
				&cpb.MetallbSpec{
					ManifestDir: "/foo",
					IpCount:     100,
				},
			},
			CniSpec: &cpb.CreateClusterRequest_Meshnet{
				&cpb.MeshnetSpec{},
			},
		},
		wantErr: "failed to validate path \"/foo\"",
	}, {
		desc:    "empty kind spec",
		req:     &cpb.CreateClusterRequest{},
		wantErr: "cluster type not supported:",
	}, {
		desc: "empty ingress spec",
		req: &cpb.CreateClusterRequest{
			ClusterSpec: &cpb.CreateClusterRequest_Kind{
				&cpb.KindSpec{
					Name:    "kne",
					Recycle: true,
					Version: "0.11.1",
					Image:   "kindest/node:v1.22.1",
				},
			},
			CniSpec: &cpb.CreateClusterRequest_Meshnet{
				&cpb.MeshnetSpec{
					ManifestDir: "../../bar",
				},
			},
		},
		wantErr: "ingress spec not supported:",
	}, {
		desc: "bad meshnet manifest dir",
		req: &cpb.CreateClusterRequest{
			ClusterSpec: &cpb.CreateClusterRequest_Kind{
				&cpb.KindSpec{
					Name:    "kne",
					Recycle: true,
					Version: "0.11.1",
					Image:   "kindest/node:v1.22.1",
				},
			},
			IngressSpec: &cpb.CreateClusterRequest_Metallb{
				&cpb.MetallbSpec{
					IpCount: 100,
				},
			},
			CniSpec: &cpb.CreateClusterRequest_Meshnet{
				&cpb.MeshnetSpec{
					ManifestDir: "/foo",
				},
			},
		},
		wantErr: "failed to validate path \"/foo\"",
	}}
	for _, tt := range tests {
		t.Run(tt.desc, func(t *testing.T) {
			d, err := newDeployment(tt.req)
			if s := errdiff.Substring(err, tt.wantErr); s != "" {
				t.Fatalf("unexpected error: %s", s)
			}
			if err != nil {
				return
			}
			t.Log(d)
		})
	}

}
<|endoftext|>


<commit_before>// Copyright 2014 Markus Dittrich. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//
// nutmeg is a unit and regression test framework for MCell
package main

import (
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"math/rand"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"time"
)

// global settings
// NOTE: With exception of rng these should eventually come from a settings file
var testNames []string
var mcellPath string
var testDir string
var rng *rand.Rand

// command line flags
var listTestsFlag bool
var runTestsFlag bool
var cleanTestOutput bool
var testSelection string
var numSimJobs int
var numTestJobs int

// initialize list of available unit tests
func init() {
	flag.BoolVar(&listTestsFlag, "l", false, "show available test cases")
	flag.BoolVar(&runTestsFlag, "r", false, "run specified tests")
	flag.BoolVar(&cleanTestOutput, "c", false, "clean temporary test data")
	flag.StringVar(&testSelection, "s", "all", "select test to run (default: all tests)")
	flag.IntVar(&numSimJobs, "n", 2, "number of concurrent simulation jobs (default: 2)")
	flag.IntVar(&numTestJobs, "m", 2, "number of concurrent test jobs (default: 2)")

	rng = rand.New(rand.NewSource(time.Now().UnixNano()))
	testDir = "/Users/markus/programming/go/src/github.com/haskelladdict/nutmeg/tests/"
	mcellPath = "/Users/markus/programming/c/mcell/mcell-trunk/build/mcell"
}

// main routine
func main() {

	testNames, err := gatherTests(testDir)
	if err != nil {
		log.Fatal("Could not determine list of available test cases")
	}

	flag.Parse()

	switch {
	case listTestsFlag:
		fmt.Println("Available tests:")
		fmt.Println("----------------")
		for i, t := range testNames {
			fmt.Printf("[%d] %-20s\n", i, t)
		}

	case cleanTestOutput:
		tests := extractTestCases(testSelection)
		if err := cleanOutput(tests); err != nil {
			log.Fatal(err)
		}

	case runTestsFlag:
		tests := extractTestCases(testSelection)
		runTests(tests)

	default:
		flag.PrintDefaults()
	}
}

// extractTestCases parses the test selection string and assembles the list
// of requested test cases as fully qualified paths.
// NOTE: The form of the selection string is of the form
//              1,2,5:10,55
//
// Here, each number or range of numbers refers to indexed test cases as
// provided by the -s commandline flag.
// A special case is "all" which refers to all tests.
func extractTestCases(selection string) []string {

	var selectedNames []string
	if selection == "all" {
		selectedNames = testNames
	} else {
		for _, s := range strings.Split(selection, ",") {
			item := strings.TrimSpace(s)

			var items []int
			var err error
			if strings.Contains(item, ":") {
				if items, err = convertRangeToList(item); err != nil {
					log.Printf(fmt.Sprint(err))
					continue
				}
			} else {
				i, err := strconv.Atoi(item)
				if err != nil {
					log.Printf("invalid test selection %s ... skipping", item)
					continue
				}
				items = []int{i}
			}

			for _, i := range items {
				if i < 0 || i >= len(testNames) {
					log.Printf("test selection %d out of valid range (%d-%d) ... skipping",
						i, 0, len(testNames)-1)
					continue
				}
				selectedNames = append(selectedNames, testNames[i])
			}
		}
	}

	testPaths := make([]string, len(selectedNames))
	for i, name := range selectedNames {
		testPaths[i] = filepath.Join(testDir, name)
	}
	return testPaths
}

// convertRangeToList converts a single string containing a range statement
// of the form "4:9" into an explicit integer list describing the
// range [4, 5, 6, 7, 8, 9]
func convertRangeToList(rangeStatement string) ([]int, error) {

	rangeEndpoints := strings.Split(rangeStatement, ":")
	if len(rangeEndpoints) != 2 {
		return nil, errors.New(
			fmt.Sprintf("range selection %s not valid", rangeStatement))
	}

	var rangeBegin int
	var err error
	if rangeBegin, err = strconv.Atoi(rangeEndpoints[0]); err != nil {
		return nil, errors.New(
			fmt.Sprintf("invalid range start character %s", rangeEndpoints[0]))
	}

	var rangeEnd int
	if rangeEnd, err = strconv.Atoi(rangeEndpoints[1]); err != nil {
		return nil, errors.New(
			fmt.Sprintf("invalid range end character %s", rangeEndpoints[1]))
	}

	var newRange []int
	for i := rangeBegin; i <= rangeEnd; i++ {
		newRange = append(newRange, i)
	}

	return newRange, nil
}

// gatherTests determines the list of available test cases and orders them
// alphabetically
func gatherTests(testDir string) ([]string, error) {
	dirContent, err := ioutil.ReadDir(testDir)
	if err != nil {
		return nil, err
	}

	var tests []string
	for _, c := range dirContent {
		if c.IsDir() {
			tests = append(tests, c.Name())
		}
	}
	sort.Strings(tests)

	return tests, nil
}
<commit_msg>Simplified commandline flags.<commit_after>// Copyright 2014 Markus Dittrich. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//
// nutmeg is a unit and regression test framework for MCell
package main

import (
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"math/rand"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"time"
)

// global settings
// NOTE: With exception of rng these should eventually come from a settings file
var mcellPath string
var testDir string
var rng *rand.Rand

// command line flags
var listTestsFlag bool
var cleanTestOutput bool

//var testSelection string
var testSelection string
var numSimJobs int
var numTestJobs int

// initialize list of available unit tests
func init() {
	flag.BoolVar(&listTestsFlag, "l", false, "show available test cases")
	flag.BoolVar(&cleanTestOutput, "c", false, "clean temporary test data")
	flag.StringVar(&testSelection, "r", "", "run specified tests (i, i:j, or 'all')")
	flag.IntVar(&numSimJobs, "n", 2, "number of concurrent simulation jobs (default: 2)")
	flag.IntVar(&numTestJobs, "m", 2, "number of concurrent test jobs (default: 2)")

	rng = rand.New(rand.NewSource(time.Now().UnixNano()))
	testDir = "/Users/markus/programming/go/src/github.com/haskelladdict/nutmeg/tests/"
	mcellPath = "/Users/markus/programming/c/mcell/mcell-trunk/build/mcell"
}

// main routine
func main() {

	testNames, err := gatherTests(testDir)
	if err != nil {
		log.Fatal("Could not determine list of available test cases")
	}

	flag.Parse()

	switch {
	case listTestsFlag:
		fmt.Println("Available tests:")
		fmt.Println("----------------")
		for i, t := range testNames {
			fmt.Printf("[%d] %-20s\n", i, t)
		}

	case cleanTestOutput:
		tests := extractTestCases(testSelection, testNames)
		if err := cleanOutput(tests); err != nil {
			log.Fatal(err)
		}

	case testSelection != "":
		tests := extractTestCases(testSelection, testNames)
		runTests(tests)

	default:
		flag.PrintDefaults()
	}
}

// extractTestCases parses the test selection string and assembles the list
// of requested test cases as fully qualified paths.
// NOTE: The form of the selection string is of the form
//              1,2,5:10,55
//
// Here, each number or range of numbers refers to indexed test cases as
// provided by the -s commandline flag.
// A special case is "all" which refers to all tests.
func extractTestCases(selection string, testNames []string) []string {

	var selectedNames []string
	if selection == "all" {
		selectedNames = testNames
	} else {
		for _, s := range strings.Split(selection, ",") {
			item := strings.TrimSpace(s)

			var items []int
			var err error
			if strings.Contains(item, ":") {
				if items, err = convertRangeToList(item); err != nil {
					log.Printf(fmt.Sprint(err))
					continue
				}
			} else {
				i, err := strconv.Atoi(item)
				if err != nil {
					log.Printf("invalid test selection %s ... skipping", item)
					continue
				}
				items = []int{i}
			}

			for _, i := range items {
				if i < 0 || i >= len(testNames) {
					log.Printf("test selection %d out of valid range (%d-%d) ... skipping",
						i, 0, len(testNames)-1)
					continue
				}
				selectedNames = append(selectedNames, testNames[i])
			}
		}
	}

	testPaths := make([]string, len(selectedNames))
	for i, name := range selectedNames {
		testPaths[i] = filepath.Join(testDir, name)
	}
	return testPaths
}

// convertRangeToList converts a single string containing a range statement
// of the form "4:9" into an explicit integer list describing the
// range [4, 5, 6, 7, 8, 9]
func convertRangeToList(rangeStatement string) ([]int, error) {

	rangeEndpoints := strings.Split(rangeStatement, ":")
	if len(rangeEndpoints) != 2 {
		return nil, errors.New(
			fmt.Sprintf("range selection %s not valid", rangeStatement))
	}

	var rangeBegin int
	var err error
	if rangeBegin, err = strconv.Atoi(rangeEndpoints[0]); err != nil {
		return nil, errors.New(
			fmt.Sprintf("invalid range start character %s", rangeEndpoints[0]))
	}

	var rangeEnd int
	if rangeEnd, err = strconv.Atoi(rangeEndpoints[1]); err != nil {
		return nil, errors.New(
			fmt.Sprintf("invalid range end character %s", rangeEndpoints[1]))
	}

	var newRange []int
	for i := rangeBegin; i <= rangeEnd; i++ {
		newRange = append(newRange, i)
	}

	return newRange, nil
}

// gatherTests determines the list of available test cases and orders them
// alphabetically
func gatherTests(testDir string) ([]string, error) {
	dirContent, err := ioutil.ReadDir(testDir)
	if err != nil {
		return nil, err
	}

	var tests []string
	for _, c := range dirContent {
		if c.IsDir() {
			tests = append(tests, c.Name())
		}
	}
	sort.Strings(tests)

	return tests, nil
}
<|endoftext|>