<commit_before>package core

import (
	"bytes"
	"fmt"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/Workiva/go-datastructures/queue"
)

// A TaskType identifies the kind of task returned from NextTask()
type TaskType int

// Note that the ordering of these determines priority.
const (
	// These are for subinclude targets which should be processed before others.
	SubincludeBuild TaskType = iota
	SubincludeParse
	Build
	Parse
	Test
	Stop
)

type pendingTask struct {
	Label    BuildLabel // Label of target to parse
	Dependor BuildLabel // The target that depended on it (only for parse tasks)
	Type     TaskType
}

func (t pendingTask) Compare(that queue.Item) int {
	return int(t.Type - that.(pendingTask).Type)
}

// Passed about to track the current state of the build.
type BuildState struct {
	Graph *BuildGraph
	// Stream of pending tasks
	pendingTasks *queue.PriorityQueue
	// Stream of results from the build
	Results chan *BuildResult
	// Configuration options
	Config *Configuration
	// Hashes of variouts bits of the configuration, used for incrementality.
	Hashes struct {
		// Hash of the general config, not including specialised bits.
		Config []byte
		// Hash of the config relating to containerisation for tests.
		Containerisation []byte
	}
	// Level of verbosity during the build
	Verbosity int
	// Cache to store / retrieve old build results.
	Cache *Cache
	// Targets that we were originally requested to build
	OriginalTargets []BuildLabel
	// Arguments to tests.
	TestArgs []string
	// Labels of targets that we will include / exclude
	Include, Exclude []string
	// Actual targets to exclude from discovery
	ExcludeTargets []BuildLabel
	// True if we require rule hashes to be correctly verified (usually the case).
	VerifyHashes bool
	// Aggregated coverage for this run
	Coverage TestCoverage
	// True if tests should calculate coverage metrics
	NeedCoverage bool
	// True if we intend to build targets. False if we're just parsing
	// (although some may be built if they're needed for parse).
	NeedBuild bool
	// True if we're running tests. False if we're only building or parsing.
	NeedTests bool
	// True if we want to calculate target hashes (ie. 'plz hash').
	NeedHashesOnly bool
	// Number of times to run each test target. 0 == once each, plus flakes if necessary.
	NumTestRuns int
	// True to print the build / test commands as they're run
	PrintCommands bool
	// True to clean working directories after successful builds.
	CleanWorkdirs bool
	// Number of running workers
	numWorkers int
	// Used to count the number of currently active/pending targets
	numActive  int
	numPending int
	numDone    int
	mutex      sync.Mutex
}

// Singleton instance of one of these. Tried to avoid introducing it but it ended up being
// inevitable to make some of the parsing code work.
var State *BuildState

func (state *BuildState) AddActiveTarget() {
	state.mutex.Lock()
	state.numActive++
	state.mutex.Unlock()
}

func (state *BuildState) AddPendingParse(label, dependor BuildLabel, forSubinclude bool) {
	state.mutex.Lock()
	state.numActive++
	state.numPending++
	state.mutex.Unlock()
	if forSubinclude {
		state.pendingTasks.Put(pendingTask{Label: label, Dependor: dependor, Type: SubincludeParse})
	} else {
		state.pendingTasks.Put(pendingTask{Label: label, Dependor: dependor, Type: Parse})
	}
}

func (state *BuildState) AddPendingBuild(label BuildLabel, forSubinclude bool) {
	if forSubinclude {
		state.addPending(label, SubincludeBuild)
	} else {
		state.addPending(label, Build)
	}
}

func (state *BuildState) AddPendingTest(label BuildLabel) {
	if state.NeedTests {
		state.addPending(label, Test)
	}
}

// NextTask receives the next task that should be processed according to the priority queues.
func (state *BuildState) NextTask() (BuildLabel, BuildLabel, TaskType) {
	t, err := state.pendingTasks.Get(1)
	if err != nil {
		log.Fatalf("error receiving next task: %s", err)
	}
	task := t[0].(pendingTask)
	return task.Label, task.Dependor, task.Type
}

func (state *BuildState) addPending(label BuildLabel, t TaskType) {
	state.mutex.Lock()
	state.numPending++
	state.mutex.Unlock()
	state.pendingTasks.Put(pendingTask{Label: label, Type: t})
}

// TaskDone indicates that a single task is finished. Should be called after one is finished with
// a task returned from NextTask().
func (state *BuildState) TaskDone() {
	state.mutex.Lock()
	state.numDone++
	state.numPending--
	if state.numPending <= 0 {
		state.Stop(state.numWorkers)
	}
	state.mutex.Unlock()
}

// Stop adds n stop tasks to the list of pending tasks, which stops n workers.
func (state *BuildState) Stop(n int) {
	for i := 0; i < n; i++ {
		state.pendingTasks.Put(pendingTask{Type: Stop})
	}
}

// IsOriginalTarget returns true if a target is an original target, ie. one specified on the command line.
func (state *BuildState) IsOriginalTarget(label BuildLabel) bool {
	for _, original := range state.OriginalTargets {
		if original == label || (original.IsAllTargets() && original.PackageName == label.PackageName) {
			return true
		}
	}
	return false
}

// SetIncludeAndExclude sets the include / exclude labels.
// Handles build labels on Exclude so should be preferred over setting them directly.
func (state *BuildState) SetIncludeAndExclude(include, exclude []string) {
	state.Include = include
	for _, e := range exclude {
		if LooksLikeABuildLabel(e) {
			state.ExcludeTargets = append(state.ExcludeTargets, parseMaybeRelativeBuildLabel(e, ""))
		} else {
			state.Exclude = append(state.Exclude, e)
		}
	}
}

// AddOriginalTarget adds one of the original targets and enqueues it for parsing / building.
func (state *BuildState) AddOriginalTarget(label BuildLabel) {
	// Check it's not excluded first.
	for _, e := range state.ExcludeTargets {
		if e.includes(label) {
			return
		}
	}
	state.OriginalTargets = append(state.OriginalTargets, label)
	state.AddPendingParse(label, OriginalTarget, false)
}

func (state *BuildState) LogBuildResult(tid int, label BuildLabel, status BuildResultStatus, description string) {
	state.Results <- &BuildResult{
		ThreadId:    tid,
		Time:        time.Now(),
		Label:       label,
		Status:      status,
		Err:         nil,
		Description: description,
	}
}

func (state *BuildState) LogTestResult(tid int, label BuildLabel, status BuildResultStatus, results TestResults, coverage TestCoverage, err error, format string, args ...interface{}) {
	state.Results <- &BuildResult{
		ThreadId:    tid,
		Time:        time.Now(),
		Label:       label,
		Status:      status,
		Err:         err,
		Description: fmt.Sprintf(format, args...),
		Tests:       results,
	}
	state.mutex.Lock()
	defer state.mutex.Unlock()
	state.Coverage.Aggregate(coverage)
}

func (state *BuildState) LogBuildError(tid int, label BuildLabel, status BuildResultStatus, err error, format string, args ...interface{}) {
	state.Results <- &BuildResult{
		ThreadId:    tid,
		Time:        time.Now(),
		Label:       label,
		Status:      status,
		Err:         err,
		Description: fmt.Sprintf(format, args...),
	}
}

func (state *BuildState) NumActive() int {
	state.mutex.Lock()
	defer state.mutex.Unlock()
	return state.numActive
}

func (state *BuildState) NumDone() int {
	state.mutex.Lock()
	defer state.mutex.Unlock()
	return state.numDone
}

// ExpandOriginalTargets expands any pseudo-targets (ie. :all, ... has already been resolved to a bunch :all targets)
// from the set of original targets.
func (state *BuildState) ExpandOriginalTargets() BuildLabels {
	ret := BuildLabels{}
	for _, label := range state.OriginalTargets {
		if label.IsAllTargets() {
			for _, target := range state.Graph.PackageOrDie(label.PackageName).Targets {
				if target.ShouldInclude(state.Include, state.Exclude) && (!state.NeedTests || target.IsTest) {
					ret = append(ret, target.Label)
				}
			}
		} else {
			ret = append(ret, label)
		}
	}
	sort.Sort(ret)
	return ret
}

func NewBuildState(numThreads int, cache *Cache, verbosity int, config *Configuration) *BuildState {
	State = &BuildState{
		Graph:        NewGraph(),
		pendingTasks: queue.NewPriorityQueue(10000, true), // big hint, why not
		Results:      make(chan *BuildResult, numThreads*100),
		Config:       config,
		Verbosity:    verbosity,
		Cache:        cache,
		VerifyHashes: true,
		NeedBuild:    true,
		numActive:    1, // One for the initial target adding on the main thread.
		numPending:   1,
		Coverage:     TestCoverage{Files: map[string][]LineCoverage{}},
		numWorkers:   numThreads,
	}
	State.Hashes.Config = config.Hash()
	State.Hashes.Containerisation = config.ContainerisationHash()
	return State
}

type BuildResult struct {
	// Thread id (or goroutine id, really) that generated this result.
	ThreadId int
	// Timestamp of this event
	Time time.Time
	// Target which has just changed
	Label BuildLabel
	// Its current status
	Status BuildResultStatus
	// Error, only populated for failure statuses
	Err error
	// Description of what's going on right now.
	Description string
	// Test results
	Tests TestResults
}

func NewBuildError(tid int, label BuildLabel, status BuildResultStatus, err error, description string) BuildResult {
	return BuildResult{
		ThreadId:    tid,
		Time:        time.Now(),
		Label:       label,
		Status:      status,
		Err:         err,
		Description: description,
	}
}

type BuildResultStatus int

const (
	PackageParsing BuildResultStatus = iota
	PackageParsed
	ParseFailed
	TargetBuilding
	TargetBuildStopped
	TargetBuilt
	TargetCached
	TargetBuildFailed
	TargetTesting
	TargetTested
	TargetTestFailed
)

func (s BuildResultStatus) Category() string {
	switch s {
	case PackageParsing, PackageParsed, ParseFailed:
		return "Parse"
	case TargetBuilding, TargetBuildStopped, TargetBuilt, TargetBuildFailed:
		return "Build"
	case TargetTesting, TargetTested, TargetTestFailed:
		return "Test"
	default:
		return "Other"
	}
}

type Cache interface {
	// Stores the results of a single build target.
	Store(target *BuildTarget, key []byte)
	// Stores an extra file against a build target.
	// The file name is relative to the target's out directory.
	StoreExtra(target *BuildTarget, key []byte, file string)
	// Retrieves the results of a single build target.
	// If successful, the outputs will be placed into the output file tree.
	Retrieve(target *BuildTarget, key []byte) bool
	// Retrieves an extra file previously stored by StoreExtra.
	// If successful, the file will be placed into the output file tree.
	RetrieveExtra(target *BuildTarget, key []byte, file string) bool
	// Cleans any artifacts associated with this target from the cache, for any possible key.
	Clean(target *BuildTarget)
}

// This is a pretty simple coverage format; we record one int for each line
// stating what its coverage is.
type TestCoverage struct {
	Tests map[BuildLabel]map[string][]LineCoverage
	Files map[string][]LineCoverage
}

// Aggregates results from that coverage object into this one.
func (this *TestCoverage) Aggregate(that TestCoverage) {
	if this.Tests == nil {
		this.Tests = map[BuildLabel]map[string][]LineCoverage{}
	}
	if this.Files == nil {
		this.Files = map[string][]LineCoverage{}
	}

	// Assume that tests are independent (will currently always be the case).
	for label, coverage := range that.Tests {
		this.Tests[label] = coverage
	}
	// Files are more complex since multiple tests can cover the same file.
	// We take the best result for each line from each test.
	for filename, coverage := range that.Files {
		this.Files[filename] = MergeCoverageLines(this.Files[filename], coverage)
	}
}

func MergeCoverageLines(existing, coverage []LineCoverage) []LineCoverage {
	ret := make([]LineCoverage, len(existing))
	copy(ret, existing)
	for i, line := range coverage {
		if i >= len(ret) {
			ret = append(ret, line)
		} else if coverage[i] > ret[i] {
			ret[i] = coverage[i]
		}
	}
	return ret
}

// Returns an ordered slice of all the files we have coverage information for.
func (this TestCoverage) OrderedFiles() []string {
	files := []string{}
	for file, _ := range this.Files {
		if strings.HasPrefix(file, RepoRoot) {
			file = strings.TrimLeft(file[len(RepoRoot):], "/")
		}
		files = append(files, file)
	}
	sort.Strings(files)
	return files
}

func NewTestCoverage() TestCoverage {
	return TestCoverage{
		Tests: map[BuildLabel]map[string][]LineCoverage{},
		Files: map[string][]LineCoverage{},
	}
}

// Produce a string representation of coverage for serialising to file so we don't
// expose the internal enum values (ordering is important so we may want to insert
// new ones later. This format happens to be the same as the one Phabricator uses,
// which is mildly useful to us since we want to integrate with it anyway. See
// https://secure.phabricator.com/book/phabricator/article/arcanist_coverage/
// for more detail of how it works.
func TestCoverageString(lines []LineCoverage) string {
	var buffer bytes.Buffer
	for _, line := range lines {
		buffer.WriteRune(lineCoverageOutput[line])
	}
	return buffer.String()
}

type LineCoverage uint8

const (
	NotExecutable LineCoverage = iota // Line isn't executable (eg. comment, blank)
	Unreachable   LineCoverage = iota // Line is executable but we've determined it can't be reached. So far not used.
	Uncovered     LineCoverage = iota // Line is executable but isn't covered.
	Covered       LineCoverage = iota // Line is executable and covered.
)

var lineCoverageOutput = [...]rune{'N', 'X', 'U', 'C'} // Corresponds to ordering of enum.
<commit_msg>Convert state counter variables to use atomic operations instead of a single global mutex.<commit_after>package core

import (
	"bytes"
	"fmt"
	"sort"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/Workiva/go-datastructures/queue"
)

// A TaskType identifies the kind of task returned from NextTask()
type TaskType int

// Note that the ordering of these determines priority.
const (
	// These are for subinclude targets which should be processed before others.
	SubincludeBuild TaskType = iota
	SubincludeParse
	Build
	Parse
	Test
	Stop
)

type pendingTask struct {
	Label    BuildLabel // Label of target to parse
	Dependor BuildLabel // The target that depended on it (only for parse tasks)
	Type     TaskType
}

func (t pendingTask) Compare(that queue.Item) int {
	return int(t.Type - that.(pendingTask).Type)
}

// Passed about to track the current state of the build.
type BuildState struct {
	Graph *BuildGraph
	// Stream of pending tasks
	pendingTasks *queue.PriorityQueue
	// Stream of results from the build
	Results chan *BuildResult
	// Configuration options
	Config *Configuration
	// Hashes of variouts bits of the configuration, used for incrementality.
	Hashes struct {
		// Hash of the general config, not including specialised bits.
		Config []byte
		// Hash of the config relating to containerisation for tests.
		Containerisation []byte
	}
	// Level of verbosity during the build
	Verbosity int
	// Cache to store / retrieve old build results.
	Cache *Cache
	// Targets that we were originally requested to build
	OriginalTargets []BuildLabel
	// Arguments to tests.
	TestArgs []string
	// Labels of targets that we will include / exclude
	Include, Exclude []string
	// Actual targets to exclude from discovery
	ExcludeTargets []BuildLabel
	// True if we require rule hashes to be correctly verified (usually the case).
	VerifyHashes bool
	// Aggregated coverage for this run
	Coverage TestCoverage
	// True if tests should calculate coverage metrics
	NeedCoverage bool
	// True if we intend to build targets. False if we're just parsing
	// (although some may be built if they're needed for parse).
	NeedBuild bool
	// True if we're running tests. False if we're only building or parsing.
	NeedTests bool
	// True if we want to calculate target hashes (ie. 'plz hash').
	NeedHashesOnly bool
	// Number of times to run each test target. 0 == once each, plus flakes if necessary.
	NumTestRuns int
	// True to print the build / test commands as they're run
	PrintCommands bool
	// True to clean working directories after successful builds.
	CleanWorkdirs bool
	// Number of running workers
	numWorkers int
	// Used to count the number of currently active/pending targets
	numActive  int64
	numPending int64
	numDone    int64
	mutex      sync.Mutex
}

// Singleton instance of one of these. Tried to avoid introducing it but it ended up being
// inevitable to make some of the parsing code work.
var State *BuildState

func (state *BuildState) AddActiveTarget() {
	atomic.AddInt64(&state.numActive, 1)
}

func (state *BuildState) AddPendingParse(label, dependor BuildLabel, forSubinclude bool) {
	atomic.AddInt64(&state.numActive, 1)
	atomic.AddInt64(&state.numPending, 1)
	if forSubinclude {
		state.pendingTasks.Put(pendingTask{Label: label, Dependor: dependor, Type: SubincludeParse})
	} else {
		state.pendingTasks.Put(pendingTask{Label: label, Dependor: dependor, Type: Parse})
	}
}

func (state *BuildState) AddPendingBuild(label BuildLabel, forSubinclude bool) {
	if forSubinclude {
		state.addPending(label, SubincludeBuild)
	} else {
		state.addPending(label, Build)
	}
}

func (state *BuildState) AddPendingTest(label BuildLabel) {
	if state.NeedTests {
		state.addPending(label, Test)
	}
}

// NextTask receives the next task that should be processed according to the priority queues.
func (state *BuildState) NextTask() (BuildLabel, BuildLabel, TaskType) {
	t, err := state.pendingTasks.Get(1)
	if err != nil {
		log.Fatalf("error receiving next task: %s", err)
	}
	task := t[0].(pendingTask)
	return task.Label, task.Dependor, task.Type
}

func (state *BuildState) addPending(label BuildLabel, t TaskType) {
	atomic.AddInt64(&state.numPending, 1)
	state.pendingTasks.Put(pendingTask{Label: label, Type: t})
}

// TaskDone indicates that a single task is finished. Should be called after one is finished with
// a task returned from NextTask().
func (state *BuildState) TaskDone() {
	atomic.AddInt64(&state.numDone, 1)
	if atomic.AddInt64(&state.numPending, -1) <= 0 {
		state.Stop(state.numWorkers)
	}
}

// Stop adds n stop tasks to the list of pending tasks, which stops n workers.
func (state *BuildState) Stop(n int) {
	for i := 0; i < n; i++ {
		state.pendingTasks.Put(pendingTask{Type: Stop})
	}
}

// IsOriginalTarget returns true if a target is an original target, ie. one specified on the command line.
func (state *BuildState) IsOriginalTarget(label BuildLabel) bool {
	for _, original := range state.OriginalTargets {
		if original == label || (original.IsAllTargets() && original.PackageName == label.PackageName) {
			return true
		}
	}
	return false
}

// SetIncludeAndExclude sets the include / exclude labels.
// Handles build labels on Exclude so should be preferred over setting them directly.
func (state *BuildState) SetIncludeAndExclude(include, exclude []string) {
	state.Include = include
	for _, e := range exclude {
		if LooksLikeABuildLabel(e) {
			state.ExcludeTargets = append(state.ExcludeTargets, parseMaybeRelativeBuildLabel(e, ""))
		} else {
			state.Exclude = append(state.Exclude, e)
		}
	}
}

// AddOriginalTarget adds one of the original targets and enqueues it for parsing / building.
func (state *BuildState) AddOriginalTarget(label BuildLabel) {
	// Check it's not excluded first.
	for _, e := range state.ExcludeTargets {
		if e.includes(label) {
			return
		}
	}
	state.OriginalTargets = append(state.OriginalTargets, label)
	state.AddPendingParse(label, OriginalTarget, false)
}

func (state *BuildState) LogBuildResult(tid int, label BuildLabel, status BuildResultStatus, description string) {
	state.Results <- &BuildResult{
		ThreadId:    tid,
		Time:        time.Now(),
		Label:       label,
		Status:      status,
		Err:         nil,
		Description: description,
	}
}

func (state *BuildState) LogTestResult(tid int, label BuildLabel, status BuildResultStatus, results TestResults, coverage TestCoverage, err error, format string, args ...interface{}) {
	state.Results <- &BuildResult{
		ThreadId:    tid,
		Time:        time.Now(),
		Label:       label,
		Status:      status,
		Err:         err,
		Description: fmt.Sprintf(format, args...),
		Tests:       results,
	}
	state.mutex.Lock()
	defer state.mutex.Unlock()
	state.Coverage.Aggregate(coverage)
}

func (state *BuildState) LogBuildError(tid int, label BuildLabel, status BuildResultStatus, err error, format string, args ...interface{}) {
	state.Results <- &BuildResult{
		ThreadId:    tid,
		Time:        time.Now(),
		Label:       label,
		Status:      status,
		Err:         err,
		Description: fmt.Sprintf(format, args...),
	}
}

func (state *BuildState) NumActive() int {
	return int(atomic.LoadInt64(&state.numActive))
}

func (state *BuildState) NumDone() int {
	return int(atomic.LoadInt64(&state.numDone))
}

// ExpandOriginalTargets expands any pseudo-targets (ie. :all, ... has already been resolved to a bunch :all targets)
// from the set of original targets.
func (state *BuildState) ExpandOriginalTargets() BuildLabels {
	ret := BuildLabels{}
	for _, label := range state.OriginalTargets {
		if label.IsAllTargets() {
			for _, target := range state.Graph.PackageOrDie(label.PackageName).Targets {
				if target.ShouldInclude(state.Include, state.Exclude) && (!state.NeedTests || target.IsTest) {
					ret = append(ret, target.Label)
				}
			}
		} else {
			ret = append(ret, label)
		}
	}
	sort.Sort(ret)
	return ret
}

func NewBuildState(numThreads int, cache *Cache, verbosity int, config *Configuration) *BuildState {
	State = &BuildState{
		Graph:        NewGraph(),
		pendingTasks: queue.NewPriorityQueue(10000, true), // big hint, why not
		Results:      make(chan *BuildResult, numThreads*100),
		Config:       config,
		Verbosity:    verbosity,
		Cache:        cache,
		VerifyHashes: true,
		NeedBuild:    true,
		numActive:    1, // One for the initial target adding on the main thread.
		numPending:   1,
		Coverage:     TestCoverage{Files: map[string][]LineCoverage{}},
		numWorkers:   numThreads,
	}
	State.Hashes.Config = config.Hash()
	State.Hashes.Containerisation = config.ContainerisationHash()
	return State
}

type BuildResult struct {
	// Thread id (or goroutine id, really) that generated this result.
	ThreadId int
	// Timestamp of this event
	Time time.Time
	// Target which has just changed
	Label BuildLabel
	// Its current status
	Status BuildResultStatus
	// Error, only populated for failure statuses
	Err error
	// Description of what's going on right now.
	Description string
	// Test results
	Tests TestResults
}

func NewBuildError(tid int, label BuildLabel, status BuildResultStatus, err error, description string) BuildResult {
	return BuildResult{
		ThreadId:    tid,
		Time:        time.Now(),
		Label:       label,
		Status:      status,
		Err:         err,
		Description: description,
	}
}

type BuildResultStatus int

const (
	PackageParsing BuildResultStatus = iota
	PackageParsed
	ParseFailed
	TargetBuilding
	TargetBuildStopped
	TargetBuilt
	TargetCached
	TargetBuildFailed
	TargetTesting
	TargetTested
	TargetTestFailed
)

func (s BuildResultStatus) Category() string {
	switch s {
	case PackageParsing, PackageParsed, ParseFailed:
		return "Parse"
	case TargetBuilding, TargetBuildStopped, TargetBuilt, TargetBuildFailed:
		return "Build"
	case TargetTesting, TargetTested, TargetTestFailed:
		return "Test"
	default:
		return "Other"
	}
}

type Cache interface {
	// Stores the results of a single build target.
	Store(target *BuildTarget, key []byte)
	// Stores an extra file against a build target.
	// The file name is relative to the target's out directory.
	StoreExtra(target *BuildTarget, key []byte, file string)
	// Retrieves the results of a single build target.
	// If successful, the outputs will be placed into the output file tree.
	Retrieve(target *BuildTarget, key []byte) bool
	// Retrieves an extra file previously stored by StoreExtra.
	// If successful, the file will be placed into the output file tree.
	RetrieveExtra(target *BuildTarget, key []byte, file string) bool
	// Cleans any artifacts associated with this target from the cache, for any possible key.
	Clean(target *BuildTarget)
}

// This is a pretty simple coverage format; we record one int for each line
// stating what its coverage is.
type TestCoverage struct {
	Tests map[BuildLabel]map[string][]LineCoverage
	Files map[string][]LineCoverage
}

// Aggregates results from that coverage object into this one.
func (this *TestCoverage) Aggregate(that TestCoverage) {
	if this.Tests == nil {
		this.Tests = map[BuildLabel]map[string][]LineCoverage{}
	}
	if this.Files == nil {
		this.Files = map[string][]LineCoverage{}
	}

	// Assume that tests are independent (will currently always be the case).
	for label, coverage := range that.Tests {
		this.Tests[label] = coverage
	}
	// Files are more complex since multiple tests can cover the same file.
	// We take the best result for each line from each test.
	for filename, coverage := range that.Files {
		this.Files[filename] = MergeCoverageLines(this.Files[filename], coverage)
	}
}

func MergeCoverageLines(existing, coverage []LineCoverage) []LineCoverage {
	ret := make([]LineCoverage, len(existing))
	copy(ret, existing)
	for i, line := range coverage {
		if i >= len(ret) {
			ret = append(ret, line)
		} else if coverage[i] > ret[i] {
			ret[i] = coverage[i]
		}
	}
	return ret
}

// Returns an ordered slice of all the files we have coverage information for.
func (this TestCoverage) OrderedFiles() []string {
	files := []string{}
	for file, _ := range this.Files {
		if strings.HasPrefix(file, RepoRoot) {
			file = strings.TrimLeft(file[len(RepoRoot):], "/")
		}
		files = append(files, file)
	}
	sort.Strings(files)
	return files
}

func NewTestCoverage() TestCoverage {
	return TestCoverage{
		Tests: map[BuildLabel]map[string][]LineCoverage{},
		Files: map[string][]LineCoverage{},
	}
}

// Produce a string representation of coverage for serialising to file so we don't
// expose the internal enum values (ordering is important so we may want to insert
// new ones later. This format happens to be the same as the one Phabricator uses,
// which is mildly useful to us since we want to integrate with it anyway. See
// https://secure.phabricator.com/book/phabricator/article/arcanist_coverage/
// for more detail of how it works.
func TestCoverageString(lines []LineCoverage) string {
	var buffer bytes.Buffer
	for _, line := range lines {
		buffer.WriteRune(lineCoverageOutput[line])
	}
	return buffer.String()
}

type LineCoverage uint8

const (
	NotExecutable LineCoverage = iota // Line isn't executable (eg. comment, blank)
	Unreachable   LineCoverage = iota // Line is executable but we've determined it can't be reached. So far not used.
	Uncovered     LineCoverage = iota // Line is executable but isn't covered.
	Covered       LineCoverage = iota // Line is executable and covered.
)

var lineCoverageOutput = [...]rune{'N', 'X', 'U', 'C'} // Corresponds to ordering of enum.
<|endoftext|>


<commit_before>// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Copyright 2013 The ql Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSES/QL-LICENSE file.

package session

import (
	"context"
	"sync"
	"sync/atomic"
	"time"

	"github.com/pingcap/errors"
	"github.com/pingcap/tidb/config"
	"github.com/pingcap/tidb/domain"
	"github.com/pingcap/tidb/errno"
	"github.com/pingcap/tidb/executor"
	"github.com/pingcap/tidb/kv"
	"github.com/pingcap/tidb/parser"
	"github.com/pingcap/tidb/parser/ast"
	"github.com/pingcap/tidb/sessionctx"
	"github.com/pingcap/tidb/sessionctx/variable"
	"github.com/pingcap/tidb/util"
	"github.com/pingcap/tidb/util/chunk"
	"github.com/pingcap/tidb/util/dbterror"
	"github.com/pingcap/tidb/util/logutil"
	"github.com/pingcap/tidb/util/sqlexec"
	"go.uber.org/zap"
)

type domainMap struct {
	domains map[string]*domain.Domain
	mu      sync.RWMutex
}

func (dm *domainMap) Get(store kv.Storage) (d *domain.Domain, err error) {
	dm.mu.RLock()

	// If this is the only domain instance, and the caller doesn't provide store.
	if len(dm.domains) == 1 && store == nil {
		for _, r := range dm.domains {
			return r, nil
		}
	}

	key := store.UUID()
	d = dm.domains[key]
	dm.mu.RUnlock()
	if d != nil {
		return
	}

	ddlLease := time.Duration(atomic.LoadInt64(&schemaLease))
	statisticLease := time.Duration(atomic.LoadInt64(&statsLease))
	idxUsageSyncLease := GetIndexUsageSyncLease()
	planReplayerGCLease := GetPlanReplayerGCLease()
	err = util.RunWithRetry(util.DefaultMaxRetries, util.RetryInterval, func() (retry bool, err1 error) {
		logutil.BgLogger().Info("new domain",
			zap.String("store", store.UUID()),
			zap.Stringer("ddl lease", ddlLease),
			zap.Stringer("stats lease", statisticLease),
			zap.Stringer("index usage sync lease", idxUsageSyncLease))
		factory := createSessionFunc(store)
		sysFactory := createSessionWithDomainFunc(store)
		onClose := func() {
			dm.Delete(store)
		}
		d = domain.NewDomain(store, ddlLease, statisticLease, idxUsageSyncLease, planReplayerGCLease, factory, onClose)
		err1 = d.Init(ddlLease, sysFactory)
		if err1 != nil {
			// If we don't clean it, there are some dirty data when retrying the function of Init.
			d.Close()
			logutil.BgLogger().Error("[ddl] init domain failed",
				zap.Error(err1))
		}
		return true, err1
	})
	if err != nil {
		return nil, err
	}
	dm.Set(store, d)

	return
}

func (dm *domainMap) Delete(store kv.Storage) {
	dm.mu.Lock()
	delete(dm.domains, store.UUID())
	dm.mu.Unlock()
}

func (dm *domainMap) Set(store kv.Storage, domain *domain.Domain) {
	dm.mu.Lock()
	dm.domains[store.UUID()] = domain
	dm.mu.Unlock()
}

var (
	domap = &domainMap{
		domains: map[string]*domain.Domain{},
	}
	// store.UUID()-> IfBootstrapped
	storeBootstrapped     = make(map[string]bool)
	storeBootstrappedLock sync.Mutex

	// schemaLease is the time for re-updating remote schema.
	// In online DDL, we must wait 2 * SchemaLease time to guarantee
	// all servers get the neweset schema.
	// Default schema lease time is 1 second, you can change it with a proper time,
	// but you must know that too little may cause badly performance degradation.
	// For production, you should set a big schema lease, like 300s+.
	schemaLease = int64(1 * time.Second)

	// statsLease is the time for reload stats table.
	statsLease = int64(3 * time.Second)

	// indexUsageSyncLease is the time for index usage synchronization.
	// Because we have not completed GC and other functions, we set it to 0.
	// TODO: Set indexUsageSyncLease to 60s.
	indexUsageSyncLease = int64(0 * time.Second)

	// planReplayerGCLease is the time for plan replayer gc.
	planReplayerGCLease = int64(10 * time.Minute)
)

// ResetStoreForWithTiKVTest is only used in the test code.
// TODO: Remove domap and storeBootstrapped. Use store.SetOption() to do it.
func ResetStoreForWithTiKVTest(store kv.Storage) {
	domap.Delete(store)
	unsetStoreBootstrapped(store.UUID())
}

func setStoreBootstrapped(storeUUID string) {
	storeBootstrappedLock.Lock()
	defer storeBootstrappedLock.Unlock()
	storeBootstrapped[storeUUID] = true
}

// unsetStoreBootstrapped delete store uuid from stored bootstrapped map.
// currently this function only used for test.
func unsetStoreBootstrapped(storeUUID string) {
	storeBootstrappedLock.Lock()
	defer storeBootstrappedLock.Unlock()
	delete(storeBootstrapped, storeUUID)
}

// SetSchemaLease changes the default schema lease time for DDL.
// This function is very dangerous, don't use it if you really know what you do.
// SetSchemaLease only affects not local storage after bootstrapped.
func SetSchemaLease(lease time.Duration) {
	atomic.StoreInt64(&schemaLease, int64(lease))
}

// SetStatsLease changes the default stats lease time for loading stats info.
func SetStatsLease(lease time.Duration) {
	atomic.StoreInt64(&statsLease, int64(lease))
}

// SetIndexUsageSyncLease changes the default index usage sync lease time for loading info.
func SetIndexUsageSyncLease(lease time.Duration) {
	atomic.StoreInt64(&indexUsageSyncLease, int64(lease))
}

// GetIndexUsageSyncLease returns the index usage sync lease time.
func GetIndexUsageSyncLease() time.Duration {
	return time.Duration(atomic.LoadInt64(&indexUsageSyncLease))
}

// SetPlanReplayerGCLease changes the default plan repalyer gc lease time.
func SetPlanReplayerGCLease(lease time.Duration) {
	atomic.StoreInt64(&planReplayerGCLease, int64(lease))
}

// GetPlanReplayerGCLease returns the plan replayer gc lease time.
func GetPlanReplayerGCLease() time.Duration {
	return time.Duration(atomic.LoadInt64(&planReplayerGCLease))
}

// DisableStats4Test disables the stats for tests.
func DisableStats4Test() {
	SetStatsLease(-1)
}

// Parse parses a query string to raw ast.StmtNode.
func Parse(ctx sessionctx.Context, src string) ([]ast.StmtNode, error) {
	logutil.BgLogger().Debug("compiling", zap.String("source", src))
	sessVars := ctx.GetSessionVars()
	p := parser.New()
	p.SetParserConfig(sessVars.BuildParserConfig())
	p.SetSQLMode(sessVars.SQLMode)
	stmts, warns, err := p.ParseSQL(src, sessVars.GetParseParams()...)
	for _, warn := range warns {
		sessVars.StmtCtx.AppendWarning(warn)
	}
	if err != nil {
		logutil.BgLogger().Warn("compiling",
			zap.String("source", src),
			zap.Error(err))
		return nil, err
	}
	return stmts, nil
}

func recordAbortTxnDuration(sessVars *variable.SessionVars) {
	duration := time.Since(sessVars.TxnCtx.CreateTime).Seconds()
	if sessVars.TxnCtx.IsPessimistic {
		transactionDurationPessimisticAbort.Observe(duration)
	} else {
		transactionDurationOptimisticAbort.Observe(duration)
	}
}

func finishStmt(ctx context.Context, se *session, meetsErr error, sql sqlexec.Statement) error {
	sessVars := se.sessionVars
	if !sql.IsReadOnly(sessVars) {
		// All the history should be added here.
		if meetsErr == nil && sessVars.TxnCtx.CouldRetry {
			GetHistory(se).Add(sql, sessVars.StmtCtx)
		}

		// Handle the stmt commit/rollback.
		if se.txn.Valid() {
			if meetsErr != nil {
				se.StmtRollback()
			} else {
				se.StmtCommit()
			}
		}
	}
	err := autoCommitAfterStmt(ctx, se, meetsErr, sql)
	if se.txn.pending() {
		// After run statement finish, txn state is still pending means the
		// statement never need a Txn(), such as:
		//
		// set @@tidb_general_log = 1
		// set @@autocommit = 0
		// select 1
		//
		// Reset txn state to invalid to dispose the pending start ts.
		se.txn.changeToInvalid()
	}
	if err != nil {
		return err
	}
	return checkStmtLimit(ctx, se)
}

func autoCommitAfterStmt(ctx context.Context, se *session, meetsErr error, sql sqlexec.Statement) error {
	sessVars := se.sessionVars
	if meetsErr != nil {
		if !sessVars.InTxn() {
			logutil.BgLogger().Info("rollbackTxn called due to ddl/autocommit failure")
			se.RollbackTxn(ctx)
			recordAbortTxnDuration(sessVars)
		} else if se.txn.Valid() && se.txn.IsPessimistic() && executor.ErrDeadlock.Equal(meetsErr) {
			logutil.BgLogger().Info("rollbackTxn for deadlock", zap.Uint64("txn", se.txn.StartTS()))
			se.RollbackTxn(ctx)
			recordAbortTxnDuration(sessVars)
		}
		return meetsErr
	}

	if !sessVars.InTxn() {
		if err := se.CommitTxn(ctx); err != nil {
			if _, ok := sql.(*executor.ExecStmt).StmtNode.(*ast.CommitStmt); ok {
				err = errors.Annotatef(err, "previous statement: %s", se.GetSessionVars().PrevStmt)
			}
			return err
		}
		return nil
	}
	return nil
}

func checkStmtLimit(ctx context.Context, se *session) error {
	// If the user insert, insert, insert ... but never commit, TiDB would OOM.
	// So we limit the statement count in a transaction here.
	var err error
	sessVars := se.GetSessionVars()
	history := GetHistory(se)
	if history.Count() > int(config.GetGlobalConfig().Performance.StmtCountLimit) {
		if !sessVars.BatchCommit {
			se.RollbackTxn(ctx)
			return errors.Errorf("statement count %d exceeds the transaction limitation, autocommit = %t",
				history.Count(), sessVars.IsAutocommit())
		}
		err = se.NewTxn(ctx)
		// The transaction does not committed yet, we need to keep it in transaction.
		// The last history could not be "commit"/"rollback" statement.
		// It means it is impossible to start a new transaction at the end of the transaction.
		// Because after the server executed "commit"/"rollback" statement, the session is out of the transaction.
		sessVars.SetInTxn(true)
	}
	return err
}

// GetHistory get all stmtHistory in current txn. Exported only for test.
func GetHistory(ctx sessionctx.Context) *StmtHistory {
	hist, ok := ctx.GetSessionVars().TxnCtx.History.(*StmtHistory)
	if ok {
		return hist
	}
	hist = new(StmtHistory)
	ctx.GetSessionVars().TxnCtx.History = hist
	return hist
}

// GetRows4Test gets all the rows from a RecordSet, only used for test.
func GetRows4Test(ctx context.Context, sctx sessionctx.Context, rs sqlexec.RecordSet) ([]chunk.Row, error) {
	if rs == nil {
		return nil, nil
	}
	var rows []chunk.Row
	req := rs.NewChunk(nil)
	// Must reuse `req` for imitating server.(*clientConn).writeChunks
	for {
		err := rs.Next(ctx, req)
		if err != nil {
			return nil, err
		}
		if req.NumRows() == 0 {
			break
		}

		iter := chunk.NewIterator4Chunk(req.CopyConstruct())
		for row := iter.Begin(); row != iter.End(); row = iter.Next() {
			rows = append(rows, row)
		}
	}
	return rows, nil
}

// ResultSetToStringSlice changes the RecordSet to [][]string.
func ResultSetToStringSlice(ctx context.Context, s Session, rs sqlexec.RecordSet) ([][]string, error) {
	rows, err := GetRows4Test(ctx, s, rs)
	if err != nil {
		return nil, err
	}
	err = rs.Close()
	if err != nil {
		return nil, err
	}
	sRows := make([][]string, len(rows))
	for i := range rows {
		row := rows[i]
		iRow := make([]string, row.Len())
		for j := 0; j < row.Len(); j++ {
			if row.IsNull(j) {
				iRow[j] = "<nil>"
			} else {
				d := row.GetDatum(j, &rs.Fields()[j].Column.FieldType)
				iRow[j], err = d.ToString()
				if err != nil {
					return nil, err
				}
			}
		}
		sRows[i] = iRow
	}
	return sRows, nil
}

// Session errors.
var (
	ErrForUpdateCantRetry = dbterror.ClassSession.NewStd(errno.ErrForUpdateCantRetry)
)
<commit_msg>session: release lock correctly for tidb to exit (#34126)<commit_after>// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Copyright 2013 The ql Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSES/QL-LICENSE file.

package session

import (
	"context"
	"sync"
	"sync/atomic"
	"time"

	"github.com/pingcap/errors"
	"github.com/pingcap/tidb/config"
	"github.com/pingcap/tidb/domain"
	"github.com/pingcap/tidb/errno"
	"github.com/pingcap/tidb/executor"
	"github.com/pingcap/tidb/kv"
	"github.com/pingcap/tidb/parser"
	"github.com/pingcap/tidb/parser/ast"
	"github.com/pingcap/tidb/sessionctx"
	"github.com/pingcap/tidb/sessionctx/variable"
	"github.com/pingcap/tidb/util"
	"github.com/pingcap/tidb/util/chunk"
	"github.com/pingcap/tidb/util/dbterror"
	"github.com/pingcap/tidb/util/logutil"
	"github.com/pingcap/tidb/util/sqlexec"
	"go.uber.org/zap"
)

type domainMap struct {
	domains map[string]*domain.Domain
	mu      sync.RWMutex
}

func (dm *domainMap) Get(store kv.Storage) (d *domain.Domain, err error) {
	dm.mu.RLock()

	// If this is the only domain instance, and the caller doesn't provide store.
	if len(dm.domains) == 1 && store == nil {
		for _, r := range dm.domains {
			dm.mu.RUnlock()
			return r, nil
		}
	}

	key := store.UUID()
	d = dm.domains[key]
	dm.mu.RUnlock()
	if d != nil {
		return
	}

	ddlLease := time.Duration(atomic.LoadInt64(&schemaLease))
	statisticLease := time.Duration(atomic.LoadInt64(&statsLease))
	idxUsageSyncLease := GetIndexUsageSyncLease()
	planReplayerGCLease := GetPlanReplayerGCLease()
	err = util.RunWithRetry(util.DefaultMaxRetries, util.RetryInterval, func() (retry bool, err1 error) {
		logutil.BgLogger().Info("new domain",
			zap.String("store", store.UUID()),
			zap.Stringer("ddl lease", ddlLease),
			zap.Stringer("stats lease", statisticLease),
			zap.Stringer("index usage sync lease", idxUsageSyncLease))
		factory := createSessionFunc(store)
		sysFactory := createSessionWithDomainFunc(store)
		onClose := func() {
			dm.Delete(store)
		}
		d = domain.NewDomain(store, ddlLease, statisticLease, idxUsageSyncLease, planReplayerGCLease, factory, onClose)
		err1 = d.Init(ddlLease, sysFactory)
		if err1 != nil {
			// If we don't clean it, there are some dirty data when retrying the function of Init.
			d.Close()
			logutil.BgLogger().Error("[ddl] init domain failed",
				zap.Error(err1))
		}
		return true, err1
	})
	if err != nil {
		return nil, err
	}
	dm.Set(store, d)

	return
}

func (dm *domainMap) Delete(store kv.Storage) {
	dm.mu.Lock()
	delete(dm.domains, store.UUID())
	dm.mu.Unlock()
}

func (dm *domainMap) Set(store kv.Storage, domain *domain.Domain) {
	dm.mu.Lock()
	dm.domains[store.UUID()] = domain
	dm.mu.Unlock()
}

var (
	domap = &domainMap{
		domains: map[string]*domain.Domain{},
	}
	// store.UUID()-> IfBootstrapped
	storeBootstrapped     = make(map[string]bool)
	storeBootstrappedLock sync.Mutex

	// schemaLease is the time for re-updating remote schema.
	// In online DDL, we must wait 2 * SchemaLease time to guarantee
	// all servers get the neweset schema.
	// Default schema lease time is 1 second, you can change it with a proper time,
	// but you must know that too little may cause badly performance degradation.
	// For production, you should set a big schema lease, like 300s+.
	schemaLease = int64(1 * time.Second)

	// statsLease is the time for reload stats table.
	statsLease = int64(3 * time.Second)

	// indexUsageSyncLease is the time for index usage synchronization.
	// Because we have not completed GC and other functions, we set it to 0.
	// TODO: Set indexUsageSyncLease to 60s.
	indexUsageSyncLease = int64(0 * time.Second)

	// planReplayerGCLease is the time for plan replayer gc.
	planReplayerGCLease = int64(10 * time.Minute)
)

// ResetStoreForWithTiKVTest is only used in the test code.
// TODO: Remove domap and storeBootstrapped. Use store.SetOption() to do it.
func ResetStoreForWithTiKVTest(store kv.Storage) {
	domap.Delete(store)
	unsetStoreBootstrapped(store.UUID())
}

func setStoreBootstrapped(storeUUID string) {
	storeBootstrappedLock.Lock()
	defer storeBootstrappedLock.Unlock()
	storeBootstrapped[storeUUID] = true
}

// unsetStoreBootstrapped delete store uuid from stored bootstrapped map.
// currently this function only used for test.
func unsetStoreBootstrapped(storeUUID string) {
	storeBootstrappedLock.Lock()
	defer storeBootstrappedLock.Unlock()
	delete(storeBootstrapped, storeUUID)
}

// SetSchemaLease changes the default schema lease time for DDL.
// This function is very dangerous, don't use it if you really know what you do.
// SetSchemaLease only affects not local storage after bootstrapped.
func SetSchemaLease(lease time.Duration) {
	atomic.StoreInt64(&schemaLease, int64(lease))
}

// SetStatsLease changes the default stats lease time for loading stats info.
func SetStatsLease(lease time.Duration) {
	atomic.StoreInt64(&statsLease, int64(lease))
}

// SetIndexUsageSyncLease changes the default index usage sync lease time for loading info.
func SetIndexUsageSyncLease(lease time.Duration) {
	atomic.StoreInt64(&indexUsageSyncLease, int64(lease))
}

// GetIndexUsageSyncLease returns the index usage sync lease time.
func GetIndexUsageSyncLease() time.Duration {
	return time.Duration(atomic.LoadInt64(&indexUsageSyncLease))
}

// SetPlanReplayerGCLease changes the default plan repalyer gc lease time.
func SetPlanReplayerGCLease(lease time.Duration) {
	atomic.StoreInt64(&planReplayerGCLease, int64(lease))
}

// GetPlanReplayerGCLease returns the plan replayer gc lease time.
func GetPlanReplayerGCLease() time.Duration {
	return time.Duration(atomic.LoadInt64(&planReplayerGCLease))
}

// DisableStats4Test disables the stats for tests.
func DisableStats4Test() {
	SetStatsLease(-1)
}

// Parse parses a query string to raw ast.StmtNode.
func Parse(ctx sessionctx.Context, src string) ([]ast.StmtNode, error) {
	logutil.BgLogger().Debug("compiling", zap.String("source", src))
	sessVars := ctx.GetSessionVars()
	p := parser.New()
	p.SetParserConfig(sessVars.BuildParserConfig())
	p.SetSQLMode(sessVars.SQLMode)
	stmts, warns, err := p.ParseSQL(src, sessVars.GetParseParams()...)
	for _, warn := range warns {
		sessVars.StmtCtx.AppendWarning(warn)
	}
	if err != nil {
		logutil.BgLogger().Warn("compiling",
			zap.String("source", src),
			zap.Error(err))
		return nil, err
	}
	return stmts, nil
}

func recordAbortTxnDuration(sessVars *variable.SessionVars) {
	duration := time.Since(sessVars.TxnCtx.CreateTime).Seconds()
	if sessVars.TxnCtx.IsPessimistic {
		transactionDurationPessimisticAbort.Observe(duration)
	} else {
		transactionDurationOptimisticAbort.Observe(duration)
	}
}

func finishStmt(ctx context.Context, se *session, meetsErr error, sql sqlexec.Statement) error {
	sessVars := se.sessionVars
	if !sql.IsReadOnly(sessVars) {
		// All the history should be added here.
		if meetsErr == nil && sessVars.TxnCtx.CouldRetry {
			GetHistory(se).Add(sql, sessVars.StmtCtx)
		}

		// Handle the stmt commit/rollback.
		if se.txn.Valid() {
			if meetsErr != nil {
				se.StmtRollback()
			} else {
				se.StmtCommit()
			}
		}
	}
	err := autoCommitAfterStmt(ctx, se, meetsErr, sql)
	if se.txn.pending() {
		// After run statement finish, txn state is still pending means the
		// statement never need a Txn(), such as:
		//
		// set @@tidb_general_log = 1
		// set @@autocommit = 0
		// select 1
		//
		// Reset txn state to invalid to dispose the pending start ts.
		se.txn.changeToInvalid()
	}
	if err != nil {
		return err
	}
	return checkStmtLimit(ctx, se)
}

func autoCommitAfterStmt(ctx context.Context, se *session, meetsErr error, sql sqlexec.Statement) error {
	sessVars := se.sessionVars
	if meetsErr != nil {
		if !sessVars.InTxn() {
			logutil.BgLogger().Info("rollbackTxn called due to ddl/autocommit failure")
			se.RollbackTxn(ctx)
			recordAbortTxnDuration(sessVars)
		} else if se.txn.Valid() && se.txn.IsPessimistic() && executor.ErrDeadlock.Equal(meetsErr) {
			logutil.BgLogger().Info("rollbackTxn for deadlock", zap.Uint64("txn", se.txn.StartTS()))
			se.RollbackTxn(ctx)
			recordAbortTxnDuration(sessVars)
		}
		return meetsErr
	}

	if !sessVars.InTxn() {
		if err := se.CommitTxn(ctx); err != nil {
			if _, ok := sql.(*executor.ExecStmt).StmtNode.(*ast.CommitStmt); ok {
				err = errors.Annotatef(err, "previous statement: %s", se.GetSessionVars().PrevStmt)
			}
			return err
		}
		return nil
	}
	return nil
}

func checkStmtLimit(ctx context.Context, se *session) error {
	// If the user insert, insert, insert ... but never commit, TiDB would OOM.
	// So we limit the statement count in a transaction here.
	var err error
	sessVars := se.GetSessionVars()
	history := GetHistory(se)
	if history.Count() > int(config.GetGlobalConfig().Performance.StmtCountLimit) {
		if !sessVars.BatchCommit {
			se.RollbackTxn(ctx)
			return errors.Errorf("statement count %d exceeds the transaction limitation, autocommit = %t",
				history.Count(), sessVars.IsAutocommit())
		}
		err = se.NewTxn(ctx)
		// The transaction does not committed yet, we need to keep it in transaction.
		// The last history could not be "commit"/"rollback" statement.
		// It means it is impossible to start a new transaction at the end of the transaction.
		// Because after the server executed "commit"/"rollback" statement, the session is out of the transaction.
		sessVars.SetInTxn(true)
	}
	return err
}

// GetHistory get all stmtHistory in current txn. Exported only for test.
func GetHistory(ctx sessionctx.Context) *StmtHistory {
	hist, ok := ctx.GetSessionVars().TxnCtx.History.(*StmtHistory)
	if ok {
		return hist
	}
	hist = new(StmtHistory)
	ctx.GetSessionVars().TxnCtx.History = hist
	return hist
}

// GetRows4Test gets all the rows from a RecordSet, only used for test.
func GetRows4Test(ctx context.Context, sctx sessionctx.Context, rs sqlexec.RecordSet) ([]chunk.Row, error) {
	if rs == nil {
		return nil, nil
	}
	var rows []chunk.Row
	req := rs.NewChunk(nil)
	// Must reuse `req` for imitating server.(*clientConn).writeChunks
	for {
		err := rs.Next(ctx, req)
		if err != nil {
			return nil, err
		}
		if req.NumRows() == 0 {
			break
		}

		iter := chunk.NewIterator4Chunk(req.CopyConstruct())
		for row := iter.Begin(); row != iter.End(); row = iter.Next() {
			rows = append(rows, row)
		}
	}
	return rows, nil
}

// ResultSetToStringSlice changes the RecordSet to [][]string.
func ResultSetToStringSlice(ctx context.Context, s Session, rs sqlexec.RecordSet) ([][]string, error) {
	rows, err := GetRows4Test(ctx, s, rs)
	if err != nil {
		return nil, err
	}
	err = rs.Close()
	if err != nil {
		return nil, err
	}
	sRows := make([][]string, len(rows))
	for i := range rows {
		row := rows[i]
		iRow := make([]string, row.Len())
		for j := 0; j < row.Len(); j++ {
			if row.IsNull(j) {
				iRow[j] = "<nil>"
			} else {
				d := row.GetDatum(j, &rs.Fields()[j].Column.FieldType)
				iRow[j], err = d.ToString()
				if err != nil {
					return nil, err
				}
			}
		}
		sRows[i] = iRow
	}
	return sRows, nil
}

// Session errors.
var (
	ErrForUpdateCantRetry = dbterror.ClassSession.NewStd(errno.ErrForUpdateCantRetry)
)
<|endoftext|>


<commit_before>// Copyright (c) 2019 Blacknon. All rights reserved.
// Use of this source code is governed by an MIT license
// that can be found in the LICENSE file.

// This file describes the code of the built-in command used by lsftp.
// It is quite big in that relationship. Maybe it will be separated or repaired soon.

package sftp

import (
	"bufio"
	"fmt"
	"os"
	"path/filepath"
	"time"

	"github.com/blacknon/lssh/common"
	"github.com/urfave/cli"
)

// cat is remote cat command
func (r *RunSftp) cat(args []string) {
	// create app
	app := cli.NewApp()
	// app.UseShortOptionHandling = true

	// set help message
	app.CustomAppHelpTemplate = helptext

	app.Name = "cat"
	app.Usage = "lsftp build-in command: cat [remote machine cat]"
	app.ArgsUsage = "[PATH]"
	app.HideHelp = true
	app.HideVersion = true
	app.EnableBashCompletion = true

	app.Action = func(c *cli.Context) error {
		// 1st arg only
		path := c.Args().First()

		for server, client := range r.Client {
			// set ftp client
			ftp := client.Connect

			// Create Output
			client.Output.Create(server)
			w := client.Output.NewWriter()

			// set arg path
			if !filepath.IsAbs(path) {
				path = filepath.Join(client.Pwd, path)
			}

			// open file
			f, err := ftp.Open(path)
			if err != nil {
				fmt.Fprintln(w, err)
				return nil
			}

			// read file to Output.Writer
			_, err = f.WriteTo(w)

			if err != nil {
				fmt.Fprintln(w, err)
			}

		}

		// wait 0.3 sec
		time.Sleep(300 * time.Millisecond)

		return nil
	}

	// parse short options
	args = common.ParseArgs(app.Flags, args)
	app.Run(args)
}

// lcat is local cat command
func (r *RunSftp) lcat(args []string) {
	// create app
	app := cli.NewApp()
	// app.UseShortOptionHandling = true

	// set help message
	app.CustomAppHelpTemplate = helptext

	app.Name = "lcat"
	app.Usage = "lsftp build-in command: lcat [local machine cat]"
	app.ArgsUsage = "[PATH]"
	app.HideHelp = true
	app.HideVersion = true
	app.EnableBashCompletion = true

	app.Action = func(c *cli.Context) error {
		// 1st arg only
		path := c.Args().First()

		// open file
		f, err := os.Open(path)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			return nil
		}

		// printout file
		sc := bufio.NewScanner(f)
		for sc.Scan() {
			fmt.Println(sc.Text())
		}

		return nil
	}

	// parse short options
	args = common.ParseArgs(app.Flags, args)
	app.Run(args)

}
<commit_msg>update<commit_after>// Copyright (c) 2019 Blacknon. All rights reserved.
// Use of this source code is governed by an MIT license
// that can be found in the LICENSE file.

// This file describes the code of the built-in command used by lsftp.
// It is quite big in that relationship. Maybe it will be separated or repaired soon.

package sftp

import (
	"bufio"
	"fmt"
	"os"
	"path/filepath"
	"time"

	"github.com/blacknon/lssh/common"
	"github.com/urfave/cli"
)

// TODO(blacknon): ディレクトリ指定時にエラーにする

// cat is remote cat command
func (r *RunSftp) cat(args []string) {
	// create app
	app := cli.NewApp()
	// app.UseShortOptionHandling = true

	// set help message
	app.CustomAppHelpTemplate = helptext

	app.Name = "cat"
	app.Usage = "lsftp build-in command: cat [remote machine cat]"
	app.ArgsUsage = "[PATH]"
	app.HideHelp = true
	app.HideVersion = true
	app.EnableBashCompletion = true

	app.Action = func(c *cli.Context) error {
		// 1st arg only
		path := c.Args().First()

		for server, client := range r.Client {
			// set ftp client
			ftp := client.Connect

			// Create Output
			client.Output.Create(server)
			w := client.Output.NewWriter()

			// set arg path
			if !filepath.IsAbs(path) {
				path = filepath.Join(client.Pwd, path)
			}

			// open file
			f, err := ftp.Open(path)
			if err != nil {
				fmt.Fprintln(w, err)
				return nil
			}

			// read file to Output.Writer
			_, err = f.WriteTo(w)

			if err != nil {
				fmt.Fprintln(w, err)
			}

		}

		// wait 0.3 sec
		time.Sleep(300 * time.Millisecond)

		return nil
	}

	// parse short options
	args = common.ParseArgs(app.Flags, args)
	app.Run(args)
}

// lcat is local cat command
func (r *RunSftp) lcat(args []string) {
	// create app
	app := cli.NewApp()
	// app.UseShortOptionHandling = true

	// set help message
	app.CustomAppHelpTemplate = helptext

	app.Name = "lcat"
	app.Usage = "lsftp build-in command: lcat [local machine cat]"
	app.ArgsUsage = "[PATH]"
	app.HideHelp = true
	app.HideVersion = true
	app.EnableBashCompletion = true

	app.Action = func(c *cli.Context) error {
		// 1st arg only
		path := c.Args().First()

		// open file
		f, err := os.Open(path)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			return nil
		}

		// printout file
		sc := bufio.NewScanner(f)
		for sc.Scan() {
			fmt.Println(sc.Text())
		}

		return nil
	}

	// parse short options
	args = common.ParseArgs(app.Flags, args)
	app.Run(args)

}
<|endoftext|>


<commit_before>package core

import (
	"crypto/sha1"
	"encoding/base64"
	id3 "github.com/mikkyang/id3-go"
	"github.com/mikkyang/id3-go/v2"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"time"
)

type Source struct {
	channel   Channel
	Root      string
	PublicUrl string
	recursive bool
	fileType  string
	autoImage bool
	image     []byte
}

type MediaFile struct {
	*id3.File
	defaultName string
}

type MediaItem struct {
	Item
}

func trimmed(value string) string {
	cutset := string(rune(0))
	return strings.Trim(strings.TrimRight(value, cutset), " ")
}

func fileUrl(relativePath string, baseUrl string) string {
	Url, _ := url.Parse(baseUrl)
	Url.Path += relativePath
	Url.Path = strings.Replace(Url.Path, "//", "/", -1)
	return Url.String()
}

func formatYear(year string) string {
	if len(year) > 0 {
		t, err := time.Parse("20060102", year)
		if err != nil {
			t, err = time.Parse("20060102", year[0:len(year)-1])
			if err != nil {
				t, err = time.Parse("2006", year)
				if err != nil {
					t, err = time.Parse("20060201", year)
					if err != nil {
						return ""
					}
				}
			}
		}
		return t.Format(time.RFC1123Z)
	}
	return trimmed(year)
}

func (fd *MediaFile) titleArtist() string {
	title := trimmed(fd.Title())
	author := trimmed(fd.Artist())
	var res string
	if len(title) > 0 {
		res = title
	} else {
		res = author
		if len(author) > 0 {
			res += " - "
		}
		res += fd.defaultName
	}
	return res
}

func (fd *MediaFile) readImage() []byte {
	var data []byte
	apic := fd.Frame("APIC")
	if apic != nil {
		switch t := apic.(type) {
		case *v2.ImageFrame:
			data = v2.ImageFrame(*t).Data()
		}
	}
	return data
}

func (fd *MediaFile) copyCategoryTo(m *MediaItem) {
	tcon := fd.Frame("TCON")
	if tcon != nil {
		m.Item.Categories = append(m.Item.Categories, Text{Value: trimmed(tcon.String())})
	}
}

func (fd *MediaFile) yearFormatted() string {
	return formatYear(fd.Year())
}

func (fd *MediaFile) copyMetaTo(m *MediaItem) {
	m.Title = fd.titleArtist()
	m.Item.PubDate = fd.yearFormatted()
	fd.copyCategoryTo(m)
}

func (m *MediaItem) addMeta(path, defaultName string, source *Source) {
	fd, err := id3.Open(path)
	if err != nil {
		m.Item.Title = defaultName
	} else {
		media := &MediaFile{fd, defaultName}
		defer fd.Close()
		media.copyMetaTo(m)
		if source.autoImage && len(source.image) == 0 {
			source.SetImage(media.readImage())
		}
	}
}

func NewSource(root string, recursive bool, publicUrl,
	title, description, language, filetype string, autoImage bool) *Source {
	channel := &Channel{
		PubDate:     time.Now().Format(time.RFC1123Z),
		Title:       title,
		Link:        publicUrl,
		Description: description,
		Language:    language}
	return &Source{
		Root:      root,
		recursive: recursive,
		PublicUrl: publicUrl,
		channel:   *channel,
		fileType:  filetype,
		autoImage: autoImage}
}

func (s *Source) SetImage(image []byte) {
	s.image = image
	if len(image) > 0 {
		hasher := sha1.New()
		hasher.Write(image)
		s.channel.Images = append(s.channel.Images,
			Image{Title: s.channel.Title, Link: s.channel.Link, Url: s.channel.Link + base64.URLEncoding.EncodeToString(hasher.Sum(nil))})
	}
}

func (s *Source) SetChannelImageUrl(url string) {
	s.channel.Images = append(s.channel.Images, Image{Title: s.channel.Title, Link: s.channel.Link, Url: url})
}

func (s *Source) addFile(path string, info os.FileInfo) {
	url := fileUrl(path[len(s.Root)-1:], s.PublicUrl)
	item := MediaItem{Item{Enclosure: Enclosure{Length: info.Size(), Type: "audio/mpeg",
		Url: url}, Guid: url}}
	item.addMeta(path, info.Name(), s)
	s.channel.Items = append(s.channel.Items, item.Item)
}

func (s *Source) HandleWalk() filepath.WalkFunc {
	return func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if !s.recursive && path != s.Root && info.IsDir() {
			return filepath.SkipDir
		}

		if !!info.IsDir() {
			return nil
		}

		matched, _ := filepath.Match("*."+s.fileType, info.Name())
		if matched {
			s.addFile(path, info)
		}
		return nil
	}
}

func (s *Source) Rss() *Rss {
	return &Rss{Channel: s.channel, Version: "2.0", NS: ITunesNs}
}
<commit_msg>simplify the formatYear, remove nested if<commit_after>package core

import (
	"crypto/sha1"
	"encoding/base64"
	id3 "github.com/mikkyang/id3-go"
	"github.com/mikkyang/id3-go/v2"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"time"
)

type Source struct {
	channel   Channel
	Root      string
	PublicUrl string
	recursive bool
	fileType  string
	autoImage bool
	image     []byte
}

type MediaFile struct {
	*id3.File
	defaultName string
}

type MediaItem struct {
	Item
}

func trimmed(value string) string {
	cutset := string(rune(0))
	return strings.Trim(strings.TrimRight(value, cutset), " ")
}

func fileUrl(relativePath string, baseUrl string) string {
	Url, _ := url.Parse(baseUrl)
	Url.Path += relativePath
	Url.Path = strings.Replace(Url.Path, "//", "/", -1)
	return Url.String()
}

func formatYearRFC1123Z(pattern, year string) (string, error) {
	t, err := time.Parse(pattern, year)
	if err != nil {
		return "", err
	}
	return t.Format(time.RFC1123Z), nil
}

func formatYear(year string) string {
	if len(year) > 0 {
		t, err := formatYearRFC1123Z("20060102", year)
		if err == nil {
			return t
		}
		t, err = formatYearRFC1123Z("2006", year)
		if err == nil {
			return t
		}
		t, err = formatYearRFC1123Z("20060201", year)
		if err == nil {
			return t
		}
	}
	return ""
}

func (fd *MediaFile) titleArtist() string {
	title := trimmed(fd.Title())
	author := trimmed(fd.Artist())
	var res string
	if len(title) > 0 {
		res = title
	} else {
		res = author
		if len(author) > 0 {
			res += " - "
		}
		res += fd.defaultName
	}
	return res
}

func (fd *MediaFile) readImage() []byte {
	var data []byte
	apic := fd.Frame("APIC")
	if apic != nil {
		switch t := apic.(type) {
		case *v2.ImageFrame:
			data = v2.ImageFrame(*t).Data()
		}
	}
	return data
}

func (fd *MediaFile) copyCategoryTo(m *MediaItem) {
	tcon := fd.Frame("TCON")
	if tcon != nil {
		m.Item.Categories = append(m.Item.Categories, Text{Value: trimmed(tcon.String())})
	}
}

func (fd *MediaFile) yearFormatted() string {
	return formatYear(fd.Year())
}

func (fd *MediaFile) copyMetaTo(m *MediaItem) {
	m.Title = fd.titleArtist()
	m.Item.PubDate = fd.yearFormatted()
	fd.copyCategoryTo(m)
}

func (m *MediaItem) addMeta(path, defaultName string, source *Source) {
	fd, err := id3.Open(path)
	if err != nil {
		m.Item.Title = defaultName
	} else {
		media := &MediaFile{fd, defaultName}
		defer fd.Close()
		media.copyMetaTo(m)
		if source.autoImage && len(source.image) == 0 {
			source.SetImage(media.readImage())
		}
	}
}

func NewSource(root string, recursive bool, publicUrl,
	title, description, language, filetype string, autoImage bool) *Source {
	channel := &Channel{
		PubDate:     time.Now().Format(time.RFC1123Z),
		Title:       title,
		Link:        publicUrl,
		Description: description,
		Language:    language}
	return &Source{
		Root:      root,
		recursive: recursive,
		PublicUrl: publicUrl,
		channel:   *channel,
		fileType:  filetype,
		autoImage: autoImage}
}

func (s *Source) SetImage(image []byte) {
	s.image = image
	if len(image) > 0 {
		hasher := sha1.New()
		hasher.Write(image)
		s.channel.Images = append(s.channel.Images,
			Image{Title: s.channel.Title, Link: s.channel.Link, Url: s.channel.Link + base64.URLEncoding.EncodeToString(hasher.Sum(nil))})
	}
}

func (s *Source) SetChannelImageUrl(url string) {
	s.channel.Images = append(s.channel.Images, Image{Title: s.channel.Title, Link: s.channel.Link, Url: url})
}

func (s *Source) addFile(path string, info os.FileInfo) {
	url := fileUrl(path[len(s.Root)-1:], s.PublicUrl)
	item := MediaItem{Item{Enclosure: Enclosure{Length: info.Size(), Type: "audio/mpeg",
		Url: url}, Guid: url}}
	item.addMeta(path, info.Name(), s)
	s.channel.Items = append(s.channel.Items, item.Item)
}

func (s *Source) HandleWalk() filepath.WalkFunc {
	return func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if !s.recursive && path != s.Root && info.IsDir() {
			return filepath.SkipDir
		}

		if !!info.IsDir() {
			return nil
		}

		matched, _ := filepath.Match("*."+s.fileType, info.Name())
		if matched {
			s.addFile(path, info)
		}
		return nil
	}
}

func (s *Source) Rss() *Rss {
	return &Rss{Channel: s.channel, Version: "2.0", NS: ITunesNs}
}
<|endoftext|>


<commit_before><commit_msg>Fix do not generate new DeviceID if one is empty<commit_after><|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"path/filepath"

	"github.com/codegangsta/cli"
	"github.com/golang/glog"
	"github.com/hyperhq/runv/driverloader"
	"github.com/hyperhq/runv/hypervisor"
	templatecore "github.com/hyperhq/runv/template"
)

var manageSubCmds = []cli.Command{
	createTemplateCommand,
}

var manageCommand = cli.Command{
	Name:        "manage",
	Usage:       "manage VMs, network, defaults ....",
	ArgsUsage:   "COMMAND [arguments...]",
	Subcommands: manageSubCmds,
	Action: func(context *cli.Context) {
		cli.ShowSubcommandHelp(context)
	},
}

var createTemplateCommand = cli.Command{
	Name: "create-template",
	Flags: []cli.Flag{
		cli.IntFlag{
			Name:  "cpu",
			Value: 1,
			Usage: "the initial number of CPUs of the tempalte VM",
		},
		cli.IntFlag{
			Name:  "mem",
			Value: 128,
			Usage: "the initial size of memory of the template VM",
		},
	},
	Usage: "create a template VM on the directory specified by the global option --template",
	Action: func(context *cli.Context) {
		absOption := func(option string) string {
			path := context.GlobalString(option)
			if path == "" {
				fmt.Printf("The global option --%s should be specified\n", option)
				os.Exit(-1)
			}
			path, eabs := filepath.Abs(path)
			if eabs != nil {
				fmt.Printf("Failed to get the abs path of %s: %v\n", option, eabs)
				os.Exit(-1)
			}
			return path
		}
		kernel := absOption("kernel")
		initrd := absOption("initrd")
		template := absOption("template")

		if err := os.MkdirAll(template, 0700); err != nil {
			fmt.Printf("Failed to create the template directory: %v\n", err)
			os.Exit(-1)
		}

		if context.GlobalBool("debug") {
			flag.CommandLine.Parse([]string{"-v", "3", "--log_dir", context.GlobalString("log_dir"), "--alsologtostderr"})
		} else {
			flag.CommandLine.Parse([]string{"-v", "1", "--log_dir", context.GlobalString("log_dir")})
		}

		var err error
		if hypervisor.HDriver, err = driverloader.Probe(context.GlobalString("driver")); err != nil {
			glog.V(1).Infof("%v\n", err)
			fmt.Printf("Failed to setup the driver: %v\n", err)
			os.Exit(-1)
		}

		if _, err := templatecore.CreateTemplateVM(template, "", context.Int("cpu"), context.Int("mem"), kernel, initrd); err != nil {
			fmt.Printf("Failed to create the template: %v\n", err)
			os.Exit(-1)
		}
	},
}
<commit_msg>add sub command `runv --template /path_to_template manage remove-template`<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"path/filepath"
	"syscall"

	"github.com/codegangsta/cli"
	"github.com/golang/glog"
	"github.com/hyperhq/runv/driverloader"
	"github.com/hyperhq/runv/hypervisor"
	templatecore "github.com/hyperhq/runv/template"
)

var manageSubCmds = []cli.Command{
	createTemplateCommand,
	removeTemplateCommand,
}

var manageCommand = cli.Command{
	Name:        "manage",
	Usage:       "manage VMs, network, defaults ....",
	ArgsUsage:   "COMMAND [arguments...]",
	Subcommands: manageSubCmds,
	Action: func(context *cli.Context) {
		cli.ShowSubcommandHelp(context)
	},
}

var createTemplateCommand = cli.Command{
	Name: "create-template",
	Flags: []cli.Flag{
		cli.IntFlag{
			Name:  "cpu",
			Value: 1,
			Usage: "the initial number of CPUs of the tempalte VM",
		},
		cli.IntFlag{
			Name:  "mem",
			Value: 128,
			Usage: "the initial size of memory of the template VM",
		},
	},
	Usage: "create a template VM on the directory specified by the global option --template",
	Action: func(context *cli.Context) {
		absOption := func(option string) string {
			path := context.GlobalString(option)
			if path == "" {
				fmt.Printf("The global option --%s should be specified\n", option)
				os.Exit(-1)
			}
			path, eabs := filepath.Abs(path)
			if eabs != nil {
				fmt.Printf("Failed to get the abs path of %s: %v\n", option, eabs)
				os.Exit(-1)
			}
			return path
		}
		kernel := absOption("kernel")
		initrd := absOption("initrd")
		template := absOption("template")

		if err := os.MkdirAll(template, 0700); err != nil {
			fmt.Printf("Failed to create the template directory: %v\n", err)
			os.Exit(-1)
		}

		if context.GlobalBool("debug") {
			flag.CommandLine.Parse([]string{"-v", "3", "--log_dir", context.GlobalString("log_dir"), "--alsologtostderr"})
		} else {
			flag.CommandLine.Parse([]string{"-v", "1", "--log_dir", context.GlobalString("log_dir")})
		}

		var err error
		if hypervisor.HDriver, err = driverloader.Probe(context.GlobalString("driver")); err != nil {
			glog.V(1).Infof("%v\n", err)
			fmt.Printf("Failed to setup the driver: %v\n", err)
			os.Exit(-1)
		}

		if _, err := templatecore.CreateTemplateVM(template, "", context.Int("cpu"), context.Int("mem"), kernel, initrd); err != nil {
			fmt.Printf("Failed to create the template: %v\n", err)
			os.Exit(-1)
		}
	},
}

var removeTemplateCommand = cli.Command{
	Name:  "remove-template",
	Usage: "remove the template VM on the directory specified by the global option --template",
	Action: func(context *cli.Context) {
		if err := syscall.Unmount(context.GlobalString("template"), 0); err != nil {
			fmt.Printf("Failed to remove the template: %v\n", err)
			os.Exit(-1)
		}
	},
}
<|endoftext|>


<commit_before>package dropbox

import (
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestSharing_CreateSharedLink(t *testing.T) {
	c := client()
	out, err := c.Sharing.CreateSharedLink(&CreateSharedLinkInput{
		Path:     "/Readme.md",
		ShortURL: true,
	})

	assert.NoError(t, err, "error sharing file")
	assert.Equal(t, "/Readme.md", out.Path)
}
<commit_msg>fix sharing test<commit_after>package dropbox

import (
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestSharing_CreateSharedLink(t *testing.T) {
	c := client()
	out, err := c.Sharing.CreateSharedLink(&CreateSharedLinkInput{
		Path:     "/hello.txt",
		ShortURL: true,
	})

	assert.NoError(t, err, "error sharing file")
	assert.Equal(t, "/hello.txt", out.Path)
}
<|endoftext|>


<commit_before>package sessions

import (
	"context"
	"encoding/json"
	"sync"
	"time"

	"github.com/cozy/cozy-stack/pkg/config"
	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/instance"
	"github.com/cozy/cozy-stack/pkg/logger"
	"github.com/cozy/cozy-stack/pkg/utils"
)

var log = logger.WithNamespace("sessions")

const redisRegistationKey = "registration-logins"

type registrationEntry struct {
	Domain       string
	ClientID     string
	LoginEntryID string
	Expire       time.Time
}

func (r *registrationEntry) Key() string {
	return r.Domain + "|" + r.ClientID
}

var (
	registrationExpirationDuration = 5 * time.Minute

	registrationsMap     map[string]registrationEntry
	registrationsMapLock sync.Mutex
)

// SweepLoginRegistrations starts the login registration process.
//
// This process involving a queue of registration login entries is necessary to
// distinguish "normal" logins from logins to give right to an OAuth
// application.
//
// Since we cannot really distinguish between them other than trusting the
// user, we send a notification to the user by following this process:
//   - if we identify a login for a device registration — by looking at the
//     redirection address — we push an entry onto the queu
//   - if we do not receive the activation of the device by the user in 5
//     minutes, we send a notification for a "normal" login
//   - otherwise we send a notification for the activation of a new device.
func SweepLoginRegistrations() utils.Shutdowner {
	closed := make(chan struct{})
	go func() {
		waitDuration := registrationExpirationDuration / 2
		for {
			select {
			case <-time.After(waitDuration):
				var err error
				waitDuration, err = sweepRegistrations()
				if err != nil {
					log.Errorf("Could not sweep registration queue: %s", err)
				}
				if waitDuration <= 0 {
					waitDuration = registrationExpirationDuration
				}
			case <-closed:
				return
			}
		}
	}()
	return &sweeper{closed}
}

type sweeper struct {
	closed chan struct{}
}

func (s *sweeper) Shutdown(ctx context.Context) error {
	select {
	case s.closed <- struct{}{}:
	case <-ctx.Done():
	}
	return nil
}

// PushLoginRegistration pushes a new login into the registration queue.
func PushLoginRegistration(domain string, login *LoginEntry, clientID string) error {
	entry := registrationEntry{
		Domain:       domain,
		ClientID:     clientID,
		LoginEntryID: login.ID(),
		Expire:       time.Now(),
	}

	if cli := config.GetConfig().SessionStorage.Client(); cli != nil {
		b, err := json.Marshal(entry)
		if err != nil {
			return err
		}
		return cli.HSet(redisRegistationKey, entry.Key(), b).Err()
	}

	registrationsMapLock.Lock()
	registrationsMap[entry.Key()] = entry
	registrationsMapLock.Unlock()
	return nil
}

// RemoveLoginRegistration removes a login from the registration map.
func RemoveLoginRegistration(domain, clientID string) error {
	var entryPtr *registrationEntry
	key := domain + "|" + clientID
	if cli := config.GetConfig().SessionStorage.Client(); cli != nil {
		b, err := cli.HGet(redisRegistationKey, key).Result()
		if err != nil {
			return err
		}
		var entry registrationEntry
		if err = json.Unmarshal([]byte(b), &entry); err != nil {
			return err
		}
		if err = cli.HDel(redisRegistationKey, key).Err(); err != nil {
			return err
		}
		entryPtr = &entry
	} else {
		registrationsMapLock.Lock()
		entry, ok := registrationsMap[key]
		if ok {
			delete(registrationsMap, key)
			entryPtr = &entry
		}
		registrationsMapLock.Unlock()
	}
	if entryPtr != nil {
		sendRegistrationNotification(entryPtr, true)
	}
	return nil
}

func sweepRegistrations() (waitDuration time.Duration, err error) {
	var expiredLogins []registrationEntry

	now := time.Now()
	if cli := config.GetConfig().SessionStorage.Client(); cli != nil {
		var vals map[string]string
		vals, err = cli.HGetAll(redisRegistationKey).Result()
		if err != nil {
			return
		}

		var deletedKeys []string
		for key, data := range vals {
			var entry registrationEntry
			if err = json.Unmarshal([]byte(data), &entry); err != nil {
				deletedKeys = append(deletedKeys, key)
				continue
			}
			diff := entry.Expire.Sub(now)
			if diff < -24*time.Hour {
				// skip too old entries
				deletedKeys = append(deletedKeys, entry.Key())
			} else if diff <= 10*time.Second {
				expiredLogins = append(expiredLogins, entry)
				deletedKeys = append(deletedKeys, entry.Key())
			} else if waitDuration == 0 || waitDuration > diff {
				waitDuration = diff
			}
		}

		if len(deletedKeys) > 0 {
			err = cli.HDel(redisRegistationKey, deletedKeys...).Err()
		}
	} else {
		registrationsMapLock.Lock()

		var deletedKeys []string
		for _, entry := range registrationsMap {
			diff := entry.Expire.Sub(now)
			if diff < -24*time.Hour {
				// skip too old entries
				deletedKeys = append(deletedKeys, entry.Key())
			} else if diff <= 10*time.Second {
				expiredLogins = append(expiredLogins, entry)
				deletedKeys = append(deletedKeys, entry.Key())
			} else if waitDuration == 0 || waitDuration > diff {
				waitDuration = diff
			}
		}

		for _, key := range deletedKeys {
			delete(registrationsMap, key)
		}

		registrationsMapLock.Unlock()
	}

	if len(expiredLogins) > 0 {
		sendExpiredRegistrationNotifications(expiredLogins)
	}

	return
}

func sendRegistrationNotification(entry *registrationEntry, registrationNotification bool) error {
	var login LoginEntry
	i, err := instance.Get(entry.Domain)
	if err != nil {
		return err
	}
	err = couchdb.GetDoc(i, consts.SessionsLogins, entry.LoginEntryID, &login)
	if err != nil {
		return err
	}
	var clientID string
	if registrationNotification {
		clientID = entry.ClientID
	}
	return sendLoginNotification(i, &login, clientID)
}

func sendExpiredRegistrationNotifications(entries []registrationEntry) {
	for _, entry := range entries {
		sendRegistrationNotification(&entry, false)
	}
}
<commit_msg>Missing initialisation of registrations map<commit_after>package sessions

import (
	"context"
	"encoding/json"
	"sync"
	"time"

	"github.com/cozy/cozy-stack/pkg/config"
	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/instance"
	"github.com/cozy/cozy-stack/pkg/logger"
	"github.com/cozy/cozy-stack/pkg/utils"
)

var log = logger.WithNamespace("sessions")

const redisRegistationKey = "registration-logins"

type registrationEntry struct {
	Domain       string
	ClientID     string
	LoginEntryID string
	Expire       time.Time
}

func (r *registrationEntry) Key() string {
	return r.Domain + "|" + r.ClientID
}

var (
	registrationExpirationDuration = 5 * time.Minute

	registrationsMap     map[string]registrationEntry
	registrationsMapLock sync.Mutex
)

// SweepLoginRegistrations starts the login registration process.
//
// This process involving a queue of registration login entries is necessary to
// distinguish "normal" logins from logins to give right to an OAuth
// application.
//
// Since we cannot really distinguish between them other than trusting the
// user, we send a notification to the user by following this process:
//   - if we identify a login for a device registration — by looking at the
//     redirection address — we push an entry onto the queu
//   - if we do not receive the activation of the device by the user in 5
//     minutes, we send a notification for a "normal" login
//   - otherwise we send a notification for the activation of a new device.
func SweepLoginRegistrations() utils.Shutdowner {
	closed := make(chan struct{})
	go func() {
		waitDuration := registrationExpirationDuration / 2
		for {
			select {
			case <-time.After(waitDuration):
				var err error
				waitDuration, err = sweepRegistrations()
				if err != nil {
					log.Errorf("Could not sweep registration queue: %s", err)
				}
				if waitDuration <= 0 {
					waitDuration = registrationExpirationDuration
				}
			case <-closed:
				return
			}
		}
	}()
	return &sweeper{closed}
}

type sweeper struct {
	closed chan struct{}
}

func (s *sweeper) Shutdown(ctx context.Context) error {
	select {
	case s.closed <- struct{}{}:
	case <-ctx.Done():
	}
	return nil
}

// PushLoginRegistration pushes a new login into the registration queue.
func PushLoginRegistration(domain string, login *LoginEntry, clientID string) error {
	entry := registrationEntry{
		Domain:       domain,
		ClientID:     clientID,
		LoginEntryID: login.ID(),
		Expire:       time.Now(),
	}

	if cli := config.GetConfig().SessionStorage.Client(); cli != nil {
		b, err := json.Marshal(entry)
		if err != nil {
			return err
		}
		return cli.HSet(redisRegistationKey, entry.Key(), b).Err()
	}

	registrationsMapLock.Lock()
	if registrationsMap == nil {
		registrationsMap = make(map[string]registrationEntry)
	}
	registrationsMap[entry.Key()] = entry
	registrationsMapLock.Unlock()
	return nil
}

// RemoveLoginRegistration removes a login from the registration map.
func RemoveLoginRegistration(domain, clientID string) error {
	var entryPtr *registrationEntry
	key := domain + "|" + clientID
	if cli := config.GetConfig().SessionStorage.Client(); cli != nil {
		b, err := cli.HGet(redisRegistationKey, key).Result()
		if err != nil {
			return err
		}
		var entry registrationEntry
		if err = json.Unmarshal([]byte(b), &entry); err != nil {
			return err
		}
		if err = cli.HDel(redisRegistationKey, key).Err(); err != nil {
			return err
		}
		entryPtr = &entry
	} else {
		registrationsMapLock.Lock()
		entry, ok := registrationsMap[key]
		if ok {
			delete(registrationsMap, key)
			entryPtr = &entry
		}
		registrationsMapLock.Unlock()
	}
	if entryPtr != nil {
		sendRegistrationNotification(entryPtr, true)
	}
	return nil
}

func sweepRegistrations() (waitDuration time.Duration, err error) {
	var expiredLogins []registrationEntry

	now := time.Now()
	if cli := config.GetConfig().SessionStorage.Client(); cli != nil {
		var vals map[string]string
		vals, err = cli.HGetAll(redisRegistationKey).Result()
		if err != nil {
			return
		}

		var deletedKeys []string
		for key, data := range vals {
			var entry registrationEntry
			if err = json.Unmarshal([]byte(data), &entry); err != nil {
				deletedKeys = append(deletedKeys, key)
				continue
			}
			diff := entry.Expire.Sub(now)
			if diff < -24*time.Hour {
				// skip too old entries
				deletedKeys = append(deletedKeys, entry.Key())
			} else if diff <= 10*time.Second {
				expiredLogins = append(expiredLogins, entry)
				deletedKeys = append(deletedKeys, entry.Key())
			} else if waitDuration == 0 || waitDuration > diff {
				waitDuration = diff
			}
		}

		if len(deletedKeys) > 0 {
			err = cli.HDel(redisRegistationKey, deletedKeys...).Err()
		}
	} else {
		registrationsMapLock.Lock()

		var deletedKeys []string
		for _, entry := range registrationsMap {
			diff := entry.Expire.Sub(now)
			if diff < -24*time.Hour {
				// skip too old entries
				deletedKeys = append(deletedKeys, entry.Key())
			} else if diff <= 10*time.Second {
				expiredLogins = append(expiredLogins, entry)
				deletedKeys = append(deletedKeys, entry.Key())
			} else if waitDuration == 0 || waitDuration > diff {
				waitDuration = diff
			}
		}

		for _, key := range deletedKeys {
			delete(registrationsMap, key)
		}

		registrationsMapLock.Unlock()
	}

	if len(expiredLogins) > 0 {
		sendExpiredRegistrationNotifications(expiredLogins)
	}

	return
}

func sendRegistrationNotification(entry *registrationEntry, registrationNotification bool) error {
	var login LoginEntry
	i, err := instance.Get(entry.Domain)
	if err != nil {
		return err
	}
	err = couchdb.GetDoc(i, consts.SessionsLogins, entry.LoginEntryID, &login)
	if err != nil {
		return err
	}
	var clientID string
	if registrationNotification {
		clientID = entry.ClientID
	}
	return sendLoginNotification(i, &login, clientID)
}

func sendExpiredRegistrationNotifications(entries []registrationEntry) {
	for _, entry := range entries {
		sendRegistrationNotification(&entry, false)
	}
}
<|endoftext|>


<commit_before>// +build !linux

/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mount

type Mounter struct {
	mounterPath string
}

func (mounter *Mounter) Mount(source string, target string, fstype string, options []string) error {
	return nil
}

func (mounter *Mounter) Unmount(target string) error {
	return nil
}

func (mounter *Mounter) List() ([]MountPoint, error) {
	return []MountPoint{}, nil
}

func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) {
	return true, nil
}

func (mounter *Mounter) GetDeviceNameFromMount(mountPath, pluginDir string) (string, error) {
	return "", nil
}

func (mounter *Mounter) DeviceOpened(pathname string) (bool, error) {
	return false, nil
}

func (mounter *Mounter) PathIsDevice(pathname string) (bool, error) {
	return true, nil
}

func (mounter *SafeFormatAndMount) formatAndMount(source string, target string, fstype string, options []string) error {
	return nil
}

func (mounter *SafeFormatAndMount) diskLooksUnformatted(disk string) (bool, error) {
	return true, nil
}
<commit_msg>add IsNotMountPoint() to mount_unsupported.go<commit_after>// +build !linux

/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mount

type Mounter struct {
	mounterPath string
}

func (mounter *Mounter) Mount(source string, target string, fstype string, options []string) error {
	return nil
}

func (mounter *Mounter) Unmount(target string) error {
	return nil
}

func (mounter *Mounter) List() ([]MountPoint, error) {
	return []MountPoint{}, nil
}

func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) {
	return true, nil
}

func (mounter *Mounter) GetDeviceNameFromMount(mountPath, pluginDir string) (string, error) {
	return "", nil
}

func (mounter *Mounter) DeviceOpened(pathname string) (bool, error) {
	return false, nil
}

func (mounter *Mounter) PathIsDevice(pathname string) (bool, error) {
	return true, nil
}

func (mounter *SafeFormatAndMount) formatAndMount(source string, target string, fstype string, options []string) error {
	return nil
}

func (mounter *SafeFormatAndMount) diskLooksUnformatted(disk string) (bool, error) {
	return true, nil
}

func IsNotMountPoint(file string) (bool, error) {
	return true, nil
}
<|endoftext|>


<commit_before>package ehttp

import (
	"fmt"
	"net/http"
)

// Common errors.
var (
	InternalError = NewErrorf(http.StatusInternalServerError, "internal error")
	Unauthtorized = NewErrorf(http.StatusUnauthorized, "unauthorized")
)

// Error is a basic error including the http return code.
type Error struct {
	code  int
	error error
}

// Code is an accessor for the error code.
func (e Error) Code() int {
	return e.code
}

// Error implements the error interface.
func (e Error) Error() string {
	if e.error == nil {
		return "<nil>"
	}
	return e.error.Error()
}

// NewErrorf creates a new http error including a status code.
func NewErrorf(code int, f string, args ...interface{}) error {
	return &Error{
		code:  code,
		error: fmt.Errorf(f, args...),
	}
}

// NewError creates a new http error including a status code.
func NewError(code int, err error) error {
	return &Error{
		code:  code,
		error: err,
	}
}
<commit_msg>Add badrequest common error<commit_after>package ehttp

import (
	"fmt"
	"net/http"
)

// Common errors.
var (
	InternalServerError = NewErrorf(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
	BadRequest          = NewErrorf(http.StatusBadRequest, http.StatusText(http.StatusBadRequest))
	Unauthtorized       = NewErrorf(http.StatusUnauthorized, http.StatusText(http.StatusUnauthorized))
)

// Error is a basic error including the http return code.
type Error struct {
	code  int
	error error
}

// Code is an accessor for the error code.
func (e Error) Code() int {
	return e.code
}

// Error implements the error interface.
func (e Error) Error() string {
	if e.error == nil {
		return "<nil>"
	}
	return e.error.Error()
}

// NewErrorf creates a new http error including a status code.
func NewErrorf(code int, f string, args ...interface{}) error {
	return &Error{
		code:  code,
		error: fmt.Errorf(f, args...),
	}
}

// NewError creates a new http error including a status code.
func NewError(code int, err error) error {
	return &Error{
		code:  code,
		error: err,
	}
}
<|endoftext|>