<commit_before>package main

//import "github.com/stretchr/powerwalk"
import "flag"
import "fmt"
import (
	"bufio"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"time"
	//"runtime"
)

const Usage = `
    Usage of findref:

    findref [options] match_regex [start_dir] [filename_regex]

    Arguments:

        match_regex:  This is an RE2 regular expression that will be matched against lines
                      in each file, with matches being displayed to the user.

        start_dir:  This optional argument sets the starting directory to crawl looking
                    for eligible files with lines matching match_regex.  Default value
                    is the current working directory, AKA $PWD or '.'

        filename_regex:  This optional argument restricts the set of files checked for
                         matching lines.  Eligible files must match this expression.
                         Default value matches all files

    Options:

        -d | --debug
              Enable debug mode
        -h | --hidden
              Include hidden files and files in hidden directories
        -i | --ignore-case
              Ignore case in regex (overrides smart-case)
        -m | --match-case
              Match regex case (if unset smart-case is used)
        -s | --stats
              Track basic statistics and print them on exit
        -v | --version
              Print current version and exit
`

const Version = "0.0.6"
const Date = "2017-09-27"

const Red = "\033[0;31m"
const Blue = "\033[0;34m"
const Cyan = "\033[0;36m"
const Green = "\033[0;32m"
const Purple = "\033[0;35m"
const Restore = "\033[0m"
const LightRed = "\033[1;31m"

/* Not currently used
const Black = "\033[0;30m"
const Brown = "\033[0;33m"
const White = "\033[1;37m"
const Yellow = "\033[1;33m"
const DarkGray = "\033[1;30m"
const LightGray = "\033[0;37m"
const LightBlue = "\033[1;34m"
const LightCyan = "\033[1;36m"
const LightGreen = "\033[1;32m"
const LightPurple = "\033[1;35m"
*/

var FILE_PROCESSING_COMPLETE error = nil

/* Shared flags */
var Debug bool = false
var IncludeHidden bool = false
var TrackStats bool = false

/* Shared regular expressions */
var matchRegex *regexp.Regexp = nil
var filenameRegex *regexp.Regexp = regexp.MustCompile(".*")
var hiddenFileRegex *regexp.Regexp = regexp.MustCompile(`(^|\/)\.`)

/* Statistics */
var filesScanned int = 0
var linesScanned int = 0
var matchesFound int = 0
var startTime time.Time

func elapsedTime() time.Duration {
	return time.Now().Sub(startTime)
}

func usageAndExit() {
	flag.Usage()
	os.Exit(1)
}

func debug(a ...interface{}) {
	if Debug {
		fmt.Println(a...)
	}
}

func printMatch(path string, lineNumber int, line []byte, match []int) {
	//fmt.Println(Purple + path + Restore + Green + ":" + strconv.Itoa(lineNumber) + ":" + Restore + string(line[:match[0]]) + LightRed + string(line[match[0]:match[1]]) + Restore + string(line[match[1]:]))
	fmt.Printf("%s%s%s%s:%s:%s%s%s%s%s%s\n",
		Purple,
		path,
		Restore,
		Green,
		strconv.Itoa(lineNumber),
		Restore,
		string(line[:match[0]]),
		LightRed,
		string(line[match[0]:match[1]]),
		Restore,
		string(line[match[1]:]),
	)
}

func passesFileFilter(path string) bool {
	return filenameRegex.MatchString(path)
}

func isHidden(path string) bool {
	// Ignore hidden files unless the IncludeHidden flag is set
	return path != "." && !IncludeHidden && hiddenFileRegex.MatchString(path)
}

func containsNullByte(line []byte) bool {
	for _, el := range line {
		if el == 0 {
			return true
		}
	}
	return false
}

func incrLineCount() {
	if TrackStats {
		linesScanned++
	}
}

func incrFileCount() {

	if TrackStats {
		filesScanned++
	}
}

func incrMatchCount() {
	if TrackStats {
		matchesFound++
	}
}

func checkForMatches(path string) error {
	debug(Blue+"Checking file for matches:"+Restore, path)

	file, err := os.Open(path)
	if err != nil {
		debug(Red+"Error opening file at '"+path+"'.  It might be a directory.  Err: "+Restore, err)
		return FILE_PROCESSING_COMPLETE
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	var lineNumber int = 0
	for scanner.Scan() {
		line := scanner.Bytes()
		lineNumber++
		incrLineCount()
		if containsNullByte(line) {
			// This is a binary file.  Skip it!
			debug(Blue+"Not processing binary file:"+Restore, path)
			return FILE_PROCESSING_COMPLETE
		}
		if matchIndex := matchRegex.FindIndex(line); matchIndex != nil {
			// we have a match! loc == nil means no match so just ignore that case
			incrMatchCount()
			printMatch(path, lineNumber, line, matchIndex)
		}
	}

	if err := scanner.Err(); err != nil {
		debug(Red+"Error scanning line from file '"+path+"'. File will be skipped.  Err: "+Restore, err)
		return FILE_PROCESSING_COMPLETE
	}
	return FILE_PROCESSING_COMPLETE
}

func processFile(path string, info os.FileInfo, err error) error {
	incrFileCount()

	if err != nil {
		debug("filepath.Walk encountered error with path '"+path+"'", err)
		return FILE_PROCESSING_COMPLETE
	}

	if info.IsDir() {
		if isHidden(path) {
			debug(Blue, "Directory", path, "is hidden and will be pruned", Restore)
			return filepath.SkipDir // skip the whole sub-contents of this hidden directory
		} else {
			return FILE_PROCESSING_COMPLETE
		}
	}

	if passesFileFilter(path) {
		debug(Blue+"Passes file filter:", path)
		if isHidden(path) {
			debug(Blue + "Hidden file '" + Restore + path + Blue + "' not processed")
			return FILE_PROCESSING_COMPLETE
		}
		return checkForMatches(path)
	} else {
		debug(Blue + "Ignoring file cause it doesn't match filter: " + Restore + path)
	}
	return FILE_PROCESSING_COMPLETE
}

func getMatchRegex(ignoreCase bool, matchCase bool, usersRegex string) *regexp.Regexp {
	// If ignore case is set, ignore the case of the regex.
	// if match-case is not set, use smart case which means if it's all lower case be case-insensitive,
	// but if there's capitals then be case-sensitive
	if ignoreCase || (!matchCase && !regexp.MustCompile("[A-Z]").MatchString(usersRegex)) {
		debug(Blue, "Match regex will be case-insensitive", Restore)
		return regexp.MustCompile("(?i)" + usersRegex)
	} else {
		debug(Blue, "Match regex will be exactly as user provided", Restore)
		return regexp.MustCompile(usersRegex)
	}
}

func printVersionAndExit() {
	fmt.Printf("%s%s%s%s%s%s\n", Cyan, "findref version ", Version, " released on ", Date, Restore)
}

func main() {
	sPtr := flag.Bool("s", false, "Alias for --stats")
	dPtr := flag.Bool("d", false, "Alias for --debug")
	hPtr := flag.Bool("h", false, "Alias for --hidden")
	vPtr := flag.Bool("v", false, "Alias for --version")
	mPtr := flag.Bool("m", false, "Alias for --match-case")
	iPtr := flag.Bool("i", false, "Alias for --ignore-case")
	helpPtr := flag.Bool("help", false, "Show usage")
	statsPtr := flag.Bool("stats", false, "Track and display statistics")
	debugPtr := flag.Bool("debug", false, "Enable debug mode")
	hiddenPtr := flag.Bool("hidden", false, "Include hidden files and files in hidden directories")
	versionPtr := flag.Bool("version", false, "Print current version and exit")
	matchCasePtr := flag.Bool("match-case", false, "Match regex case (if unset smart-case is used)")
	ignoreCasePtr := flag.Bool("ignore-case", false, "Ignore case in regex (overrides smart-case)")

	flag.Parse()
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "%s\n", Usage)
	}

	if *vPtr || *versionPtr {
		printVersionAndExit()
		os.Exit(0)
	}

	if *helpPtr {
		usageAndExit()
	}

	*matchCasePtr = *matchCasePtr || *mPtr
	*ignoreCasePtr = *ignoreCasePtr || *iPtr
	TrackStats = *statsPtr || *sPtr
	IncludeHidden = *hiddenPtr || *hPtr
	Debug = *debugPtr || *dPtr

	if TrackStats {
		startTime = time.Now()
		debug(Blue, "Start time is:", Restore, startTime.String())
	}

	debug(Blue, "stats enabled: ", Restore, TrackStats)
	debug(Blue, "match-case enabled: ", Restore, *matchCasePtr)
	debug(Blue, "ignore-case enabled: ", Restore, *ignoreCasePtr)
	debug(Blue, "include hidden files: ", Restore, IncludeHidden)
	debug(Blue, "debug mode: ", Restore, Debug)

	rootDir := "."

	if len(flag.Args()) < 1 {
		fmt.Errorf("%s", "Must specify regex to match against files")
		usageAndExit()
	} else if len(flag.Args()) > 3 {
		fmt.Errorf("%s", "Too many args (expected 1 <= 3)")
		usageAndExit()
	} else {
		matchRegex = getMatchRegex(*ignoreCasePtr, *matchCasePtr, flag.Args()[0])

		if len(flag.Args()) >= 2 {
			rootDir = flag.Args()[1]
		}
		if len(flag.Args()) == 3 {
			filenameRegex = regexp.MustCompile(flag.Args()[2])
		}
	}

	debug(Blue, "matchRegex: ", Restore, matchRegex.String())
	debug(Blue, "rootDir: ", Restore, rootDir)
	debug(Blue, "fileRegex: ", Restore, filenameRegex.String())

	filepath.Walk(rootDir, processFile)

	// TODO: Switch to powerwalk for performance:  https://github.com/stretchr/powerwalk
	//runtime.GOMAXPROCS(runtime.NumCPU())
	//powerwalk.Walk(rootDir, processFile)

	if TrackStats {
		fmt.Printf("%sElapsed time:%s  %s\n", Cyan, Restore, elapsedTime().String())
		fmt.Printf("%sLines scanned:%s %d\n", Cyan, Restore, linesScanned)
		fmt.Printf("%sFiles scanned:%s %d\n", Cyan, Restore, filesScanned)
		fmt.Printf("%sMatches found:%s %d\n", Cyan, Restore, matchesFound)
	}
}
<commit_msg>Add option to turn off color<commit_after>package main

//import "github.com/stretchr/powerwalk"
import "flag"
import "fmt"
import (
	"bufio"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"time"
	//"runtime"
)

const Usage = `
    Usage of findref:

    findref [options] match_regex [start_dir] [filename_regex]

    Arguments:

        match_regex:  This is an RE2 regular expression that will be matched against lines
                      in each file, with matches being displayed to the user.

        start_dir:  This optional argument sets the starting directory to crawl looking
                    for eligible files with lines matching match_regex.  Default value
                    is the current working directory, AKA $PWD or '.'

        filename_regex:  This optional argument restricts the set of files checked for
                         matching lines.  Eligible files must match this expression.
                         Default value matches all files

    Options:

        -n | --no-color
              Disable colorized output
        -d | --debug
              Enable debug mode
        -h | --hidden
              Include hidden files and files in hidden directories
        -i | --ignore-case
              Ignore case in regex (overrides smart-case)
        -m | --match-case
              Match regex case (if unset smart-case is used)
        -s | --stats
              Track basic statistics and print them on exit
        -v | --version
              Print current version and exit
`

const Version = "0.0.6"
const Date = "2017-09-27"

/* Colors */
var Red string = "\033[0;31m"
var Blue string = "\033[0;34m"
var Cyan string = "\033[0;36m"
var Green string = "\033[0;32m"
var Black string = "\033[0;30m"
var Brown string = "\033[0;33m"
var White string = "\033[1;37m"
var Yellow string = "\033[1;33m"
var Purple string = "\033[0;35m"
var Restore string = "\033[0m"
var LightRed string = "\033[1;31m"
var DarkGray string = "\033[1;30m"
var LightGray string = "\033[0;37m"
var LightBlue string = "\033[1;34m"
var LightCyan string = "\033[1;36m"
var LightGreen string = "\033[1;32m"
var LightPurple string = "\033[1;35m"

var FILE_PROCESSING_COMPLETE error = nil

/* Shared flags */
var Debug bool = false
var IncludeHidden bool = false
var TrackStats bool = false

/* Shared regular expressions */
var matchRegex *regexp.Regexp = nil
var filenameRegex *regexp.Regexp = regexp.MustCompile(".*")
var hiddenFileRegex *regexp.Regexp = regexp.MustCompile(`(^|\/)\.`)

/* Statistics */
var filesScanned int = 0
var linesScanned int = 0
var matchesFound int = 0
var startTime time.Time

func zeroColors() {
	Red = ""
	Blue = ""
	Cyan = ""
	Green = ""
	Black = ""
	Brown = ""
	White = ""
	Yellow = ""
	Purple = ""
	Restore = ""
	LightRed = ""
	DarkGray = ""
	LightGray = ""
	LightBlue = ""
	LightCyan = ""
	LightGreen = ""
	LightPurple = ""
}

func elapsedTime() time.Duration {
	return time.Now().Sub(startTime)
}

func usageAndExit() {
	flag.Usage()
	os.Exit(1)
}

func debug(a ...interface{}) {
	if Debug {
		fmt.Println(a...)
	}
}

func printMatch(path string, lineNumber int, line []byte, match []int) {
	//fmt.Println(Purple + path + Restore + Green + ":" + strconv.Itoa(lineNumber) + ":" + Restore + string(line[:match[0]]) + LightRed + string(line[match[0]:match[1]]) + Restore + string(line[match[1]:]))
	fmt.Printf("%s%s%s%s:%s:%s%s%s%s%s%s\n",
		Purple,
		path,
		Restore,
		Green,
		strconv.Itoa(lineNumber),
		Restore,
		string(line[:match[0]]),
		LightRed,
		string(line[match[0]:match[1]]),
		Restore,
		string(line[match[1]:]),
	)
}

func passesFileFilter(path string) bool {
	return filenameRegex.MatchString(path)
}

func isHidden(path string) bool {
	// Ignore hidden files unless the IncludeHidden flag is set
	return path != "." && !IncludeHidden && hiddenFileRegex.MatchString(path)
}

func containsNullByte(line []byte) bool {
	for _, el := range line {
		if el == 0 {
			return true
		}
	}
	return false
}

func incrLineCount() {
	if TrackStats {
		linesScanned++
	}
}

func incrFileCount() {

	if TrackStats {
		filesScanned++
	}
}

func incrMatchCount() {
	if TrackStats {
		matchesFound++
	}
}

func checkForMatches(path string) error {
	debug(Blue+"Checking file for matches:"+Restore, path)

	file, err := os.Open(path)
	if err != nil {
		debug(Red+"Error opening file at '"+path+"'.  It might be a directory.  Err: "+Restore, err)
		return FILE_PROCESSING_COMPLETE
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	var lineNumber int = 0
	for scanner.Scan() {
		line := scanner.Bytes()
		lineNumber++
		incrLineCount()
		if containsNullByte(line) {
			// This is a binary file.  Skip it!
			debug(Blue+"Not processing binary file:"+Restore, path)
			return FILE_PROCESSING_COMPLETE
		}
		if matchIndex := matchRegex.FindIndex(line); matchIndex != nil {
			// we have a match! loc == nil means no match so just ignore that case
			incrMatchCount()
			printMatch(path, lineNumber, line, matchIndex)
		}
	}

	if err := scanner.Err(); err != nil {
		debug(Red+"Error scanning line from file '"+path+"'. File will be skipped.  Err: "+Restore, err)
		return FILE_PROCESSING_COMPLETE
	}
	return FILE_PROCESSING_COMPLETE
}

func processFile(path string, info os.FileInfo, err error) error {
	incrFileCount()

	if err != nil {
		debug("filepath.Walk encountered error with path '"+path+"'", err)
		return FILE_PROCESSING_COMPLETE
	}

	if info.IsDir() {
		if isHidden(path) {
			debug(Blue, "Directory", path, "is hidden and will be pruned", Restore)
			return filepath.SkipDir // skip the whole sub-contents of this hidden directory
		} else {
			return FILE_PROCESSING_COMPLETE
		}
	}

	if passesFileFilter(path) {
		debug(Blue+"Passes file filter:", path)
		if isHidden(path) {
			debug(Blue + "Hidden file '" + Restore + path + Blue + "' not processed")
			return FILE_PROCESSING_COMPLETE
		}
		return checkForMatches(path)
	} else {
		debug(Blue + "Ignoring file cause it doesn't match filter: " + Restore + path)
	}
	return FILE_PROCESSING_COMPLETE
}

func getMatchRegex(ignoreCase bool, matchCase bool, usersRegex string) *regexp.Regexp {
	// If ignore case is set, ignore the case of the regex.
	// if match-case is not set, use smart case which means if it's all lower case be case-insensitive,
	// but if there's capitals then be case-sensitive
	if ignoreCase || (!matchCase && !regexp.MustCompile("[A-Z]").MatchString(usersRegex)) {
		debug(Blue, "Match regex will be case-insensitive", Restore)
		return regexp.MustCompile("(?i)" + usersRegex)
	} else {
		debug(Blue, "Match regex will be exactly as user provided", Restore)
		return regexp.MustCompile(usersRegex)
	}
}

func printVersionAndExit() {
	fmt.Printf("%s%s%s%s%s%s\n", Cyan, "findref version ", Version, " released on ", Date, Restore)
}

func main() {
	sPtr := flag.Bool("s", false, "Alias for --stats")
	dPtr := flag.Bool("d", false, "Alias for --debug")
	hPtr := flag.Bool("h", false, "Alias for --hidden")
	vPtr := flag.Bool("v", false, "Alias for --version")
	nPtr := flag.Bool("n", false, "Alias for --no-color")
	mPtr := flag.Bool("m", false, "Alias for --match-case")
	iPtr := flag.Bool("i", false, "Alias for --ignore-case")
	helpPtr := flag.Bool("help", false, "Show usage")
	statsPtr := flag.Bool("stats", false, "Track and display statistics")
	debugPtr := flag.Bool("debug", false, "Enable debug mode")
	hiddenPtr := flag.Bool("hidden", false, "Include hidden files and files in hidden directories")
	versionPtr := flag.Bool("version", false, "Print current version and exit")
	nocolorPtr := flag.Bool("no-color", false, "Don't use color in output")
	matchCasePtr := flag.Bool("match-case", false, "Match regex case (if unset smart-case is used)")
	ignoreCasePtr := flag.Bool("ignore-case", false, "Ignore case in regex (overrides smart-case)")

	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "%s\n", Usage)
	}
	flag.Parse()

	if *vPtr || *versionPtr {
		printVersionAndExit()
		os.Exit(0)
	}

	if *helpPtr {
		usageAndExit()
	}

	if *nPtr || *nocolorPtr {
		debug("Color output is disabled")
		zeroColors()
	}

	*matchCasePtr = *matchCasePtr || *mPtr
	*ignoreCasePtr = *ignoreCasePtr || *iPtr
	TrackStats = *statsPtr || *sPtr
	IncludeHidden = *hiddenPtr || *hPtr
	Debug = *debugPtr || *dPtr

	if TrackStats {
		startTime = time.Now()
		debug(Blue, "Start time is:", Restore, startTime.String())
	}

	debug(Blue, "stats enabled: ", Restore, TrackStats)
	debug(Blue, "match-case enabled: ", Restore, *matchCasePtr)
	debug(Blue, "ignore-case enabled: ", Restore, *ignoreCasePtr)
	debug(Blue, "include hidden files: ", Restore, IncludeHidden)
	debug(Blue, "debug mode: ", Restore, Debug)

	rootDir := "."

	if len(flag.Args()) < 1 {
		fmt.Errorf("%s", "Must specify regex to match against files")
		usageAndExit()
	} else if len(flag.Args()) > 3 {
		fmt.Errorf("%s", "Too many args (expected 1 <= 3)")
		usageAndExit()
	} else {
		matchRegex = getMatchRegex(*ignoreCasePtr, *matchCasePtr, flag.Args()[0])

		if len(flag.Args()) >= 2 {
			rootDir = flag.Args()[1]
		}
		if len(flag.Args()) == 3 {
			filenameRegex = regexp.MustCompile(flag.Args()[2])
		}
	}

	debug(Blue, "matchRegex: ", Restore, matchRegex.String())
	debug(Blue, "rootDir: ", Restore, rootDir)
	debug(Blue, "fileRegex: ", Restore, filenameRegex.String())

	filepath.Walk(rootDir, processFile)

	// TODO: Switch to powerwalk for performance:  https://github.com/stretchr/powerwalk
	//runtime.GOMAXPROCS(runtime.NumCPU())
	//powerwalk.Walk(rootDir, processFile)

	if TrackStats {
		fmt.Printf("%sElapsed time:%s  %s\n", Cyan, Restore, elapsedTime().String())
		fmt.Printf("%sLines scanned:%s %d\n", Cyan, Restore, linesScanned)
		fmt.Printf("%sFiles scanned:%s %d\n", Cyan, Restore, filesScanned)
		fmt.Printf("%sMatches found:%s %d\n", Cyan, Restore, matchesFound)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package common

import (
	"github.com/knative/eventing/test/base/resources"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

// DefaultClusterChannelProvisioner is the default ClusterChannelProvisioner we will run tests against.
const DefaultClusterChannelProvisioner = resources.InMemoryProvisioner

// ValidProvisionersMap saves the provisioner-features mapping.
// Each pair means the provisioner support the list of features.
var ValidProvisionersMap = map[string]ChannelConfig{
	resources.InMemoryProvisioner: {
		Features:     []Feature{FeatureBasic},
		CRDSupported: true,
	},
	resources.GCPPubSubProvisioner: {
		Features: []Feature{FeatureBasic, FeatureRedelivery, FeaturePersistence},
	},
	resources.KafkaProvisioner: {
		Features:     []Feature{FeatureBasic, FeatureRedelivery, FeaturePersistence},
		CRDSupported: true,
	},
	resources.NatssProvisioner: ChannelConfig{
		Features:     []Feature{FeatureBasic, FeatureRedelivery, FeaturePersistence},
		CRDSupported: true,
	},
}

// ChannelConfig includes general configuration for a Channel provisioner.
type ChannelConfig struct {
	Features     []Feature
	CRDSupported bool
}

// ProvisionerChannelMap saves the mapping between provisioners and CRD channel typemeta.
// TODO(Fredy-Z): this map will not be needed anymore when we delete the provisioner implementation.
var ProvisionerChannelMap = map[string]*metav1.TypeMeta{
	resources.KafkaProvisioner:    KafkaChannelTypeMeta,
	resources.InMemoryProvisioner: InMemoryChannelTypeMeta,
	resources.NatssProvisioner:    NatssChannelTypeMeta,
}

// Feature is the feature supported by the Channel provisioner.
type Feature string

const (
	// FeatureBasic is the feature that should be supported by all Channel provisioners
	FeatureBasic Feature = "basic"
	// FeatureRedelivery means if downstream rejects an event, that request will be attempted again.
	FeatureRedelivery Feature = "redelivery"
	// FeaturePersistence means if Channel's Pod goes down, all events already ACKed by the Channel
	// will persist and be retransmitted when the Pod restarts.
	FeaturePersistence Feature = "persistence"
)
<commit_msg>golang format tools (#1391)<commit_after>/*
Copyright 2019 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package common

import (
	"github.com/knative/eventing/test/base/resources"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

// DefaultClusterChannelProvisioner is the default ClusterChannelProvisioner we will run tests against.
const DefaultClusterChannelProvisioner = resources.InMemoryProvisioner

// ValidProvisionersMap saves the provisioner-features mapping.
// Each pair means the provisioner support the list of features.
var ValidProvisionersMap = map[string]ChannelConfig{
	resources.InMemoryProvisioner: {
		Features:     []Feature{FeatureBasic},
		CRDSupported: true,
	},
	resources.GCPPubSubProvisioner: {
		Features: []Feature{FeatureBasic, FeatureRedelivery, FeaturePersistence},
	},
	resources.KafkaProvisioner: {
		Features:     []Feature{FeatureBasic, FeatureRedelivery, FeaturePersistence},
		CRDSupported: true,
	},
	resources.NatssProvisioner: {
		Features:     []Feature{FeatureBasic, FeatureRedelivery, FeaturePersistence},
		CRDSupported: true,
	},
}

// ChannelConfig includes general configuration for a Channel provisioner.
type ChannelConfig struct {
	Features     []Feature
	CRDSupported bool
}

// ProvisionerChannelMap saves the mapping between provisioners and CRD channel typemeta.
// TODO(Fredy-Z): this map will not be needed anymore when we delete the provisioner implementation.
var ProvisionerChannelMap = map[string]*metav1.TypeMeta{
	resources.KafkaProvisioner:    KafkaChannelTypeMeta,
	resources.InMemoryProvisioner: InMemoryChannelTypeMeta,
	resources.NatssProvisioner:    NatssChannelTypeMeta,
}

// Feature is the feature supported by the Channel provisioner.
type Feature string

const (
	// FeatureBasic is the feature that should be supported by all Channel provisioners
	FeatureBasic Feature = "basic"
	// FeatureRedelivery means if downstream rejects an event, that request will be attempted again.
	FeatureRedelivery Feature = "redelivery"
	// FeaturePersistence means if Channel's Pod goes down, all events already ACKed by the Channel
	// will persist and be retransmitted when the Pod restarts.
	FeaturePersistence Feature = "persistence"
)
<|endoftext|>


<commit_before>//   Copyright 2017 MSolution.IO
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.

// periodic runs periodic tasks. Tasks can be registered to a Scheduler which
// may or may not be "ticking", i.e. running its tasks.
package periodic

import (
	"context"
	"sync"
	"time"

	"github.com/trackit/jsonlog"
)

// contextKey is an unexported type to avoid collisions in context.Context
// values.
type contextKey uint

// taskSignal is the type for signals sent to task tickers.
type taskSignal uint

const (
	Hourly      = 1 * time.Hour
	Daily       = 24 * time.Hour
	TwiceDaily  = Daily / 2
	ThriceDaily = Daily / 3

	// TaskTime is a key for a context.Context where the starting time of
	// the current request is stored.
	TaskTime = contextKey(iota)

	// taskStop instructs a task ticker to stop running its task and return.
	taskStop = taskSignal(iota)
)

// Task is a task that can be scheduled.
type Task func(context.Context) error

// taskRegistration is a task registration that may or may not be ticking.
type taskRegistration struct {
	Name    string `json:"name"`
	task    Task
	Period  time.Duration
	ticker  *time.Ticker
	control chan taskSignal
}

// Scheduler runs registered periodic tasks. Its zero value is a valid
// Scheduler that doesn't tick and has no registered task. It may be used in
// parallel.
type Scheduler struct {
	running       bool
	registrations []taskRegistration
	mutex         sync.RWMutex
}

// Ticking returns whether the Scheduler is currently ticking.
func (s *Scheduler) Ticking() bool {
	s.mutex.RLock()
	defer s.mutex.RUnlock()
	return s.running
}

// Register registers a Task to the Scheduler to be run at period p. If the
// Scheduler is ticking, the task starts ticking immediately.
func (s *Scheduler) Register(t Task, p time.Duration, n string) {
	r := taskRegistration{
		task:   t,
		Period: p,
		Name:   n,
	}
	s.mutex.Lock()
	defer s.mutex.Unlock()
	if s.running {
		r.start()
	}
	s.registrations = append(s.registrations, r)
}

// Start starts a Scheduler. Starting an already started scheduler is
// functionally a noop, though an error will be logged.
func (s *Scheduler) Start() {
	s.mutex.Lock()
	defer s.mutex.Unlock()
	if s.running {
		jsonlog.Error("Attempt to start already started scheduler. Ignoring.", nil)
	} else {
		for i := range s.registrations {
			s.registrations[i].start()
		}
		s.running = true
	}
}

// Stop stops a Scheduler. Stopping an already stopped scheduler is
// functionally a noop, though an error will be logged.
func (s *Scheduler) Stop() {
	s.mutex.Lock()
	defer s.mutex.Unlock()
	if s.running {
		for i := range s.registrations {
			s.registrations[i].stop()
		}
		s.running = false
	}
}

// start starts a taskRegistration, having it tick and run its task
// periodically.
func (t *taskRegistration) start() {
	if t.control == nil {
		t.ticker = time.NewTicker(t.Period)
		t.control = make(chan taskSignal)
		go t.tick()
	} else {
		jsonlog.Error("Attempt to start already started task. Ignoring.", t)
	}
}

// run runs the taskRegistration's task in the current goroutine.
func (t *taskRegistration) run(d time.Time) error {
	ctx := context.Background()
	ctx = context.WithValue(ctx, TaskTime, d)
	return t.task(ctx)
}

// getTickerChan gets the taskRegistration's ticker channel, creating it if it
// doesn’t exist yet. This is the channel where a time.Ticker outputs to run
// the periodic tasks.
func (t *taskRegistration) getTickerChan() <-chan time.Time {
	c := t.ticker
	if c == nil {
		return nil
	} else {
		return c.C
	}
}

// tick starts periodic tasks in response to ticks from t.ticker. The tasks are
// started in their own goroutine using t.run.
func (t *taskRegistration) tick() {
tickloop:
	for {
		select {
		case d := <-t.getTickerChan():
			go t.run(d)
		case s := <-t.control:
			switch s {
			case taskStop:
				close(t.control)
				t.control = nil
				t.ticker.Stop()
				t.ticker = nil
				break tickloop
			}
		}
	}
}

// stop stops a taskRegistration from ticking. Currently running tasks are not
// cancelled.
func (t *taskRegistration) stop() {
	if t.control != nil {
		t.control <- taskStop
	} else {
		jsonlog.Error("Attempt to stop an already stopped task. Ignoring.", t)
	}
}
<commit_msg>periodic: Improve readability of the `tick` function.<commit_after>//   Copyright 2017 MSolution.IO
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.

// periodic runs periodic tasks. Tasks can be registered to a Scheduler which
// may or may not be "ticking", i.e. running its tasks.
package periodic

import (
	"context"
	"sync"
	"time"

	"github.com/trackit/jsonlog"
)

// contextKey is an unexported type to avoid collisions in context.Context
// values.
type contextKey uint

// taskSignal is the type for signals sent to task tickers.
type taskSignal uint

const (
	Hourly      = 1 * time.Hour
	Daily       = 24 * time.Hour
	TwiceDaily  = Daily / 2
	ThriceDaily = Daily / 3

	// TaskTime is a key for a context.Context where the starting time of
	// the current request is stored.
	TaskTime = contextKey(iota)

	// taskStop instructs a task ticker to stop running its task and return.
	taskStop = taskSignal(iota)
)

// Task is a task that can be scheduled.
type Task func(context.Context) error

// taskRegistration is a task registration that may or may not be ticking.
type taskRegistration struct {
	Name    string `json:"name"`
	task    Task
	Period  time.Duration
	ticker  *time.Ticker
	control chan taskSignal
}

// Scheduler runs registered periodic tasks. Its zero value is a valid
// Scheduler that doesn't tick and has no registered task. It may be used in
// parallel.
type Scheduler struct {
	running       bool
	registrations []taskRegistration
	mutex         sync.RWMutex
}

// Ticking returns whether the Scheduler is currently ticking.
func (s *Scheduler) Ticking() bool {
	s.mutex.RLock()
	defer s.mutex.RUnlock()
	return s.running
}

// Register registers a Task to the Scheduler to be run at period p. If the
// Scheduler is ticking, the task starts ticking immediately.
func (s *Scheduler) Register(t Task, p time.Duration, n string) {
	r := taskRegistration{
		task:   t,
		Period: p,
		Name:   n,
	}
	s.mutex.Lock()
	defer s.mutex.Unlock()
	if s.running {
		r.start()
	}
	s.registrations = append(s.registrations, r)
}

// Start starts a Scheduler. Starting an already started scheduler is
// functionally a noop, though an error will be logged.
func (s *Scheduler) Start() {
	s.mutex.Lock()
	defer s.mutex.Unlock()
	if s.running {
		jsonlog.Error("Attempt to start already started scheduler. Ignoring.", nil)
	} else {
		for i := range s.registrations {
			s.registrations[i].start()
		}
		s.running = true
	}
}

// Stop stops a Scheduler. Stopping an already stopped scheduler is
// functionally a noop, though an error will be logged.
func (s *Scheduler) Stop() {
	s.mutex.Lock()
	defer s.mutex.Unlock()
	if s.running {
		for i := range s.registrations {
			s.registrations[i].stop()
		}
		s.running = false
	}
}

// start starts a taskRegistration, having it tick and run its task
// periodically.
func (t *taskRegistration) start() {
	if t.control == nil {
		t.ticker = time.NewTicker(t.Period)
		t.control = make(chan taskSignal)
		go t.tick()
	} else {
		jsonlog.Error("Attempt to start already started task. Ignoring.", t)
	}
}

// run runs the taskRegistration's task in the current goroutine.
func (t *taskRegistration) run(d time.Time) error {
	ctx := context.Background()
	ctx = context.WithValue(ctx, TaskTime, d)
	return t.task(ctx)
}

// getTickerChan gets the taskRegistration's ticker channel, creating it if it
// doesn’t exist yet. This is the channel where a time.Ticker outputs to run
// the periodic tasks.
func (t *taskRegistration) getTickerChan() <-chan time.Time {
	c := t.ticker
	if c == nil {
		return nil
	} else {
		return c.C
	}
}

// tick starts periodic tasks in response to ticks from t.ticker. The tasks are
// started in their own goroutine using t.run.
func (t *taskRegistration) tick() {
	for {
		select {
		case d := <-t.getTickerChan():
			go t.run(d)
		case s := <-t.control:
			switch s {
			case taskStop:
				close(t.control)
				t.control = nil
				t.ticker.Stop()
				t.ticker = nil
				return
			}
		}
	}
}

// stop stops a taskRegistration from ticking. Currently running tasks are not
// cancelled.
func (t *taskRegistration) stop() {
	if t.control != nil {
		t.control <- taskStop
	} else {
		jsonlog.Error("Attempt to stop an already stopped task. Ignoring.", t)
	}
}
<|endoftext|>


<commit_before>//go:build linux
// +build linux

package scsi

import (
	"context"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/pkg/errors"
	"go.opencensus.io/trace"
	"golang.org/x/sys/unix"

	"github.com/Microsoft/hcsshim/internal/guest/storage"
	"github.com/Microsoft/hcsshim/internal/guest/storage/crypt"
	dm "github.com/Microsoft/hcsshim/internal/guest/storage/devicemapper"
	"github.com/Microsoft/hcsshim/internal/log"
	"github.com/Microsoft/hcsshim/internal/oc"
	"github.com/Microsoft/hcsshim/internal/protocol/guestrequest"
	"github.com/Microsoft/hcsshim/internal/protocol/guestresource"
	"github.com/Microsoft/hcsshim/pkg/securitypolicy"
)

// Test dependencies
var (
	osMkdirAll  = os.MkdirAll
	osRemoveAll = os.RemoveAll
	unixMount   = unix.Mount

	// controllerLunToName is stubbed to make testing `Mount` easier.
	controllerLunToName = ControllerLunToName
	// createVerityTarget is stubbed for unit testing `Mount`
	createVerityTarget = dm.CreateVerityTarget
	// removeDevice is stubbed for unit testing `Mount`
	removeDevice = dm.RemoveDevice
)

const (
	scsiDevicesPath  = "/sys/bus/scsi/devices"
	vmbusDevicesPath = "/sys/bus/vmbus/devices"
	verityDeviceFmt  = "verity-scsi-contr%d-lun%d-%s"
)

// fetchActualControllerNumber retrieves the actual controller number assigned to a SCSI controller
// with number `passedController`.
// When HCS creates the UVM it adds 4 SCSI controllers to the UVM but the 1st SCSI
// controller according to HCS can actually show up as 2nd, 3rd or 4th controller inside
// the UVM. So the i'th controller from HCS' perspective could actually be j'th controller
// inside the UVM. However, we can refer to the SCSI controllers with their GUIDs (that
// are hardcoded) and then using that GUID find out the SCSI controller number inside the
// guest. This function does exactly that.
func fetchActualControllerNumber(ctx context.Context, passedController uint8) (uint8, error) {
	// find the controller number by looking for a file named host<N> (e.g host1, host3 etc.)
	// `N` is the controller number.
	// Full file path would be /sys/bus/vmbus/devices/<controller-guid>/host<N>.
	controllerDirPath := path.Join(vmbusDevicesPath, guestrequest.ScsiControllerGuids[passedController])
	entries, err := ioutil.ReadDir(controllerDirPath)
	if err != nil {
		return 0, err
	}

	for _, entry := range entries {
		baseName := path.Base(entry.Name())
		if !strings.HasPrefix(baseName, "host") {
			continue
		}
		controllerStr := baseName[len("host"):]
		controllerNum, err := strconv.ParseUint(controllerStr, 10, 8)
		if err != nil {
			return 0, fmt.Errorf("failed to parse controller number from %s: %w", baseName, err)
		}
		return uint8(controllerNum), nil
	}
	return 0, fmt.Errorf("host<N> directory not found inside %s", controllerDirPath)
}

// mount creates a mount from the SCSI device on `controller` index `lun` to
// `target`
//
// `target` will be created. On mount failure the created `target` will be
// automatically cleaned up.
//
// If `encrypted` is set to true, the SCSI device will be encrypted using
// dm-crypt.
func mount(
	ctx context.Context,
	controller,
	lun uint8,
	target string,
	readonly bool,
	encrypted bool,
	options []string,
	verityInfo *guestresource.DeviceVerityInfo,
	securityPolicy securitypolicy.SecurityPolicyEnforcer,
) (err error) {
	spnCtx, span := trace.StartSpan(ctx, "scsi::Mount")
	defer span.End()
	defer func() { oc.SetSpanStatus(span, err) }()

	span.AddAttributes(
		trace.Int64Attribute("controller", int64(controller)),
		trace.Int64Attribute("lun", int64(lun)))

	source, err := controllerLunToName(spnCtx, controller, lun)
	if err != nil {
		return err
	}

	if readonly {
		// containers only have read-only layers so only enforce for them
		var deviceHash string
		if verityInfo != nil {
			deviceHash = verityInfo.RootDigest
		}

		err = securityPolicy.EnforceDeviceMountPolicy(target, deviceHash)
		if err != nil {
			return errors.Wrapf(err, "won't mount scsi controller %d lun %d onto %s", controller, lun, target)
		}

		if verityInfo != nil {
			dmVerityName := fmt.Sprintf(verityDeviceFmt, controller, lun, deviceHash)
			if source, err = createVerityTarget(spnCtx, source, dmVerityName, verityInfo); err != nil {
				return err
			}
			defer func() {
				if err != nil {
					if err := removeDevice(dmVerityName); err != nil {
						log.G(spnCtx).WithError(err).WithField("verityTarget", dmVerityName).Debug("failed to cleanup verity target")
					}
				}
			}()
		}
	}

	if err := osMkdirAll(target, 0700); err != nil {
		return err
	}
	defer func() {
		if err != nil {
			osRemoveAll(target)
		}
	}()

	// we only care about readonly mount option when mounting the device
	var flags uintptr
	data := ""
	if readonly {
		flags |= unix.MS_RDONLY
		data = "noload"
	}

	if encrypted {
		encryptedSource, err := crypt.EncryptDevice(spnCtx, source)
		if err != nil {
			return errors.Wrapf(err, "failed to mount encrypted device: "+source)
		}
		source = encryptedSource
	}

	for {
		if err := unixMount(source, target, "ext4", flags, data); err != nil {
			// The `source` found by controllerLunToName can take some time
			// before its actually available under `/dev/sd*`. Retry while we
			// wait for `source` to show up.
			if err == unix.ENOENT {
				select {
				case <-ctx.Done():
					return ctx.Err()
				default:
					time.Sleep(10 * time.Millisecond)
					continue
				}
			}
			return err
		}
		break
	}

	// remount the target to account for propagation flags
	_, pgFlags, _ := storage.ParseMountOptions(options)
	if len(pgFlags) != 0 {
		for _, pg := range pgFlags {
			if err := unixMount(target, target, "", pg, ""); err != nil {
				return err
			}
		}
	}

	return nil
}

// Mount is just a wrapper over actual mount call. This wrapper finds out the controller
// number from the controller GUID string and calls mount.
func Mount(
	ctx context.Context,
	controller,
	lun uint8,
	target string,
	readonly bool,
	encrypted bool,
	options []string,
	verityInfo *guestresource.DeviceVerityInfo,
	securityPolicy securitypolicy.SecurityPolicyEnforcer,
) (err error) {
	cNum, err := fetchActualControllerNumber(ctx, controller)
	if err != nil {
		return err
	}
	return mount(ctx, cNum, lun, target, readonly, encrypted, options, verityInfo, securityPolicy)
}

// unmount unmounts a SCSI device mounted at `target`.
//
// If `encrypted` is true, it removes all its associated dm-crypto state.
func unmount(
	ctx context.Context,
	controller,
	lun uint8,
	target string,
	encrypted bool,
	verityInfo *guestresource.DeviceVerityInfo,
	securityPolicy securitypolicy.SecurityPolicyEnforcer,
) (err error) {
	ctx, span := trace.StartSpan(ctx, "scsi::Unmount")
	defer span.End()
	defer func() { oc.SetSpanStatus(span, err) }()

	span.AddAttributes(
		trace.Int64Attribute("controller", int64(controller)),
		trace.Int64Attribute("lun", int64(lun)),
		trace.StringAttribute("target", target))

	if err = securityPolicy.EnforceDeviceUnmountPolicy(target); err != nil {
		return errors.Wrapf(err, "unmounting scsi controller %d lun %d from  %s denied by policy", controller, lun, target)
	}

	// Unmount unencrypted device
	if err := storage.UnmountPath(ctx, target, true); err != nil {
		return errors.Wrapf(err, "unmount failed: "+target)
	}

	if verityInfo != nil {
		dmVerityName := fmt.Sprintf(verityDeviceFmt, controller, lun, verityInfo.RootDigest)
		if err := removeDevice(dmVerityName); err != nil {
			// Ignore failures, since the path has been unmounted at this point.
			log.G(ctx).WithError(err).Debugf("failed to remove dm verity target: %s", dmVerityName)
		}
	}

	if encrypted {
		if err := crypt.CleanupCryptDevice(target); err != nil {
			return errors.Wrapf(err, "failed to cleanup dm-crypt state: "+target)
		}
	}

	return nil
}

// Unmount is just a wrapper over actual unmount call. This wrapper finds out the controller
// number from the controller GUID string and calls mount.
func Unmount(
	ctx context.Context,
	controller,
	lun uint8,
	target string,
	encrypted bool,
	verityInfo *guestresource.DeviceVerityInfo,
	securityPolicy securitypolicy.SecurityPolicyEnforcer,
) (err error) {
	cNum, err := fetchActualControllerNumber(ctx, controller)
	if err != nil {
		return err
	}
	return unmount(ctx, cNum, lun, target, encrypted, verityInfo, securityPolicy)
}

// ControllerLunToName finds the `/dev/sd*` path to the SCSI device on
// `controller` index `lun`.
func ControllerLunToName(ctx context.Context, controller, lun uint8) (_ string, err error) {
	ctx, span := trace.StartSpan(ctx, "scsi::ControllerLunToName")
	defer span.End()
	defer func() { oc.SetSpanStatus(span, err) }()

	span.AddAttributes(
		trace.Int64Attribute("controller", int64(controller)),
		trace.Int64Attribute("lun", int64(lun)))

	scsiID := fmt.Sprintf("%d:0:0:%d", controller, lun)
	// Devices matching the given SCSI code should each have a subdirectory
	// under /sys/bus/scsi/devices/<scsiID>/block.
	blockPath := filepath.Join(scsiDevicesPath, scsiID, "block")
	var deviceNames []os.FileInfo
	for {
		deviceNames, err = ioutil.ReadDir(blockPath)
		if err != nil && !os.IsNotExist(err) {
			return "", err
		}
		if len(deviceNames) == 0 {
			select {
			case <-ctx.Done():
				return "", ctx.Err()
			default:
				time.Sleep(time.Millisecond * 10)
				continue
			}
		}
		break
	}

	if len(deviceNames) > 1 {
		return "", errors.Errorf("more than one block device could match SCSI ID \"%s\"", scsiID)
	}

	devicePath := filepath.Join("/dev", deviceNames[0].Name())
	log.G(ctx).WithField("devicePath", devicePath).Debug("found device path")
	return devicePath, nil
}

// unplugDevice finds the SCSI device on `controller` index `lun` and issues a
// guest initiated unplug.
//
// If the device is not attached returns no error.
func unplugDevice(ctx context.Context, controller, lun uint8) (err error) {
	_, span := trace.StartSpan(ctx, "scsi::UnplugDevice")
	defer span.End()
	defer func() { oc.SetSpanStatus(span, err) }()

	span.AddAttributes(
		trace.Int64Attribute("controller", int64(controller)),
		trace.Int64Attribute("lun", int64(lun)))

	scsiID := fmt.Sprintf("%d:0:0:%d", controller, lun)
	f, err := os.OpenFile(filepath.Join(scsiDevicesPath, scsiID, "delete"), os.O_WRONLY, 0644)
	if err != nil {
		if os.IsNotExist(err) {
			return nil
		}
		return err
	}
	defer f.Close()

	if _, err := f.Write([]byte("1\n")); err != nil {
		return err
	}
	return nil
}

// UnplugDevice is just a wrapper over actual unplugDevice call. This wrapper finds out the controller
// number from the controller GUID string and calls unplugDevice.
func UnplugDevice(ctx context.Context, controller, lun uint8) (err error) {
	cNum, err := fetchActualControllerNumber(ctx, controller)
	if err != nil {
		return err
	}
	return unplugDevice(ctx, cNum, lun)
}
<commit_msg>Fix dm-verity target naming format in linux guest (#1338)<commit_after>//go:build linux
// +build linux

package scsi

import (
	"context"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/pkg/errors"
	"go.opencensus.io/trace"
	"golang.org/x/sys/unix"

	"github.com/Microsoft/hcsshim/internal/guest/storage"
	"github.com/Microsoft/hcsshim/internal/guest/storage/crypt"
	dm "github.com/Microsoft/hcsshim/internal/guest/storage/devicemapper"
	"github.com/Microsoft/hcsshim/internal/log"
	"github.com/Microsoft/hcsshim/internal/oc"
	"github.com/Microsoft/hcsshim/internal/protocol/guestrequest"
	"github.com/Microsoft/hcsshim/internal/protocol/guestresource"
	"github.com/Microsoft/hcsshim/pkg/securitypolicy"
)

// Test dependencies
var (
	osMkdirAll  = os.MkdirAll
	osRemoveAll = os.RemoveAll
	unixMount   = unix.Mount

	// controllerLunToName is stubbed to make testing `Mount` easier.
	controllerLunToName = ControllerLunToName
	// createVerityTarget is stubbed for unit testing `Mount`
	createVerityTarget = dm.CreateVerityTarget
	// removeDevice is stubbed for unit testing `Mount`
	removeDevice = dm.RemoveDevice
)

const (
	scsiDevicesPath  = "/sys/bus/scsi/devices"
	vmbusDevicesPath = "/sys/bus/vmbus/devices"
	verityDeviceFmt  = "dm-verity-scsi-contr%d-lun%d-%s"
)

// fetchActualControllerNumber retrieves the actual controller number assigned to a SCSI controller
// with number `passedController`.
// When HCS creates the UVM it adds 4 SCSI controllers to the UVM but the 1st SCSI
// controller according to HCS can actually show up as 2nd, 3rd or 4th controller inside
// the UVM. So the i'th controller from HCS' perspective could actually be j'th controller
// inside the UVM. However, we can refer to the SCSI controllers with their GUIDs (that
// are hardcoded) and then using that GUID find out the SCSI controller number inside the
// guest. This function does exactly that.
func fetchActualControllerNumber(ctx context.Context, passedController uint8) (uint8, error) {
	// find the controller number by looking for a file named host<N> (e.g host1, host3 etc.)
	// `N` is the controller number.
	// Full file path would be /sys/bus/vmbus/devices/<controller-guid>/host<N>.
	controllerDirPath := path.Join(vmbusDevicesPath, guestrequest.ScsiControllerGuids[passedController])
	entries, err := ioutil.ReadDir(controllerDirPath)
	if err != nil {
		return 0, err
	}

	for _, entry := range entries {
		baseName := path.Base(entry.Name())
		if !strings.HasPrefix(baseName, "host") {
			continue
		}
		controllerStr := baseName[len("host"):]
		controllerNum, err := strconv.ParseUint(controllerStr, 10, 8)
		if err != nil {
			return 0, fmt.Errorf("failed to parse controller number from %s: %w", baseName, err)
		}
		return uint8(controllerNum), nil
	}
	return 0, fmt.Errorf("host<N> directory not found inside %s", controllerDirPath)
}

// mount creates a mount from the SCSI device on `controller` index `lun` to
// `target`
//
// `target` will be created. On mount failure the created `target` will be
// automatically cleaned up.
//
// If `encrypted` is set to true, the SCSI device will be encrypted using
// dm-crypt.
func mount(
	ctx context.Context,
	controller,
	lun uint8,
	target string,
	readonly bool,
	encrypted bool,
	options []string,
	verityInfo *guestresource.DeviceVerityInfo,
	securityPolicy securitypolicy.SecurityPolicyEnforcer,
) (err error) {
	spnCtx, span := trace.StartSpan(ctx, "scsi::Mount")
	defer span.End()
	defer func() { oc.SetSpanStatus(span, err) }()

	span.AddAttributes(
		trace.Int64Attribute("controller", int64(controller)),
		trace.Int64Attribute("lun", int64(lun)))

	source, err := controllerLunToName(spnCtx, controller, lun)
	if err != nil {
		return err
	}

	if readonly {
		// containers only have read-only layers so only enforce for them
		var deviceHash string
		if verityInfo != nil {
			deviceHash = verityInfo.RootDigest
		}

		err = securityPolicy.EnforceDeviceMountPolicy(target, deviceHash)
		if err != nil {
			return errors.Wrapf(err, "won't mount scsi controller %d lun %d onto %s", controller, lun, target)
		}

		if verityInfo != nil {
			dmVerityName := fmt.Sprintf(verityDeviceFmt, controller, lun, deviceHash)
			if source, err = createVerityTarget(spnCtx, source, dmVerityName, verityInfo); err != nil {
				return err
			}
			defer func() {
				if err != nil {
					if err := removeDevice(dmVerityName); err != nil {
						log.G(spnCtx).WithError(err).WithField("verityTarget", dmVerityName).Debug("failed to cleanup verity target")
					}
				}
			}()
		}
	}

	if err := osMkdirAll(target, 0700); err != nil {
		return err
	}
	defer func() {
		if err != nil {
			osRemoveAll(target)
		}
	}()

	// we only care about readonly mount option when mounting the device
	var flags uintptr
	data := ""
	if readonly {
		flags |= unix.MS_RDONLY
		data = "noload"
	}

	if encrypted {
		encryptedSource, err := crypt.EncryptDevice(spnCtx, source)
		if err != nil {
			return errors.Wrapf(err, "failed to mount encrypted device: "+source)
		}
		source = encryptedSource
	}

	for {
		if err := unixMount(source, target, "ext4", flags, data); err != nil {
			// The `source` found by controllerLunToName can take some time
			// before its actually available under `/dev/sd*`. Retry while we
			// wait for `source` to show up.
			if err == unix.ENOENT {
				select {
				case <-ctx.Done():
					return ctx.Err()
				default:
					time.Sleep(10 * time.Millisecond)
					continue
				}
			}
			return err
		}
		break
	}

	// remount the target to account for propagation flags
	_, pgFlags, _ := storage.ParseMountOptions(options)
	if len(pgFlags) != 0 {
		for _, pg := range pgFlags {
			if err := unixMount(target, target, "", pg, ""); err != nil {
				return err
			}
		}
	}

	return nil
}

// Mount is just a wrapper over actual mount call. This wrapper finds out the controller
// number from the controller GUID string and calls mount.
func Mount(
	ctx context.Context,
	controller,
	lun uint8,
	target string,
	readonly bool,
	encrypted bool,
	options []string,
	verityInfo *guestresource.DeviceVerityInfo,
	securityPolicy securitypolicy.SecurityPolicyEnforcer,
) (err error) {
	cNum, err := fetchActualControllerNumber(ctx, controller)
	if err != nil {
		return err
	}
	return mount(ctx, cNum, lun, target, readonly, encrypted, options, verityInfo, securityPolicy)
}

// unmount unmounts a SCSI device mounted at `target`.
//
// If `encrypted` is true, it removes all its associated dm-crypto state.
func unmount(
	ctx context.Context,
	controller,
	lun uint8,
	target string,
	encrypted bool,
	verityInfo *guestresource.DeviceVerityInfo,
	securityPolicy securitypolicy.SecurityPolicyEnforcer,
) (err error) {
	ctx, span := trace.StartSpan(ctx, "scsi::Unmount")
	defer span.End()
	defer func() { oc.SetSpanStatus(span, err) }()

	span.AddAttributes(
		trace.Int64Attribute("controller", int64(controller)),
		trace.Int64Attribute("lun", int64(lun)),
		trace.StringAttribute("target", target))

	if err = securityPolicy.EnforceDeviceUnmountPolicy(target); err != nil {
		return errors.Wrapf(err, "unmounting scsi controller %d lun %d from  %s denied by policy", controller, lun, target)
	}

	// Unmount unencrypted device
	if err := storage.UnmountPath(ctx, target, true); err != nil {
		return errors.Wrapf(err, "unmount failed: "+target)
	}

	if verityInfo != nil {
		dmVerityName := fmt.Sprintf(verityDeviceFmt, controller, lun, verityInfo.RootDigest)
		if err := removeDevice(dmVerityName); err != nil {
			// Ignore failures, since the path has been unmounted at this point.
			log.G(ctx).WithError(err).Debugf("failed to remove dm verity target: %s", dmVerityName)
		}
	}

	if encrypted {
		if err := crypt.CleanupCryptDevice(target); err != nil {
			return errors.Wrapf(err, "failed to cleanup dm-crypt state: "+target)
		}
	}

	return nil
}

// Unmount is just a wrapper over actual unmount call. This wrapper finds out the controller
// number from the controller GUID string and calls mount.
func Unmount(
	ctx context.Context,
	controller,
	lun uint8,
	target string,
	encrypted bool,
	verityInfo *guestresource.DeviceVerityInfo,
	securityPolicy securitypolicy.SecurityPolicyEnforcer,
) (err error) {
	cNum, err := fetchActualControllerNumber(ctx, controller)
	if err != nil {
		return err
	}
	return unmount(ctx, cNum, lun, target, encrypted, verityInfo, securityPolicy)
}

// ControllerLunToName finds the `/dev/sd*` path to the SCSI device on
// `controller` index `lun`.
func ControllerLunToName(ctx context.Context, controller, lun uint8) (_ string, err error) {
	ctx, span := trace.StartSpan(ctx, "scsi::ControllerLunToName")
	defer span.End()
	defer func() { oc.SetSpanStatus(span, err) }()

	span.AddAttributes(
		trace.Int64Attribute("controller", int64(controller)),
		trace.Int64Attribute("lun", int64(lun)))

	scsiID := fmt.Sprintf("%d:0:0:%d", controller, lun)
	// Devices matching the given SCSI code should each have a subdirectory
	// under /sys/bus/scsi/devices/<scsiID>/block.
	blockPath := filepath.Join(scsiDevicesPath, scsiID, "block")
	var deviceNames []os.FileInfo
	for {
		deviceNames, err = ioutil.ReadDir(blockPath)
		if err != nil && !os.IsNotExist(err) {
			return "", err
		}
		if len(deviceNames) == 0 {
			select {
			case <-ctx.Done():
				return "", ctx.Err()
			default:
				time.Sleep(time.Millisecond * 10)
				continue
			}
		}
		break
	}

	if len(deviceNames) > 1 {
		return "", errors.Errorf("more than one block device could match SCSI ID \"%s\"", scsiID)
	}

	devicePath := filepath.Join("/dev", deviceNames[0].Name())
	log.G(ctx).WithField("devicePath", devicePath).Debug("found device path")
	return devicePath, nil
}

// unplugDevice finds the SCSI device on `controller` index `lun` and issues a
// guest initiated unplug.
//
// If the device is not attached returns no error.
func unplugDevice(ctx context.Context, controller, lun uint8) (err error) {
	_, span := trace.StartSpan(ctx, "scsi::UnplugDevice")
	defer span.End()
	defer func() { oc.SetSpanStatus(span, err) }()

	span.AddAttributes(
		trace.Int64Attribute("controller", int64(controller)),
		trace.Int64Attribute("lun", int64(lun)))

	scsiID := fmt.Sprintf("%d:0:0:%d", controller, lun)
	f, err := os.OpenFile(filepath.Join(scsiDevicesPath, scsiID, "delete"), os.O_WRONLY, 0644)
	if err != nil {
		if os.IsNotExist(err) {
			return nil
		}
		return err
	}
	defer f.Close()

	if _, err := f.Write([]byte("1\n")); err != nil {
		return err
	}
	return nil
}

// UnplugDevice is just a wrapper over actual unplugDevice call. This wrapper finds out the controller
// number from the controller GUID string and calls unplugDevice.
func UnplugDevice(ctx context.Context, controller, lun uint8) (err error) {
	cNum, err := fetchActualControllerNumber(ctx, controller)
	if err != nil {
		return err
	}
	return unplugDevice(ctx, cNum, lun)
}
<|endoftext|>


<commit_before>// Package syscallcompat wraps Linux-specific syscalls.
package syscallcompat

import (
	"sync"
	"syscall"

	"golang.org/x/sys/unix"

	"github.com/hanwen/go-fuse/fuse"

	"github.com/rfjakob/gocryptfs/internal/tlog"
)

const (
	_FALLOC_FL_KEEP_SIZE = 0x01

	// O_DIRECT means oncached I/O on Linux. No direct equivalent on MacOS and defined
	// to zero there.
	O_DIRECT = syscall.O_DIRECT

	// O_PATH is only defined on Linux
	O_PATH = unix.O_PATH
)

var preallocWarn sync.Once

// EnospcPrealloc preallocates ciphertext space without changing the file
// size. This guarantees that we don't run out of space while writing a
// ciphertext block (that would corrupt the block).
func EnospcPrealloc(fd int, off int64, len int64) (err error) {
	for {
		err = syscall.Fallocate(fd, _FALLOC_FL_KEEP_SIZE, off, len)
		if err == syscall.EINTR {
			// fallocate, like many syscalls, can return EINTR. This is not an
			// error and just signifies that the operation was interrupted by a
			// signal and we should try again.
			continue
		}
		if err == syscall.EOPNOTSUPP {
			// ZFS and ext3 do not support fallocate. Warn but continue anyway.
			// https://github.com/rfjakob/gocryptfs/issues/22
			preallocWarn.Do(func() {
				tlog.Warn.Printf("Warning: The underlying filesystem " +
					"does not support fallocate(2). gocryptfs will continue working " +
					"but is no longer resistant against out-of-space errors.\n")
			})
			return nil
		}
		return err
	}
}

// Fallocate wraps the Fallocate syscall.
func Fallocate(fd int, mode uint32, off int64, len int64) (err error) {
	return syscall.Fallocate(fd, mode, off, len)
}

// Openat wraps the Openat syscall.
func Openat(dirfd int, path string, flags int, mode uint32) (fd int, err error) {
	// Why would we ever want to call this without O_NOFOLLOW and O_EXCL?
	if !(flags&syscall.O_CREAT != 0 && flags&syscall.O_EXCL != 0) && flags&syscall.O_NOFOLLOW == 0 {
		tlog.Warn.Printf("Openat: adding missing O_NOFOLLOW flag")
		flags |= syscall.O_NOFOLLOW
	}
	return syscall.Openat(dirfd, path, flags, mode)
}

// Renameat wraps the Renameat syscall.
func Renameat(olddirfd int, oldpath string, newdirfd int, newpath string) (err error) {
	return syscall.Renameat(olddirfd, oldpath, newdirfd, newpath)
}

// Unlinkat syscall.
func Unlinkat(dirfd int, path string, flags int) (err error) {
	return unix.Unlinkat(dirfd, path, flags)
}

// Mknodat wraps the Mknodat syscall.
func Mknodat(dirfd int, path string, mode uint32, dev int) (err error) {
	return syscall.Mknodat(dirfd, path, mode, dev)
}

// Dup3 wraps the Dup3 syscall. We want to use Dup3 rather than Dup2 because Dup2
// is not implemented on arm64.
func Dup3(oldfd int, newfd int, flags int) (err error) {
	return syscall.Dup3(oldfd, newfd, flags)
}

// Fchmodat syscall.
func Fchmodat(dirfd int, path string, mode uint32, flags int) (err error) {
	// Linux does not support passing flags to fchmodat! From the man page:
	// AT_SYMLINK_NOFOLLOW ... This flag is not currently implemented.
	// Linux ignores any flags, but Go stdlib rejects them with EOPNOTSUPP starting
	// with Go 1.11. See https://github.com/golang/go/issues/20130 for more info.
	// TODO: Use fchmodat2 once available on Linux.
	return syscall.Fchmodat(dirfd, path, mode, 0)
}

// Fchownat syscall.
func Fchownat(dirfd int, path string, uid int, gid int, flags int) (err error) {
	// Why would we ever want to call this without AT_SYMLINK_NOFOLLOW?
	if flags&unix.AT_SYMLINK_NOFOLLOW == 0 {
		tlog.Warn.Printf("Fchownat: adding missing AT_SYMLINK_NOFOLLOW flag")
		flags |= unix.AT_SYMLINK_NOFOLLOW
	}
	return syscall.Fchownat(dirfd, path, uid, gid, flags)
}

// Symlinkat syscall.
func Symlinkat(oldpath string, newdirfd int, newpath string) (err error) {
	return unix.Symlinkat(oldpath, newdirfd, newpath)
}

// Mkdirat syscall.
func Mkdirat(dirfd int, path string, mode uint32) (err error) {
	return syscall.Mkdirat(dirfd, path, mode)
}

// Fstatat syscall.
func Fstatat(dirfd int, path string, stat *unix.Stat_t, flags int) (err error) {
	// Why would we ever want to call this without AT_SYMLINK_NOFOLLOW?
	if flags&unix.AT_SYMLINK_NOFOLLOW == 0 {
		tlog.Warn.Printf("Fstatat: adding missing AT_SYMLINK_NOFOLLOW flag")
		flags |= unix.AT_SYMLINK_NOFOLLOW
	}
	return unix.Fstatat(dirfd, path, stat, flags)
}

// Getdents syscall.
func Getdents(fd int) ([]fuse.DirEntry, error) {
	return getdents(fd)
}
<commit_msg>syscallcompat: untangle Openat flag check<commit_after>// Package syscallcompat wraps Linux-specific syscalls.
package syscallcompat

import (
	"sync"
	"syscall"

	"golang.org/x/sys/unix"

	"github.com/hanwen/go-fuse/fuse"

	"github.com/rfjakob/gocryptfs/internal/tlog"
)

const (
	_FALLOC_FL_KEEP_SIZE = 0x01

	// O_DIRECT means oncached I/O on Linux. No direct equivalent on MacOS and defined
	// to zero there.
	O_DIRECT = syscall.O_DIRECT

	// O_PATH is only defined on Linux
	O_PATH = unix.O_PATH
)

var preallocWarn sync.Once

// EnospcPrealloc preallocates ciphertext space without changing the file
// size. This guarantees that we don't run out of space while writing a
// ciphertext block (that would corrupt the block).
func EnospcPrealloc(fd int, off int64, len int64) (err error) {
	for {
		err = syscall.Fallocate(fd, _FALLOC_FL_KEEP_SIZE, off, len)
		if err == syscall.EINTR {
			// fallocate, like many syscalls, can return EINTR. This is not an
			// error and just signifies that the operation was interrupted by a
			// signal and we should try again.
			continue
		}
		if err == syscall.EOPNOTSUPP {
			// ZFS and ext3 do not support fallocate. Warn but continue anyway.
			// https://github.com/rfjakob/gocryptfs/issues/22
			preallocWarn.Do(func() {
				tlog.Warn.Printf("Warning: The underlying filesystem " +
					"does not support fallocate(2). gocryptfs will continue working " +
					"but is no longer resistant against out-of-space errors.\n")
			})
			return nil
		}
		return err
	}
}

// Fallocate wraps the Fallocate syscall.
func Fallocate(fd int, mode uint32, off int64, len int64) (err error) {
	return syscall.Fallocate(fd, mode, off, len)
}

// Openat wraps the Openat syscall.
func Openat(dirfd int, path string, flags int, mode uint32) (fd int, err error) {
	if flags&syscall.O_CREAT != 0 {
		// O_CREAT should be used with O_EXCL. O_NOFOLLOW has no effect with O_EXCL.
		if flags&syscall.O_EXCL == 0 {
			tlog.Warn.Printf("Openat: O_CREAT without O_EXCL: flags = %#x", flags)
			flags |= syscall.O_EXCL
		}
	} else {
		// If O_CREAT is not used, we should use O_NOFOLLOW
		if flags&syscall.O_NOFOLLOW == 0 {
			tlog.Warn.Printf("Openat: O_NOFOLLOW missing: flags = %#x", flags)
			flags |= syscall.O_NOFOLLOW
		}
	}
	return syscall.Openat(dirfd, path, flags, mode)
}

// Renameat wraps the Renameat syscall.
func Renameat(olddirfd int, oldpath string, newdirfd int, newpath string) (err error) {
	return syscall.Renameat(olddirfd, oldpath, newdirfd, newpath)
}

// Unlinkat syscall.
func Unlinkat(dirfd int, path string, flags int) (err error) {
	return unix.Unlinkat(dirfd, path, flags)
}

// Mknodat wraps the Mknodat syscall.
func Mknodat(dirfd int, path string, mode uint32, dev int) (err error) {
	return syscall.Mknodat(dirfd, path, mode, dev)
}

// Dup3 wraps the Dup3 syscall. We want to use Dup3 rather than Dup2 because Dup2
// is not implemented on arm64.
func Dup3(oldfd int, newfd int, flags int) (err error) {
	return syscall.Dup3(oldfd, newfd, flags)
}

// Fchmodat syscall.
func Fchmodat(dirfd int, path string, mode uint32, flags int) (err error) {
	// Linux does not support passing flags to fchmodat! From the man page:
	// AT_SYMLINK_NOFOLLOW ... This flag is not currently implemented.
	// Linux ignores any flags, but Go stdlib rejects them with EOPNOTSUPP starting
	// with Go 1.11. See https://github.com/golang/go/issues/20130 for more info.
	// TODO: Use fchmodat2 once available on Linux.
	return syscall.Fchmodat(dirfd, path, mode, 0)
}

// Fchownat syscall.
func Fchownat(dirfd int, path string, uid int, gid int, flags int) (err error) {
	// Why would we ever want to call this without AT_SYMLINK_NOFOLLOW?
	if flags&unix.AT_SYMLINK_NOFOLLOW == 0 {
		tlog.Warn.Printf("Fchownat: adding missing AT_SYMLINK_NOFOLLOW flag")
		flags |= unix.AT_SYMLINK_NOFOLLOW
	}
	return syscall.Fchownat(dirfd, path, uid, gid, flags)
}

// Symlinkat syscall.
func Symlinkat(oldpath string, newdirfd int, newpath string) (err error) {
	return unix.Symlinkat(oldpath, newdirfd, newpath)
}

// Mkdirat syscall.
func Mkdirat(dirfd int, path string, mode uint32) (err error) {
	return syscall.Mkdirat(dirfd, path, mode)
}

// Fstatat syscall.
func Fstatat(dirfd int, path string, stat *unix.Stat_t, flags int) (err error) {
	// Why would we ever want to call this without AT_SYMLINK_NOFOLLOW?
	if flags&unix.AT_SYMLINK_NOFOLLOW == 0 {
		tlog.Warn.Printf("Fstatat: adding missing AT_SYMLINK_NOFOLLOW flag")
		flags |= unix.AT_SYMLINK_NOFOLLOW
	}
	return unix.Fstatat(dirfd, path, stat, flags)
}

// Getdents syscall.
func Getdents(fd int) ([]fuse.DirEntry, error) {
	return getdents(fd)
}
<|endoftext|>


<commit_before>//  Copyright 2018 Istio Authors
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

package mcp

import (
	"context"
	"errors"
	"fmt"
	"net/url"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/gogo/protobuf/proto"
	"google.golang.org/grpc"
	"k8s.io/apimachinery/pkg/runtime/schema"

	mcp "istio.io/api/mcp/v1alpha1"
	"istio.io/istio/galley/pkg/kube/converter/legacy"
	"istio.io/istio/galley/pkg/metadata/kube"
	"istio.io/istio/mixer/pkg/config/store"
	"istio.io/istio/pkg/log"
	"istio.io/istio/pkg/mcp/client"
	"istio.io/istio/pkg/mcp/configz"
	"istio.io/istio/pkg/mcp/creds"
	"istio.io/istio/pkg/probe"
)

var scope = log.RegisterScope("mcp", "Mixer MCP client stack", 0)

const (
	mixerNodeID           = ""
	eventChannelSize      = 4096
	requiredCertCheckFreq = 500 * time.Millisecond
)

// Register registers this module as a StoreBackend.
// Do not use 'init()' for automatic registration; linker will drop
// the whole module because it looks unused.
func Register(builders map[string]store.Builder) {
	builder := func(u *url.URL, gv *schema.GroupVersion, credOptions *creds.Options) (store.Backend, error) {
		return newStore(u, credOptions, nil)
	}

	builders["mcp"] = builder
	builders["mcpi"] = builder
}

// NewStore creates a new Store instance.
func newStore(u *url.URL, credOptions *creds.Options, fn updateHookFn) (store.Backend, error) {
	insecure := true
	if u.Scheme == "mcp" {
		insecure = false
		if credOptions == nil {
			return nil, errors.New("no credentials specified with secure MCP scheme")
		}
	}

	return &backend{
		serverAddress: u.Host,
		insecure:      insecure,
		credOptions:   credOptions,
		Probe:         probe.NewProbe(),
		updateHook:    fn,
	}, nil
}

// updateHookFn is a testing hook function
type updateHookFn func()

// Store offers store.StoreBackend interface through kubernetes custom resource definitions.
type backend struct {
	// mapping of CRD <> typeURLs.
	mapping *mapping

	// Use insecure communication for gRPC.
	insecure bool

	// address of the MCP server.
	serverAddress string

	// MCP credential options
	credOptions *creds.Options

	// The cancellation function that is used to cancel gRPC/MCP operations.
	cancel context.CancelFunc

	// The in-memory state, where resources are kept for out-of-band get and list calls.
	state *state

	// channel for publishing backend events. chLock is used to protect against ch lifecycle
	// events (e.g. creation of the channel).
	// It also protects against a race, where a channel could be created while there is incoming
	// data/changes.
	chLock sync.Mutex
	ch     chan store.BackendEvent

	// The update hook that was registered. This is for testing.
	updateHook updateHookFn

	*probe.Probe
}

var _ store.Backend = &backend{}
var _ probe.SupportsProbe = &backend{}
var _ client.Updater = &backend{}

// state is the in-memory cache.
type state struct {
	sync.Mutex

	// items stored by kind, then by key.
	items map[string]map[store.Key]*store.BackEndResource
}

// Init implements store.Backend.Init.
func (b *backend) Init(kinds []string) error {
	m, err := constructMapping(kinds, kube.Types)
	if err != nil {
		return err
	}
	b.mapping = m

	typeURLs := b.mapping.typeURLs()
	scope.Infof("Requesting following types:")
	for i, url := range typeURLs {
		scope.Infof("  [%d] %s", i, url)
	}

	ctx, cancel := context.WithCancel(context.Background())

	securityOption := grpc.WithInsecure()
	if !b.insecure {
		address := b.serverAddress
		if strings.Contains(address, ":") {
			address = strings.Split(address, ":")[0]
		}

		requiredFiles := []string{b.credOptions.CertificateFile, b.credOptions.KeyFile, b.credOptions.CACertificateFile}
		log.Infof("Secure MSP configured. Waiting for required certificate files to become available: %v", requiredFiles)
		for len(requiredFiles) > 0 {
			if _, err := os.Stat(requiredFiles[0]); os.IsNotExist(err) {
				log.Infof("%v not found. Checking again in %v", requiredFiles[0], requiredCertCheckFreq)
				select {
				case <-ctx.Done():
					return ctx.Err()
				case <-time.After(requiredCertCheckFreq):
					// retry
				}
				continue
			}

			log.Infof("%v found", requiredFiles[0])
			requiredFiles = requiredFiles[1:]
		}

		watcher, err := creds.WatchFiles(ctx.Done(), b.credOptions)
		if err != nil {
			return err
		}
		credentials := creds.CreateForClient(address, watcher)
		securityOption = grpc.WithTransportCredentials(credentials)
	}

	conn, err := grpc.DialContext(ctx, b.serverAddress, securityOption)
	if err != nil {
		cancel()
		scope.Errorf("Error connecting to server: %v\n", err)
		return err
	}

	cl := mcp.NewAggregatedMeshConfigServiceClient(conn)
	c := client.New(cl, typeURLs, b, mixerNodeID, map[string]string{})
	configz.Register(c)

	b.state = &state{
		items: make(map[string]map[store.Key]*store.BackEndResource),
	}

	go c.Run(ctx)
	b.cancel = cancel

	return nil
}

// WaitForSynced implements store.Backend interface.
func (b *backend) WaitForSynced(time.Duration) error {
	// TODO(ozevren): implement for MCP
	return nil
}

// Stop implements store.backend.Stop.
func (b *backend) Stop() {
	if b.cancel != nil {
		b.cancel()
		b.cancel = nil
	}
}

// Watch creates a channel to receive the events.
func (b *backend) Watch() (<-chan store.BackendEvent, error) {
	b.chLock.Lock()
	defer b.chLock.Unlock()

	if b.ch != nil {
		return nil, fmt.Errorf("watch was already called")
	}

	b.ch = make(chan store.BackendEvent, eventChannelSize)

	return b.ch, nil
}

// Get returns a resource's spec to the key.
func (b *backend) Get(key store.Key) (*store.BackEndResource, error) {
	b.state.Lock()
	defer b.state.Unlock()

	perTypeState, found := b.state.items[key.Kind]
	if !found {
		return nil, store.ErrNotFound
	}

	item, found := perTypeState[key]
	if !found {
		return nil, store.ErrNotFound
	}

	return item, nil
}

// List returns the whole mapping from key to resource specs in the store.
func (b *backend) List() map[store.Key]*store.BackEndResource {
	b.state.Lock()
	defer b.state.Unlock()

	result := make(map[store.Key]*store.BackEndResource)
	for _, perTypeItems := range b.state.items {
		for k, v := range perTypeItems {
			result[k] = v
		}
	}

	return result
}

// Apply implements client.Updater.Apply
func (b *backend) Apply(change *client.Change) error {
	b.state.Lock()
	defer b.state.Unlock()
	defer b.callUpdateHook()

	newTypeStates := make(map[string]map[store.Key]*store.BackEndResource)
	typeURL := change.TypeURL

	scope.Debugf("Received update for: type:%s, count:%d", typeURL, len(change.Objects))

	for _, o := range change.Objects {
		var kind string
		var name string
		var contents proto.Message

		if scope.DebugEnabled() {
			scope.Debugf("Processing incoming resource: %q @%s [%s]",
				o.Metadata.Name, o.Metadata.Version, o.TypeURL)
		}

		// Demultiplex the resource, if it is a legacy type, and figure out its kind.
		if isLegacyTypeURL(typeURL) {
			// Extract the kind from payload.
			legacyResource := o.Resource.(*legacy.LegacyMixerResource)
			name = legacyResource.Name
			kind = legacyResource.Kind
			contents = legacyResource.Contents
		} else {
			// Otherwise, simply do a direct mapping from typeURL to kind
			name = o.Metadata.Name
			kind = b.mapping.kind(typeURL)
			contents = o.Resource
		}

		collection, found := newTypeStates[kind]
		if !found {
			collection = make(map[store.Key]*store.BackEndResource)
			newTypeStates[kind] = collection
		}

		// Map it to Mixer's store model, and put it in the new collection.

		key := toKey(kind, name)
		resource, err := toBackendResource(key, contents, o.Metadata.Version)
		if err != nil {
			return err
		}
		collection[key] = resource
	}

	// Lock the channel state, as we will start publishing events soon.
	b.chLock.Lock()
	defer b.chLock.Unlock()

	// Now, diff against the in-memory state and generate store events.
	for kind, newTypeState := range newTypeStates {
		oldTypeState, found := b.state.items[kind]

		// Replace the old collection with the new one.
		// We can do this, because there is no error that can be raised from now on.
		b.state.items[kind] = newTypeState

		// If the downstream users haven't started listening yet, we don't need to
		// send any events.
		if b.ch == nil {
			continue
		}

		// Otherwise, start pumping events by diffing old and new states.
		if found {
			for k := range oldTypeState {
				if _, exists := newTypeState[k]; !exists {
					b.ch <- store.BackendEvent{Key: k, Type: store.Delete}
					continue
				}
			}
		}

		for k, v := range newTypeState {
			b.ch <- store.BackendEvent{Key: k, Type: store.Update, Value: v}
		}
	}

	return nil
}

func (b *backend) callUpdateHook() {
	if b.updateHook != nil {
		b.updateHook()
	}
}
<commit_msg>replace Mutex with RWMutex in mixer mcp backend (#8267)<commit_after>//  Copyright 2018 Istio Authors
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

package mcp

import (
	"context"
	"errors"
	"fmt"
	"net/url"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/gogo/protobuf/proto"
	"google.golang.org/grpc"
	"k8s.io/apimachinery/pkg/runtime/schema"

	mcp "istio.io/api/mcp/v1alpha1"
	"istio.io/istio/galley/pkg/kube/converter/legacy"
	"istio.io/istio/galley/pkg/metadata/kube"
	"istio.io/istio/mixer/pkg/config/store"
	"istio.io/istio/pkg/log"
	"istio.io/istio/pkg/mcp/client"
	"istio.io/istio/pkg/mcp/configz"
	"istio.io/istio/pkg/mcp/creds"
	"istio.io/istio/pkg/probe"
)

var scope = log.RegisterScope("mcp", "Mixer MCP client stack", 0)

const (
	mixerNodeID           = ""
	eventChannelSize      = 4096
	requiredCertCheckFreq = 500 * time.Millisecond
)

// Register registers this module as a StoreBackend.
// Do not use 'init()' for automatic registration; linker will drop
// the whole module because it looks unused.
func Register(builders map[string]store.Builder) {
	builder := func(u *url.URL, gv *schema.GroupVersion, credOptions *creds.Options) (store.Backend, error) {
		return newStore(u, credOptions, nil)
	}

	builders["mcp"] = builder
	builders["mcpi"] = builder
}

// NewStore creates a new Store instance.
func newStore(u *url.URL, credOptions *creds.Options, fn updateHookFn) (store.Backend, error) {
	insecure := true
	if u.Scheme == "mcp" {
		insecure = false
		if credOptions == nil {
			return nil, errors.New("no credentials specified with secure MCP scheme")
		}
	}

	return &backend{
		serverAddress: u.Host,
		insecure:      insecure,
		credOptions:   credOptions,
		Probe:         probe.NewProbe(),
		updateHook:    fn,
	}, nil
}

// updateHookFn is a testing hook function
type updateHookFn func()

// Store offers store.StoreBackend interface through kubernetes custom resource definitions.
type backend struct {
	// mapping of CRD <> typeURLs.
	mapping *mapping

	// Use insecure communication for gRPC.
	insecure bool

	// address of the MCP server.
	serverAddress string

	// MCP credential options
	credOptions *creds.Options

	// The cancellation function that is used to cancel gRPC/MCP operations.
	cancel context.CancelFunc

	// The in-memory state, where resources are kept for out-of-band get and list calls.
	state *state

	// channel for publishing backend events. chLock is used to protect against ch lifecycle
	// events (e.g. creation of the channel).
	// It also protects against a race, where a channel could be created while there is incoming
	// data/changes.
	chLock sync.Mutex
	ch     chan store.BackendEvent

	// The update hook that was registered. This is for testing.
	updateHook updateHookFn

	*probe.Probe
}

var _ store.Backend = &backend{}
var _ probe.SupportsProbe = &backend{}
var _ client.Updater = &backend{}

// state is the in-memory cache.
type state struct {
	sync.RWMutex

	// items stored by kind, then by key.
	items map[string]map[store.Key]*store.BackEndResource
}

// Init implements store.Backend.Init.
func (b *backend) Init(kinds []string) error {
	m, err := constructMapping(kinds, kube.Types)
	if err != nil {
		return err
	}
	b.mapping = m

	typeURLs := b.mapping.typeURLs()
	scope.Infof("Requesting following types:")
	for i, url := range typeURLs {
		scope.Infof("  [%d] %s", i, url)
	}

	ctx, cancel := context.WithCancel(context.Background())

	securityOption := grpc.WithInsecure()
	if !b.insecure {
		address := b.serverAddress
		if strings.Contains(address, ":") {
			address = strings.Split(address, ":")[0]
		}

		requiredFiles := []string{b.credOptions.CertificateFile, b.credOptions.KeyFile, b.credOptions.CACertificateFile}
		log.Infof("Secure MSP configured. Waiting for required certificate files to become available: %v", requiredFiles)
		for len(requiredFiles) > 0 {
			if _, err := os.Stat(requiredFiles[0]); os.IsNotExist(err) {
				log.Infof("%v not found. Checking again in %v", requiredFiles[0], requiredCertCheckFreq)
				select {
				case <-ctx.Done():
					return ctx.Err()
				case <-time.After(requiredCertCheckFreq):
					// retry
				}
				continue
			}

			log.Infof("%v found", requiredFiles[0])
			requiredFiles = requiredFiles[1:]
		}

		watcher, err := creds.WatchFiles(ctx.Done(), b.credOptions)
		if err != nil {
			return err
		}
		credentials := creds.CreateForClient(address, watcher)
		securityOption = grpc.WithTransportCredentials(credentials)
	}

	conn, err := grpc.DialContext(ctx, b.serverAddress, securityOption)
	if err != nil {
		cancel()
		scope.Errorf("Error connecting to server: %v\n", err)
		return err
	}

	cl := mcp.NewAggregatedMeshConfigServiceClient(conn)
	c := client.New(cl, typeURLs, b, mixerNodeID, map[string]string{})
	configz.Register(c)

	b.state = &state{
		items: make(map[string]map[store.Key]*store.BackEndResource),
	}

	go c.Run(ctx)
	b.cancel = cancel

	return nil
}

// WaitForSynced implements store.Backend interface.
func (b *backend) WaitForSynced(time.Duration) error {
	// TODO(ozevren): implement for MCP
	return nil
}

// Stop implements store.backend.Stop.
func (b *backend) Stop() {
	if b.cancel != nil {
		b.cancel()
		b.cancel = nil
	}
}

// Watch creates a channel to receive the events.
func (b *backend) Watch() (<-chan store.BackendEvent, error) {
	b.chLock.Lock()
	defer b.chLock.Unlock()

	if b.ch != nil {
		return nil, fmt.Errorf("watch was already called")
	}

	b.ch = make(chan store.BackendEvent, eventChannelSize)

	return b.ch, nil
}

// Get returns a resource's spec to the key.
func (b *backend) Get(key store.Key) (*store.BackEndResource, error) {
	b.state.RLock()
	defer b.state.RUnlock()

	perTypeState, found := b.state.items[key.Kind]
	if !found {
		return nil, store.ErrNotFound
	}

	item, found := perTypeState[key]
	if !found {
		return nil, store.ErrNotFound
	}

	return item, nil
}

// List returns the whole mapping from key to resource specs in the store.
func (b *backend) List() map[store.Key]*store.BackEndResource {
	b.state.RLock()
	defer b.state.RUnlock()

	result := make(map[store.Key]*store.BackEndResource)
	for _, perTypeItems := range b.state.items {
		for k, v := range perTypeItems {
			result[k] = v
		}
	}

	return result
}

// Apply implements client.Updater.Apply
func (b *backend) Apply(change *client.Change) error {
	b.state.Lock()
	defer b.state.Unlock()
	defer b.callUpdateHook()

	newTypeStates := make(map[string]map[store.Key]*store.BackEndResource)
	typeURL := change.TypeURL

	scope.Debugf("Received update for: type:%s, count:%d", typeURL, len(change.Objects))

	for _, o := range change.Objects {
		var kind string
		var name string
		var contents proto.Message

		if scope.DebugEnabled() {
			scope.Debugf("Processing incoming resource: %q @%s [%s]",
				o.Metadata.Name, o.Metadata.Version, o.TypeURL)
		}

		// Demultiplex the resource, if it is a legacy type, and figure out its kind.
		if isLegacyTypeURL(typeURL) {
			// Extract the kind from payload.
			legacyResource := o.Resource.(*legacy.LegacyMixerResource)
			name = legacyResource.Name
			kind = legacyResource.Kind
			contents = legacyResource.Contents
		} else {
			// Otherwise, simply do a direct mapping from typeURL to kind
			name = o.Metadata.Name
			kind = b.mapping.kind(typeURL)
			contents = o.Resource
		}

		collection, found := newTypeStates[kind]
		if !found {
			collection = make(map[store.Key]*store.BackEndResource)
			newTypeStates[kind] = collection
		}

		// Map it to Mixer's store model, and put it in the new collection.

		key := toKey(kind, name)
		resource, err := toBackendResource(key, contents, o.Metadata.Version)
		if err != nil {
			return err
		}
		collection[key] = resource
	}

	// Lock the channel state, as we will start publishing events soon.
	b.chLock.Lock()
	defer b.chLock.Unlock()

	// Now, diff against the in-memory state and generate store events.
	for kind, newTypeState := range newTypeStates {
		oldTypeState, found := b.state.items[kind]

		// Replace the old collection with the new one.
		// We can do this, because there is no error that can be raised from now on.
		b.state.items[kind] = newTypeState

		// If the downstream users haven't started listening yet, we don't need to
		// send any events.
		if b.ch == nil {
			continue
		}

		// Otherwise, start pumping events by diffing old and new states.
		if found {
			for k := range oldTypeState {
				if _, exists := newTypeState[k]; !exists {
					b.ch <- store.BackendEvent{Key: k, Type: store.Delete}
					continue
				}
			}
		}

		for k, v := range newTypeState {
			b.ch <- store.BackendEvent{Key: k, Type: store.Update, Value: v}
		}
	}

	return nil
}

func (b *backend) callUpdateHook() {
	if b.updateHook != nil {
		b.updateHook()
	}
}
<|endoftext|>


<commit_before>package main

import (
	"math/rand"
	"time"

	"github.com/simulatedsimian/flotilla-go/flotilla"
)

// build a struct that has all the modules you need.
var modules struct {
	flotilla.Matrix
	flotilla.Touch
	flotilla.Number
}

func drawPipes(m *flotilla.Matrix) {
	gap := rand.Intn(7)
	for y := 0; y < gap; y++ {
		m.Plot(7, y, 1)
	}
	for y := gap + 2; y < 8; y++ {
		m.Plot(7, y, 1)
	}
}

type gameState struct {
	playerPos int
	deltaPos  int
	scrollPos int
	score     int
	hiscore   int
}

func main() {
	// connect to the dock
	client, err := flotilla.ConnectToDock("/dev/ttyACM0")
	flotilla.ExitOnError(err)

	// wait for all modules to be connected
	client.AquireModules(&modules)
	modules.Matrix.SetBrightness(32)

	gs := gameState{playerPos: 16 * 3}

	client.OnTick(func(t time.Time) {
		modules.Matrix.Plot(0, gs.playerPos/16, 0)

		if gs.scrollPos&7 == 0 {
			modules.Matrix.ScrollLeft(0)
			if gs.scrollPos&31 == 0 {
				drawPipes(&modules.Matrix)
			}
		}
		gs.scrollPos++

		gs.playerPos += gs.deltaPos
		gs.deltaPos++
		modules.Matrix.Plot(0, gs.playerPos/16, gs.scrollPos&1)
	})

	modules.Touch.OnChange(func(button int, pressed bool) {
		if pressed {
			gs.deltaPos -= 8
		}
	})

	// go!!
	client.Run(time.Millisecond * 50)
}
<commit_msg>impl flappy matrix example<commit_after>package main

import (
	"math/rand"
	"time"

	"github.com/simulatedsimian/flotilla-go/flotilla"
)

// build a struct that has all the modules you need.
var modules struct {
	flotilla.Matrix
	flotilla.Touch
	flotilla.Number
}

func drawPipes(m *flotilla.Matrix) {
	gap := rand.Intn(7)
	for y := 0; y < gap; y++ {
		m.Plot(7, y, 1)
	}
	for y := gap + 2; y < 8; y++ {
		m.Plot(7, y, 1)
	}
}

type gameState struct {
	playerPos int
	deltaPos  int
	scrollPos int
	score     int
	hiscore   int
}

func main() {
	// connect to the dock
	client, err := flotilla.ConnectToDock("/dev/ttyACM0")
	flotilla.ExitOnError(err)

	// wait for all modules to be connected
	client.AquireModules(&modules)
	modules.Matrix.SetBrightness(32)

	gs := gameState{playerPos: 256 * 3}

	client.OnTick(func(t time.Time) {
		modules.Matrix.Plot(0, gs.playerPos/256, 0)

		if gs.scrollPos&7 == 0 {
			modules.Matrix.ScrollLeft(0)
			if gs.scrollPos&31 == 0 {
				drawPipes(&modules.Matrix)
			}
		}
		gs.scrollPos++

		maxDelta := 32
		gs.deltaPos = flotilla.Limit(gs.deltaPos, -maxDelta, maxDelta)

		gs.playerPos += gs.deltaPos
		gs.deltaPos += 12
		modules.Matrix.Plot(0, gs.playerPos/256, gs.scrollPos&1)
	})

	modules.Touch.OnChange(func(button int, pressed bool) {
		if pressed {
			gs.deltaPos -= 98
		}
	})

	// go!!
	client.Run(time.Millisecond * 50)
}
<|endoftext|>


<commit_before>//
// Copyright (c) 2016 Dennis Chen
//
// This file is part of Clashr.
//
// Clashr is free software: you can redistribute it and/or modify it under the
// terms of the GNU Affero General Public License as published by the Free
// Software Foundation, either version 3 of the License, or (at your option)
// any later version.
//
// Clashr is distributed in the hope that it will be useful, but WITHOUT ANY
// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
// FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for
// more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with Clashr.  If not, see <http://www.gnu.org/licenses/>.

// Package cgroup provides a large amount of functionality for properly
// implementing cgroups in the judging and tracking of programs in Clashr.
// Most of this work is inspired by the DOMJudge implementation runguard.c and
// attempts to mimic a majoritiy of it's behavior
//
package cgroup

// #cgo pkg-config: libcgroup
// #include <libcgroup.h>
import "C"

import (
	"errors"
	"fmt"
)

// A uniform interface for the cgroup actions.
type Cgroup struct {
	name string
	cpus string
	mems int64
}

// OutputStats returns the memory used (in bytes) by a process controlled by a
// cgroup.
func (g *Cgroup) OutputStats() (int64, error) {
	var param *C.char

	param = C.CString(g.name)
	cg := C.cgroup_new_cgroup(param)
	if cg == nil {
		return 0, errors.New("cgroup_new_cgroup")
	}
	defer C.cgroup_free(&cg)

	if ret := C.cgroup_get_cgroup(cg); ret != 0 {
		err := fmt.Errorf("Get cgroup information: %s(d)",
			C.cgroup_strerror(ret), ret)
		return 0, err
	}
	param = C.CString("memory")
	cgController := C.cgroup_get_controller(cg, param)

	var maxUsage C.int64_t
	param = C.CString("memory.memsw.max_usage_in_bytes")
	if ret := C.cgroup_get_value_int64(cgController, param, &maxUsage); ret != 0 {
		err := fmt.Errorf("Get cgroup value: %s(%d)",
			C.cgroup_strerror(ret), ret)
		return 0, err
	}

	return int64(maxUsage), nil
}

// Create makes a new instance of the cgroup interface. It sets the memory
// limit of the cgroup to memsize, and the set of CPUs to use to cpuset. If
// memory does not need to be limited, set memsize to syscall.RLIMIT_INFINITY.
// Similarly, set cpuset to the empty string if CPUs do not need to be set.
func Create(cgroupname, cpuset string, memsize int64) (Cgroup, error) {
	var param *C.char

	param = C.CString(cgroupname)
	cg := C.cgroup_new_cgroup(param)
	if cg == nil {
		return Cgroup{}, errors.New("cgroup_new_cgroup")
	}
	defer C.cgroup_free(&cg)

	// Set memory restrictions. We limit both ram and ram+swap to the same
	// amount to prevent swapping.
	param = C.CString("memory")
	cgController := C.cgroup_add_controller(cg, param)

	mem := C.int64_t(memsize)
	param = C.CString("memory.limit_in_bytes")
	C.cgroup_add_value_int64(cgController, param, mem)
	param = C.CString("memory.memsw.limit_in_bytes")
	C.cgroup_add_value_int64(cgController, param, mem)

	// Setup CPU restrictions. The task is pinned to a specific set of CPUs
	// if a non-empty string is passed.
	if len(cpuset) > 0 {
		param = C.CString("cpuset")
		cgController = C.cgroup_add_controller(cg, param)

		var arg *C.char
		param = C.CString("cpuset.mems")
		arg = C.CString("0")
		C.cgroup_add_value_string(cgController, param, arg)
		param = C.CString("cpuset.cpus")
		arg = C.CString(cpuset)
		C.cgroup_add_value_string(cgController, param, arg)
	}

	// Perform the creation of the cgroup.
	if ret := C.cgroup_create_cgroup(cg, 1); ret != 0 {
		err := fmt.Errorf("Creating cgroup: %s(%d)",
			C.cgroup_strerror(ret), ret)
		return Cgroup{}, err
	}
	return Cgroup{cgroupname, cpuset, memsize}, nil
}

// Attach attaches a task with PID p to the cgroup.
func (g *Cgroup) Attach(p int) error {
	var param *C.char

	param = C.CString(g.name)
	cg := C.cgroup_new_cgroup(param)
	if cg != nil {
		return errors.New("cgroup_new_cgroup")
	}
	defer C.cgroup_free(&cg)

	if ret := C.cgroup_get_cgroup(cg); ret != 0 {
		err := fmt.Errorf("Get cgroup information: %s(d)",
			C.cgroup_strerror(ret), ret)
		return err
	}

	// Attach task to the cgroup
	if ret := C.cgroup_attach_task_pid(cg, C.pid_t(p)); ret != 0 {
		err := fmt.Errorf("Attach task to cgroup: %s(%d)",
			C.cgroup_strerror(ret), ret)
		return err
	}

	return nil
}
<commit_msg>cgroup: implement kill and destroy functionality for cleaning up processes.<commit_after>//
// Copyright (c) 2016 Dennis Chen
//
// This file is part of Clashr.
//
// Clashr is free software: you can redistribute it and/or modify it under the
// terms of the GNU Affero General Public License as published by the Free
// Software Foundation, either version 3 of the License, or (at your option)
// any later version.
//
// Clashr is distributed in the hope that it will be useful, but WITHOUT ANY
// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
// FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for
// more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with Clashr.  If not, see <http://www.gnu.org/licenses/>.

// Package cgroup provides a large amount of functionality for properly
// implementing cgroups in the judging and tracking of programs in Clashr.
// Most of this work is inspired by the DOMJudge implementation runguard.c and
// attempts to mimic a majoritiy of it's behavior
//
package cgroup

// #cgo pkg-config: libcgroup
// #include <libcgroup.h>
import "C"

import (
	"errors"
	"fmt"
	"syscall"
	"unsafe"
)

// A uniform interface for the cgroup actions.
type Cgroup struct {
	name string
	cpus string
	mems int64
}

// OutputStats returns the memory used (in bytes) by a process controlled by a
// cgroup.
func (g *Cgroup) OutputStats() (int64, error) {
	var param *C.char

	param = C.CString(g.name)
	cg := C.cgroup_new_cgroup(param)
	if cg == nil {
		return 0, errors.New("cgroup_new_cgroup")
	}
	defer C.cgroup_free(&cg)

	if ret := C.cgroup_get_cgroup(cg); ret != 0 {
		err := fmt.Errorf("Get cgroup information: %s(d)",
			C.cgroup_strerror(ret), ret)
		return 0, err
	}
	param = C.CString("memory")
	cgController := C.cgroup_get_controller(cg, param)

	var maxUsage C.int64_t
	param = C.CString("memory.memsw.max_usage_in_bytes")
	if ret := C.cgroup_get_value_int64(cgController, param, &maxUsage); ret != 0 {
		err := fmt.Errorf("Get cgroup value: %s(%d)",
			C.cgroup_strerror(ret), ret)
		return 0, err
	}

	return int64(maxUsage), nil
}

// Create makes a new instance of the cgroup interface. It sets the memory
// limit of the cgroup to memsize, and the set of CPUs to use to cpuset. If
// memory does not need to be limited, set memsize to syscall.RLIMIT_INFINITY.
// Similarly, set cpuset to the empty string if CPUs do not need to be set.
func Create(cgroupname, cpuset string, memsize int64) (Cgroup, error) {
	var param *C.char

	param = C.CString(cgroupname)
	cg := C.cgroup_new_cgroup(param)
	if cg == nil {
		return Cgroup{}, errors.New("cgroup_new_cgroup")
	}
	defer C.cgroup_free(&cg)

	// Set memory restrictions. We limit both ram and ram+swap to the same
	// amount to prevent swapping.
	param = C.CString("memory")
	cgController := C.cgroup_add_controller(cg, param)

	mem := C.int64_t(memsize)
	param = C.CString("memory.limit_in_bytes")
	C.cgroup_add_value_int64(cgController, param, mem)
	param = C.CString("memory.memsw.limit_in_bytes")
	C.cgroup_add_value_int64(cgController, param, mem)

	// Setup CPU restrictions. The task is pinned to a specific set of CPUs
	// if a non-empty string is passed.
	if len(cpuset) > 0 {
		param = C.CString("cpuset")
		cgController = C.cgroup_add_controller(cg, param)

		var arg *C.char
		param = C.CString("cpuset.mems")
		arg = C.CString("0")
		C.cgroup_add_value_string(cgController, param, arg)
		param = C.CString("cpuset.cpus")
		arg = C.CString(cpuset)
		C.cgroup_add_value_string(cgController, param, arg)
	}

	// Perform the creation of the cgroup.
	if ret := C.cgroup_create_cgroup(cg, 1); ret != 0 {
		err := fmt.Errorf("Creating cgroup: %s(%d)",
			C.cgroup_strerror(ret), ret)
		return Cgroup{}, err
	}
	return Cgroup{cgroupname, cpuset, memsize}, nil
}

// Attach attaches a task with PID p to the cgroup.
func (g *Cgroup) Attach(p int) error {
	var param *C.char

	param = C.CString(g.name)
	cg := C.cgroup_new_cgroup(param)
	if cg != nil {
		return errors.New("cgroup_new_cgroup")
	}
	defer C.cgroup_free(&cg)

	if ret := C.cgroup_get_cgroup(cg); ret != 0 {
		err := fmt.Errorf("Get cgroup information: %s(d)",
			C.cgroup_strerror(ret), ret)
		return err
	}

	// Attach task to the cgroup
	if ret := C.cgroup_attach_task_pid(cg, C.pid_t(p)); ret != 0 {
		err := fmt.Errorf("Attach task to cgroup: %s(%d)",
			C.cgroup_strerror(ret), ret)
		return err
	}

	return nil
}

// Kill sends SIGKILL(9) to all PIDs associated with the given cgroup.
func (g *Cgroup) Kill() error {
	cgName := C.CString(g.name)
	cgCtrl := C.CString("memory")

	var pid C.pid_t
	var handle unsafe.Pointer

	for {
		ret := C.cgroup_get_task_begin(cgName, cgCtrl, &handle, &pid)
		C.cgroup_get_task_end(&handle)
		if ret == C.ECGEOF {
			break
		}
		if err := syscall.Kill(int(pid), syscall.SIGKILL); err != nil {
			return err
		}
	}
	return nil
}

// Delete removes the cgroup from the kernel.
func (g *Cgroup) Delete() error {
	var param *C.char

	param = C.CString(g.name)
	cg := C.cgroup_new_cgroup(param)
	if cg == nil {
		err := fmt.Errorf("cgroup_new_cgroup")
		return err
	}

	if ret := C.cgroup_get_cgroup(cg); ret != 0 {
		err := fmt.Errorf("Get cgroup information: %s(%d)",
			C.cgroup_strerror(ret), ret)
		return err
	}

	// Clean up the cgroup
	if ret := C.cgroup_delete_cgroup(cg, 1); ret != 0 {
		err := fmt.Errorf("Deleteing cgroup: %s(%d)",
			C.cgroup_strerror(ret), ret)
		return err
	}
	return nil
}
<|endoftext|>