<commit_before>package boober

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/skatteetaten/aoc/pkg/openshift"
	"github.com/spf13/viper"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	//"github.com/vbatts/gogololcat/lol"
)

const SPEC_ILLEGAL = -1
const MISSING_FILE_REFERENCE = -2
const MISSING_CONFIGURATION = -3
const ILLEGAL_JSON_CONFIGURATION = -4
const CONFIGURATION_FILE_IN_ROOT = -5
const ERROR_READING_FILE = -6
const BOOBER_ERROR = -7
const ILLEGAL_FILE = -8
const INTERNAL_ERROR = -9
const FOLDER_SETUP_NOT_SUPPORTED = -10
const OPERATION_OK = 0
const SPEC_IS_FILE = 1
const SPEC_IS_FOLDER = 2

const BOOBER_NOT_INSTALLED_RESPONSE = "Application is not available"

// Struct to represent data to the Boober interface
type BooberInferface struct {
	Env         string                     `json:"env"`
	App         string                     `json:"app"`
	Affiliation string                     `json:"affiliation"`
	Files       map[string]json.RawMessage `json:"files"`
	Overrides   map[string]json.RawMessage `json:"overrides"`
}

// Structs to represent return data from the Boober interface
type BooberReturn struct {
	Sources          json.RawMessage            `json:"sources"`
	Errors           []string                   `json:"errors"`
	Valid            bool                       `json:"valid"`
	Config           json.RawMessage            `json:"config"`
	OpenshiftObjects map[string]json.RawMessage `json:"openshiftObjects"`
}

func ExecuteSetup(args []string, dryRun bool, showConfig bool, showObjects bool, verbose bool, localhost bool,
	overrideFiles []string) int {

	validateCode := validateCommand(args, overrideFiles)
	if validateCode < 0 {
		return validateCode
	}

	var absolutePath string

	absolutePath, _ = filepath.Abs(args[0])

	var envFile string      // Filename for app
	var envFolder string    // Short folder name (Env)
	var folder string       // Absolute path of folder
	var parentFolder string // Absolute path of parent

	switch validateCode {
	case SPEC_IS_FILE:
		folder = filepath.Dir(absolutePath)
		envFile = filepath.Base(absolutePath)
	case SPEC_IS_FOLDER:
		folder = absolutePath
		envFile = ""
		fmt.Println("Setup on a folder is not yet supported")
		return FOLDER_SETUP_NOT_SUPPORTED
	}

	parentFolder = filepath.Dir(folder)
	envFolder = filepath.Base(folder)

	if folder == parentFolder {
		fmt.Println("Application configuration file cannot reside in root directory")
		return CONFIGURATION_FILE_IN_ROOT
	}

	// Initialize JSON

	var booberData BooberInferface
	booberData.App = strings.TrimSuffix(envFile, filepath.Ext(envFile)) //envFile
	booberData.Env = envFolder
	booberData.Affiliation = ""

	var returnMap = Folder2Map(folder, envFolder+"/")
	var returnMap2 = Folder2Map(parentFolder, "")

	booberData.Files = CombineMaps(returnMap, returnMap2)
	booberData.Overrides = overrides2map(args, overrideFiles)

	jsonByte, ok := json.Marshal(booberData)
	if !(ok == nil) {
		fmt.Println("Internal error in marshalling Boober data: " + ok.Error())
		return INTERNAL_ERROR
	}

	jsonStr := string(jsonByte)
	if dryRun {
		//lol.Writer.Write(os.Stdout, []byte(PrettyPrintJson(jsonStr)))
		fmt.Println(string(PrettyPrintJson(jsonStr)))
	} else {
		validateCode = CallBoober(jsonStr, showConfig, showObjects, false, localhost, verbose)
		if validateCode < 0 {
			return validateCode
		}
	}
	return OPERATION_OK
}

func overrides2map(args []string, overrideFiles []string) map[string]json.RawMessage {
	var returnMap = make(map[string]json.RawMessage)
	for i := 0; i < len(overrideFiles); i++ {
		returnMap[overrideFiles[i]] = json.RawMessage(args[i+1])
	}
	return returnMap
}

func Folder2Map(folder string, prefix string) map[string]json.RawMessage {
	var returnMap = make(map[string]json.RawMessage)
	var allFilesOK bool = true

	files, _ := ioutil.ReadDir(folder)
	var filesProcessed = 0
	for _, f := range files {
		absolutePath := filepath.Join(folder, f.Name())
		if IsLegalFileFolder(absolutePath) == SPEC_IS_FILE { // Ignore folders
			matched, _ := filepath.Match("*.json", strings.ToLower(f.Name()))
			if matched {
				fileJson, err := ioutil.ReadFile(absolutePath)
				if err != nil {
					fmt.Println("Error in reading file " + absolutePath)
					os.Exit(ERROR_READING_FILE)
				}
				if IsLegalJson(string(fileJson)) {
					filesProcessed++
					returnMap[prefix+f.Name()] = fileJson
				} else {
					fmt.Println("Illegal JSON in configuration file " + absolutePath)
					allFilesOK = false
				}
				filesProcessed++
			}
		}

	}
	if !allFilesOK {
		os.Exit(ILLEGAL_JSON_CONFIGURATION)
	}
	return returnMap
}

func CombineMaps(map1 map[string]json.RawMessage, map2 map[string]json.RawMessage) map[string]json.RawMessage {
	var returnMap = make(map[string]json.RawMessage)

	for k, v := range map1 {
		returnMap[k] = v
	}
	for k, v := range map2 {
		returnMap[k] = v
	}
	return returnMap
}

func validateCommand(args []string, overrideFiles []string) int {
	var errorString = ""
	var returnCode int

	if len(args) == 0 {
		returnCode = -1
		errorString += "Missing file/folder "
	} else {
		// Chceck argument 0 for legal file / folder
		returnCode = IsLegalFileFolder(args[0])
		if returnCode < 0 {
			errorString += "Illegal file / folder: " + args[0]
			returnCode = ILLEGAL_FILE
		}

		// We have at least one argument, now there should be a correlation between the number of args
		// and the number of override (-f) flags
		if len(overrideFiles) < (len(args) - 1) {
			returnCode = MISSING_FILE_REFERENCE
			errorString += "Configuration override specified without file reference flag "
		}
		if len(overrideFiles) > (len(args) - 1) {
			returnCode = MISSING_CONFIGURATION
			errorString += "Configuration overide file reference flag specified without configuration "
		}

		// Check for legal JSON argument for each overrideFiles flag
		for i := 1; i < len(args); i++ {
			if !IsLegalJson(args[i]) {
				errorString = "Illegal JSON configuration override: " + args[i] + " "
				returnCode = ILLEGAL_JSON_CONFIGURATION
			}
		}
	}

	if returnCode < 0 {
		fmt.Println(errorString)
	}
	return returnCode
}

func IsLegalFileFolder(filespec string) int {
	var err error
	var absolutePath string
	var fi os.FileInfo

	absolutePath, err = filepath.Abs(filespec)
	fi, err = os.Stat(absolutePath)
	if os.IsNotExist(err) {
		return SPEC_ILLEGAL
	} else {
		switch mode := fi.Mode(); {
		case mode.IsDir():
			return SPEC_IS_FOLDER
		case mode.IsRegular():
			return SPEC_IS_FILE
		}
	}
	return SPEC_ILLEGAL
}

func GetBooberAddress(clusterName string, localhost bool) string {
	var booberAddress string

	if localhost {
		booberAddress = "http://localhost:8080"
	} else {
		booberAddress = "http://boober-mfp-boober." + clusterName + ".paas.skead.no"
	}
	return booberAddress
}

func GetBooberSetupUrl(clusterName string, localhost bool) string {
	return GetBooberAddress(clusterName, localhost) + "/setup"
}

func CallBoober(combindedJson string, showConfig bool, showObjects bool, api bool, localhost bool, verbose bool) int {
	//var openshiftConfig *openshift.OpenshiftConfig
	var configLocation = viper.GetString("HOME") + "/.aoc.json"

	if localhost {
		CallBooberInstance(combindedJson, showConfig, showObjects, verbose,
			GetBooberSetupUrl("localhost", localhost), "")
	} else {
		openshiftConfig, err := openshift.LoadOrInitiateConfigFile(configLocation)
		if err != nil {
			fmt.Println("Error in loading OpenShift configuration")
			return INTERNAL_ERROR
		}

		for i := range openshiftConfig.Clusters {
			if openshiftConfig.Clusters[i].Reachable {
				if !api || openshiftConfig.Clusters[i].Name == openshiftConfig.APICluster {
					CallBooberInstance(combindedJson, showConfig, showObjects, verbose,
						GetBooberSetupUrl(openshiftConfig.Clusters[i].Name, localhost),
						openshiftConfig.Clusters[i].Token)
				}
			}
		}
	}
	return OPERATION_OK
}

func CallBooberInstance(combindedJson string, showConfig bool, showObjects bool, verbose bool, url string, token string) int {

	if verbose {
		fmt.Print("Sending config to Boober at " + url + "... ")
	}

	var jsonStr = []byte(combindedJson)

	req, err := http.NewRequest(http.MethodPut, url, bytes.NewBuffer(jsonStr))
	req.Header.Set("Content-Type", "application/json")
	if err != nil {
		fmt.Println("Internal error in NewRequest: ", err)
		return INTERNAL_ERROR
	}

	req.Header.Set("Authentication", "Bearer: "+token)
	client := &http.Client{}

	resp, err := client.Do(req)
	if err != nil {
		fmt.Println("Error connecting to the Boober service on "+url+": ", err)
		return BOOBER_ERROR
	}

	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		body, _ := ioutil.ReadAll(resp.Body)
		bodyStr := string(body)
		if strings.Contains(bodyStr, BOOBER_NOT_INSTALLED_RESPONSE) {
			fmt.Println("FAIL: Boober not available on " + url + ":")
		}
		return BOOBER_ERROR
	}
	body, _ := ioutil.ReadAll(resp.Body)

	// Check return for error
	var booberReturn BooberReturn
	err = json.Unmarshal(body, &booberReturn)
	if err != nil {
		fmt.Println("Error unmashalling Boober return: " + err.Error())
		return BOOBER_ERROR
	}

	if !(booberReturn.Valid) {
		fmt.Println("Error in configuration: ")
		for _, message := range booberReturn.Errors {
			fmt.Println("  " + message)
		}
	} else {
		if verbose {
			fmt.Print("OK ")
			var countMap map[string]int = make(map[string]int)
			for key := range booberReturn.OpenshiftObjects {
				countMap[key]++
			}
			fmt.Print("(")
			for key := range countMap {
				fmt.Printf("%v: %v  ", key, countMap[key])
			}
			fmt.Println(")")
		}
	}

	if showConfig {
		fmt.Println(PrettyPrintJson(string(booberReturn.Config)))
	}

	if showObjects {
		for key := range booberReturn.OpenshiftObjects {
			fmt.Println(key)
			fmt.Println(PrettyPrintJson(string(booberReturn.OpenshiftObjects[key])))
		}
		//fmt.Println(PrettyPrintJson(string(booberReturn.OpenshiftObjects)))
	}

	return OPERATION_OK
}
<commit_msg>Localhost flag now sends the token of the API server (first reachable instance) as the bearer token to Boober<commit_after>package boober

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/skatteetaten/aoc/pkg/openshift"
	"github.com/spf13/viper"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	//"github.com/vbatts/gogololcat/lol"
)

const SPEC_ILLEGAL = -1
const MISSING_FILE_REFERENCE = -2
const MISSING_CONFIGURATION = -3
const ILLEGAL_JSON_CONFIGURATION = -4
const CONFIGURATION_FILE_IN_ROOT = -5
const ERROR_READING_FILE = -6
const BOOBER_ERROR = -7
const ILLEGAL_FILE = -8
const INTERNAL_ERROR = -9
const FOLDER_SETUP_NOT_SUPPORTED = -10
const OPERATION_OK = 0
const SPEC_IS_FILE = 1
const SPEC_IS_FOLDER = 2

const BOOBER_NOT_INSTALLED_RESPONSE = "Application is not available"

// Struct to represent data to the Boober interface
type BooberInferface struct {
	Env         string                     `json:"env"`
	App         string                     `json:"app"`
	Affiliation string                     `json:"affiliation"`
	Files       map[string]json.RawMessage `json:"files"`
	Overrides   map[string]json.RawMessage `json:"overrides"`
}

// Structs to represent return data from the Boober interface
type BooberReturn struct {
	Sources          json.RawMessage            `json:"sources"`
	Errors           []string                   `json:"errors"`
	Valid            bool                       `json:"valid"`
	Config           json.RawMessage            `json:"config"`
	OpenshiftObjects map[string]json.RawMessage `json:"openshiftObjects"`
}

func ExecuteSetup(args []string, dryRun bool, showConfig bool, showObjects bool, verbose bool, localhost bool,
	overrideFiles []string) int {

	validateCode := validateCommand(args, overrideFiles)
	if validateCode < 0 {
		return validateCode
	}

	var absolutePath string

	absolutePath, _ = filepath.Abs(args[0])

	var envFile string      // Filename for app
	var envFolder string    // Short folder name (Env)
	var folder string       // Absolute path of folder
	var parentFolder string // Absolute path of parent

	switch validateCode {
	case SPEC_IS_FILE:
		folder = filepath.Dir(absolutePath)
		envFile = filepath.Base(absolutePath)
	case SPEC_IS_FOLDER:
		folder = absolutePath
		envFile = ""
		fmt.Println("Setup on a folder is not yet supported")
		return FOLDER_SETUP_NOT_SUPPORTED
	}

	parentFolder = filepath.Dir(folder)
	envFolder = filepath.Base(folder)

	if folder == parentFolder {
		fmt.Println("Application configuration file cannot reside in root directory")
		return CONFIGURATION_FILE_IN_ROOT
	}

	// Initialize JSON

	var booberData BooberInferface
	booberData.App = strings.TrimSuffix(envFile, filepath.Ext(envFile)) //envFile
	booberData.Env = envFolder
	booberData.Affiliation = ""

	var returnMap = Folder2Map(folder, envFolder+"/")
	var returnMap2 = Folder2Map(parentFolder, "")

	booberData.Files = CombineMaps(returnMap, returnMap2)
	booberData.Overrides = overrides2map(args, overrideFiles)

	jsonByte, ok := json.Marshal(booberData)
	if !(ok == nil) {
		fmt.Println("Internal error in marshalling Boober data: " + ok.Error())
		return INTERNAL_ERROR
	}

	jsonStr := string(jsonByte)
	if dryRun {
		//lol.Writer.Write(os.Stdout, []byte(PrettyPrintJson(jsonStr)))
		fmt.Println(string(PrettyPrintJson(jsonStr)))
	} else {
		validateCode = CallBoober(jsonStr, showConfig, showObjects, false, localhost, verbose)
		if validateCode < 0 {
			return validateCode
		}
	}
	return OPERATION_OK
}

func overrides2map(args []string, overrideFiles []string) map[string]json.RawMessage {
	var returnMap = make(map[string]json.RawMessage)
	for i := 0; i < len(overrideFiles); i++ {
		returnMap[overrideFiles[i]] = json.RawMessage(args[i+1])
	}
	return returnMap
}

func Folder2Map(folder string, prefix string) map[string]json.RawMessage {
	var returnMap = make(map[string]json.RawMessage)
	var allFilesOK bool = true

	files, _ := ioutil.ReadDir(folder)
	var filesProcessed = 0
	for _, f := range files {
		absolutePath := filepath.Join(folder, f.Name())
		if IsLegalFileFolder(absolutePath) == SPEC_IS_FILE { // Ignore folders
			matched, _ := filepath.Match("*.json", strings.ToLower(f.Name()))
			if matched {
				fileJson, err := ioutil.ReadFile(absolutePath)
				if err != nil {
					fmt.Println("Error in reading file " + absolutePath)
					os.Exit(ERROR_READING_FILE)
				}
				if IsLegalJson(string(fileJson)) {
					filesProcessed++
					returnMap[prefix+f.Name()] = fileJson
				} else {
					fmt.Println("Illegal JSON in configuration file " + absolutePath)
					allFilesOK = false
				}
				filesProcessed++
			}
		}

	}
	if !allFilesOK {
		os.Exit(ILLEGAL_JSON_CONFIGURATION)
	}
	return returnMap
}

func CombineMaps(map1 map[string]json.RawMessage, map2 map[string]json.RawMessage) map[string]json.RawMessage {
	var returnMap = make(map[string]json.RawMessage)

	for k, v := range map1 {
		returnMap[k] = v
	}
	for k, v := range map2 {
		returnMap[k] = v
	}
	return returnMap
}

func validateCommand(args []string, overrideFiles []string) int {
	var errorString = ""
	var returnCode int

	if len(args) == 0 {
		returnCode = -1
		errorString += "Missing file/folder "
	} else {
		// Chceck argument 0 for legal file / folder
		returnCode = IsLegalFileFolder(args[0])
		if returnCode < 0 {
			errorString += "Illegal file / folder: " + args[0]
			returnCode = ILLEGAL_FILE
		}

		// We have at least one argument, now there should be a correlation between the number of args
		// and the number of override (-f) flags
		if len(overrideFiles) < (len(args) - 1) {
			returnCode = MISSING_FILE_REFERENCE
			errorString += "Configuration override specified without file reference flag "
		}
		if len(overrideFiles) > (len(args) - 1) {
			returnCode = MISSING_CONFIGURATION
			errorString += "Configuration overide file reference flag specified without configuration "
		}

		// Check for legal JSON argument for each overrideFiles flag
		for i := 1; i < len(args); i++ {
			if !IsLegalJson(args[i]) {
				errorString = "Illegal JSON configuration override: " + args[i] + " "
				returnCode = ILLEGAL_JSON_CONFIGURATION
			}
		}
	}

	if returnCode < 0 {
		fmt.Println(errorString)
	}
	return returnCode
}

func IsLegalFileFolder(filespec string) int {
	var err error
	var absolutePath string
	var fi os.FileInfo

	absolutePath, err = filepath.Abs(filespec)
	fi, err = os.Stat(absolutePath)
	if os.IsNotExist(err) {
		return SPEC_ILLEGAL
	} else {
		switch mode := fi.Mode(); {
		case mode.IsDir():
			return SPEC_IS_FOLDER
		case mode.IsRegular():
			return SPEC_IS_FILE
		}
	}
	return SPEC_ILLEGAL
}

func GetBooberAddress(clusterName string, localhost bool) string {
	var booberAddress string

	if localhost {
		booberAddress = "http://localhost:8080"
	} else {
		booberAddress = "http://boober-mfp-boober." + clusterName + ".paas.skead.no"
	}
	return booberAddress
}

func GetBooberSetupUrl(clusterName string, localhost bool) string {
	return GetBooberAddress(clusterName, localhost) + "/setup"
}

func GetApiCluster() *openshift.OpenshiftCluster {
	var configLocation = viper.GetString("HOME") + "/.aoc.json"
	openshiftConfig, err := openshift.LoadOrInitiateConfigFile(configLocation)
	if err != nil {
		fmt.Println("Error in loading OpenShift configuration")
		return nil
	}
	for i := range openshiftConfig.Clusters {
		if openshiftConfig.Clusters[i].Reachable {
			return openshiftConfig.Clusters[i]
		}
	}
	return nil
}

func CallBoober(combindedJson string, showConfig bool, showObjects bool, api bool, localhost bool, verbose bool) int {
	//var openshiftConfig *openshift.OpenshiftConfig
	var configLocation = viper.GetString("HOME") + "/.aoc.json"

	if localhost {
		var token string = ""
		apiCluster := GetApiCluster()
		if apiCluster != nil {
			token = apiCluster.Token
		}
		CallBooberInstance(combindedJson, showConfig, showObjects, verbose,
			GetBooberSetupUrl("localhost", localhost), token)
	} else {
		openshiftConfig, err := openshift.LoadOrInitiateConfigFile(configLocation)
		if err != nil {
			fmt.Println("Error in loading OpenShift configuration")
			return INTERNAL_ERROR
		}

		for i := range openshiftConfig.Clusters {
			if openshiftConfig.Clusters[i].Reachable {
				if !api || openshiftConfig.Clusters[i].Name == openshiftConfig.APICluster {
					CallBooberInstance(combindedJson, showConfig, showObjects, verbose,
						GetBooberSetupUrl(openshiftConfig.Clusters[i].Name, localhost),
						openshiftConfig.Clusters[i].Token)
				}
			}
		}
	}
	return OPERATION_OK
}

func CallBooberInstance(combindedJson string, showConfig bool, showObjects bool, verbose bool, url string, token string) int {

	if verbose {
		fmt.Print("Sending config to Boober at " + url + "... ")
	}

	var jsonStr = []byte(combindedJson)

	req, err := http.NewRequest(http.MethodPut, url, bytes.NewBuffer(jsonStr))
	req.Header.Set("Content-Type", "application/json")
	if err != nil {
		fmt.Println("Internal error in NewRequest: ", err)
		return INTERNAL_ERROR
	}

	req.Header.Set("Authentication", "Bearer: "+token)
	client := &http.Client{}

	resp, err := client.Do(req)
	if err != nil {
		fmt.Println("Error connecting to the Boober service on "+url+": ", err)
		return BOOBER_ERROR
	}

	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		body, _ := ioutil.ReadAll(resp.Body)
		bodyStr := string(body)
		if strings.Contains(bodyStr, BOOBER_NOT_INSTALLED_RESPONSE) {
			fmt.Println("FAIL: Boober not available on " + url + ":")
		}
		return BOOBER_ERROR
	}
	body, _ := ioutil.ReadAll(resp.Body)

	// Check return for error
	var booberReturn BooberReturn
	err = json.Unmarshal(body, &booberReturn)
	if err != nil {
		fmt.Println("Error unmashalling Boober return: " + err.Error())
		return BOOBER_ERROR
	}

	if !(booberReturn.Valid) {
		fmt.Println("Error in configuration: ")
		for _, message := range booberReturn.Errors {
			fmt.Println("  " + message)
		}
	} else {
		if verbose {
			fmt.Print("OK ")
			var countMap map[string]int = make(map[string]int)
			for key := range booberReturn.OpenshiftObjects {
				countMap[key]++
			}
			fmt.Print("(")
			for key := range countMap {
				fmt.Printf("%v: %v  ", key, countMap[key])
			}
			fmt.Println(")")
		}
	}

	if showConfig {
		fmt.Println(PrettyPrintJson(string(booberReturn.Config)))
	}

	if showObjects {
		for key := range booberReturn.OpenshiftObjects {
			fmt.Println(key)
			fmt.Println(PrettyPrintJson(string(booberReturn.OpenshiftObjects[key])))
		}
		//fmt.Println(PrettyPrintJson(string(booberReturn.OpenshiftObjects)))
	}

	return OPERATION_OK
}
<|endoftext|>


<commit_before>package consts

import (
	"github.com/cozy/cozy-stack/pkg/couchdb"
)

// Instances doc type for User's instance document
const Instances = "instances"

const (
	// Apps doc type for client-side application manifests
	Apps = "io.cozy.apps"
	// Konnectors doc type for konnector application manifests
	Konnectors = "io.cozy.konnectors"
	// KonnectorResults doc type for konnector last execution result.
	KonnectorResults = "io.cozy.konnectors.result"
	// Archives doc type for zip archives with files and directories
	Archives = "io.cozy.files.archives"
	// Doctypes doc type for doctype list
	Doctypes = "io.cozy.doctypes"
	// Files doc type for type for files and directories
	Files = "io.cozy.files"
	// Intents doc type for intents persisted in couchdb
	Intents = "io.cozy.intents"
	// Jobs doc type for queued jobs
	Jobs = "io.cozy.jobs"
	// JobEvents doc type for realt time events sent by jobs
	JobEvents = "io.cozy.jobs.events"
	// OAuthAccessCodes doc type for OAuth2 access codes
	OAuthAccessCodes = "io.cozy.oauth.access_codes"
	// OAuthClients doc type for OAuth2 clients
	OAuthClients = "io.cozy.oauth.clients"
	// Permissions doc type for permissions identifying a connection
	Permissions = "io.cozy.permissions"
	// Queues doc type for jobs queues
	Queues = "io.cozy.queues"
	// Recipients doc type for sharing recipients
	Recipients = "io.cozy.recipients"
	// RemoteRequests doc type for logging requests to remote websites
	RemoteRequests = "io.cozy.remote.requests"
	// Sessions doc type for sessions identifying a connection
	Sessions = "io.cozy.sessions"
	// Settings doc type for settings to customize an instance
	Settings = "io.cozy.settings"
	// Sharings doc type for document and file sharing
	Sharings = "io.cozy.sharings"
	// Triggers doc type for triggers, jobs launchers
	Triggers = "io.cozy.triggers"
	// Accounts doc type for accounts
	Accounts = "io.cozy.accounts"
	// AccountTypes doc type for account types
	AccountTypes = "io.cozy.account_types"
)

const (
	// DriveSlug is the slug of the default app, files, where the user is
	// redirected after login.
	DriveSlug = "drive"
	// OnboardingSlug is the slug of the onboarding app, where the user is
	// redirected when he has no passphrase.
	OnboardingSlug = "onboarding"
	// StoreSlug is the slug of the only app that can install other apps.
	// FIXME update me when the store app will be available
	StoreSlug = "settings"
	// CollectSlug is the slug of the only app that can install konnectors.
	CollectSlug = "collect"
)

const (
	// DirType is the type attribute for directories
	DirType = "directory"
	// FileType is the type attribute for files
	FileType = "file"
)

const (
	// RootDirID is the root directory identifier
	RootDirID = "io.cozy.files.root-dir"
	// TrashDirID is the trash directory identifier
	TrashDirID = "io.cozy.files.trash-dir"
)

const (
	// ContextSettingsID is the id of the settings JSON-API response for the context
	ContextSettingsID = "io.cozy.settings.context"
	// DiskUsageID is the id of the settings JSON-API response for disk-usage
	DiskUsageID = "io.cozy.settings.disk-usage"
	// InstanceSettingsID is the id of settings document for the instance
	InstanceSettingsID = "io.cozy.settings.instance"
	// SharingSettingsID is the id of the settings document for sharings.
	SharingSettingsID = "io.cozy.settings.sharings"
)

const (
	// OneShotSharing is a sharing with no continuous updates
	OneShotSharing = "one-shot"
	// MasterSlaveSharing is a sharing with unilateral continuous updates
	MasterSlaveSharing = "master-slave"
	// MasterMasterSharing is a sharing with bilateral continuous updates
	MasterMasterSharing = "master-master"
)

const (
	// SharingStatusPending is the sharing pending status
	SharingStatusPending = "pending"
	// SharingStatusRefused is the sharing refused status
	SharingStatusRefused = "refused"
	// SharingStatusAccepted is the sharing accepted status
	SharingStatusAccepted = "accepted"
	// SharingStatusError is when the request could not be sent
	SharingStatusError = "error"
	// SharingStatusUnregistered is when the sharer could not register herself
	// as an OAuth client at the recipient's
	SharingStatusUnregistered = "unregistered"
	// SharingStatusMailNotSent is when the mail invitation was not sent
	SharingStatusMailNotSent = "mail-not-sent"
	// SharingStatusRevoked is to tell if a recipient is revoked.
	SharingStatusRevoked = "revoked"
)

const (
	// WorkerTypeSharingUpdates is the string representation of the type of
	// workers that deals with updating sharings.
	WorkerTypeSharingUpdates = "sharingupdates"
)

const (
	// SelectorReferencedBy is the "referenced_by" selector.
	SelectorReferencedBy = couchdb.SelectorReferencedBy
)

const (
	// QueryParamRev is the key for the revision value in a query string. In
	// web/data the revision is expected as "rev", not "Rev".
	QueryParamRev = "rev"
	// QueryParamDirID is the key for the `DirID` field of a vfs.FileDoc or
	// vfs.DirDoc, in a query string.
	QueryParamDirID = "Dir_id"
	// QueryParamName is the key for the name value in a query string.
	QueryParamName = "Name"
	// QueryParamType is the key for the `type` value (file or directory) in
	// a query string.
	QueryParamType = "Type"
	// QueryParamExecutable is key for the `executable` field of a vfs.FileDoc
	// in a query string.
	QueryParamExecutable = "Executable"
	// QueryParamCreatedAt is the key for the `created_at` value in a query
	// string.
	QueryParamCreatedAt = "Created_at"
	// QueryParamUpdatedAt is the key for the `Updated_at` value in a query
	// string.
	QueryParamUpdatedAt = "Updated_at"
	// QueryParamRecursive is the key for the `recursive` value in a query
	// string.
	QueryParamRecursive = "Recursive"
	// QueryParamTags is the key for the `tags` values in a query string.
	QueryParamTags = "Tags"
	// QueryParamReferencedBy is the key for the `referenced_by` values in a
	// query string.
	QueryParamReferencedBy = "Referenced_by"
	// QueryParamSharer is used to tell if the user that received the query is
	// the sharer or not.
	QueryParamSharer = "Sharer"
	// QueryParamAppSlug is used to transmit the application slug in a query
	// string.
	QueryParamAppSlug = "App_slug"
	// QueryParamDocType is used to transmit the doctype in a query string.
	QueryParamDocType = "Doctype"
	// QueryParamSharingID is used to transmit the sharingID in a query string.
	QueryParamSharingID = "Sharing_id"
)

// AppsRegistry is an hard-coded list of known apps, with their source URLs
// TODO remove it when we will have a true registry
var AppsRegistry = map[string]string{
	"onboarding": "git://github.com/cozy/cozy-onboarding-v3.git#build",
	"drive":      "git://github.com/cozy/cozy-drive.git#build",
	"photos":     "git://github.com/cozy/cozy-photos-v3.git#build",
	"settings":   "git://github.com/cozy/cozy-settings.git#build",
	"collect":    "git://github.com/cozy/cozy-collect.git#build",
}
<commit_msg>Use contact doctype for sharing recipients<commit_after>package consts

import (
	"github.com/cozy/cozy-stack/pkg/couchdb"
)

// Instances doc type for User's instance document
const Instances = "instances"

const (
	// Apps doc type for client-side application manifests
	Apps = "io.cozy.apps"
	// Konnectors doc type for konnector application manifests
	Konnectors = "io.cozy.konnectors"
	// KonnectorResults doc type for konnector last execution result.
	KonnectorResults = "io.cozy.konnectors.result"
	// Archives doc type for zip archives with files and directories
	Archives = "io.cozy.files.archives"
	// Doctypes doc type for doctype list
	Doctypes = "io.cozy.doctypes"
	// Files doc type for type for files and directories
	Files = "io.cozy.files"
	// Intents doc type for intents persisted in couchdb
	Intents = "io.cozy.intents"
	// Jobs doc type for queued jobs
	Jobs = "io.cozy.jobs"
	// JobEvents doc type for realt time events sent by jobs
	JobEvents = "io.cozy.jobs.events"
	// OAuthAccessCodes doc type for OAuth2 access codes
	OAuthAccessCodes = "io.cozy.oauth.access_codes"
	// OAuthClients doc type for OAuth2 clients
	OAuthClients = "io.cozy.oauth.clients"
	// Permissions doc type for permissions identifying a connection
	Permissions = "io.cozy.permissions"
	// Queues doc type for jobs queues
	Queues = "io.cozy.queues"
	// Recipients doc type for sharing recipients
	Recipients = "io.cozy.contacts"
	// RemoteRequests doc type for logging requests to remote websites
	RemoteRequests = "io.cozy.remote.requests"
	// Sessions doc type for sessions identifying a connection
	Sessions = "io.cozy.sessions"
	// Settings doc type for settings to customize an instance
	Settings = "io.cozy.settings"
	// Sharings doc type for document and file sharing
	Sharings = "io.cozy.sharings"
	// Triggers doc type for triggers, jobs launchers
	Triggers = "io.cozy.triggers"
	// Accounts doc type for accounts
	Accounts = "io.cozy.accounts"
	// AccountTypes doc type for account types
	AccountTypes = "io.cozy.account_types"
)

const (
	// DriveSlug is the slug of the default app, files, where the user is
	// redirected after login.
	DriveSlug = "drive"
	// OnboardingSlug is the slug of the onboarding app, where the user is
	// redirected when he has no passphrase.
	OnboardingSlug = "onboarding"
	// StoreSlug is the slug of the only app that can install other apps.
	// FIXME update me when the store app will be available
	StoreSlug = "settings"
	// CollectSlug is the slug of the only app that can install konnectors.
	CollectSlug = "collect"
)

const (
	// DirType is the type attribute for directories
	DirType = "directory"
	// FileType is the type attribute for files
	FileType = "file"
)

const (
	// RootDirID is the root directory identifier
	RootDirID = "io.cozy.files.root-dir"
	// TrashDirID is the trash directory identifier
	TrashDirID = "io.cozy.files.trash-dir"
)

const (
	// ContextSettingsID is the id of the settings JSON-API response for the context
	ContextSettingsID = "io.cozy.settings.context"
	// DiskUsageID is the id of the settings JSON-API response for disk-usage
	DiskUsageID = "io.cozy.settings.disk-usage"
	// InstanceSettingsID is the id of settings document for the instance
	InstanceSettingsID = "io.cozy.settings.instance"
	// SharingSettingsID is the id of the settings document for sharings.
	SharingSettingsID = "io.cozy.settings.sharings"
)

const (
	// OneShotSharing is a sharing with no continuous updates
	OneShotSharing = "one-shot"
	// MasterSlaveSharing is a sharing with unilateral continuous updates
	MasterSlaveSharing = "master-slave"
	// MasterMasterSharing is a sharing with bilateral continuous updates
	MasterMasterSharing = "master-master"
)

const (
	// SharingStatusPending is the sharing pending status
	SharingStatusPending = "pending"
	// SharingStatusRefused is the sharing refused status
	SharingStatusRefused = "refused"
	// SharingStatusAccepted is the sharing accepted status
	SharingStatusAccepted = "accepted"
	// SharingStatusError is when the request could not be sent
	SharingStatusError = "error"
	// SharingStatusUnregistered is when the sharer could not register herself
	// as an OAuth client at the recipient's
	SharingStatusUnregistered = "unregistered"
	// SharingStatusMailNotSent is when the mail invitation was not sent
	SharingStatusMailNotSent = "mail-not-sent"
	// SharingStatusRevoked is to tell if a recipient is revoked.
	SharingStatusRevoked = "revoked"
)

const (
	// WorkerTypeSharingUpdates is the string representation of the type of
	// workers that deals with updating sharings.
	WorkerTypeSharingUpdates = "sharingupdates"
)

const (
	// SelectorReferencedBy is the "referenced_by" selector.
	SelectorReferencedBy = couchdb.SelectorReferencedBy
)

const (
	// QueryParamRev is the key for the revision value in a query string. In
	// web/data the revision is expected as "rev", not "Rev".
	QueryParamRev = "rev"
	// QueryParamDirID is the key for the `DirID` field of a vfs.FileDoc or
	// vfs.DirDoc, in a query string.
	QueryParamDirID = "Dir_id"
	// QueryParamName is the key for the name value in a query string.
	QueryParamName = "Name"
	// QueryParamType is the key for the `type` value (file or directory) in
	// a query string.
	QueryParamType = "Type"
	// QueryParamExecutable is key for the `executable` field of a vfs.FileDoc
	// in a query string.
	QueryParamExecutable = "Executable"
	// QueryParamCreatedAt is the key for the `created_at` value in a query
	// string.
	QueryParamCreatedAt = "Created_at"
	// QueryParamUpdatedAt is the key for the `Updated_at` value in a query
	// string.
	QueryParamUpdatedAt = "Updated_at"
	// QueryParamRecursive is the key for the `recursive` value in a query
	// string.
	QueryParamRecursive = "Recursive"
	// QueryParamTags is the key for the `tags` values in a query string.
	QueryParamTags = "Tags"
	// QueryParamReferencedBy is the key for the `referenced_by` values in a
	// query string.
	QueryParamReferencedBy = "Referenced_by"
	// QueryParamSharer is used to tell if the user that received the query is
	// the sharer or not.
	QueryParamSharer = "Sharer"
	// QueryParamAppSlug is used to transmit the application slug in a query
	// string.
	QueryParamAppSlug = "App_slug"
	// QueryParamDocType is used to transmit the doctype in a query string.
	QueryParamDocType = "Doctype"
	// QueryParamSharingID is used to transmit the sharingID in a query string.
	QueryParamSharingID = "Sharing_id"
)

// AppsRegistry is an hard-coded list of known apps, with their source URLs
// TODO remove it when we will have a true registry
var AppsRegistry = map[string]string{
	"onboarding": "git://github.com/cozy/cozy-onboarding-v3.git#build",
	"drive":      "git://github.com/cozy/cozy-drive.git#build",
	"photos":     "git://github.com/cozy/cozy-photos-v3.git#build",
	"settings":   "git://github.com/cozy/cozy-settings.git#build",
	"collect":    "git://github.com/cozy/cozy-collect.git#build",
}
<|endoftext|>


<commit_before>// Package daemon implements a service for mediating access to the data store,
// and its client.
//
// Most RPCs exposed by the service correspond to the methods of Store in the
// store package and are not documented here.
package daemon

import (
	"net"
	"os"
	"os/signal"
	"syscall"

	"src.elv.sh/pkg/daemon/internal/api"
	"src.elv.sh/pkg/logutil"
	"src.elv.sh/pkg/prog"
	"src.elv.sh/pkg/rpc"
	"src.elv.sh/pkg/store"
)

var logger = logutil.GetLogger("[daemon] ")

// Program is the daemon subprogram.
type Program struct {
	run   bool
	paths *prog.DaemonPaths
	// Used in tests.
	serveOpts ServeOpts
}

func (p *Program) RegisterFlags(fs *prog.FlagSet) {
	fs.BoolVar(&p.run, "daemon", false,
		"[internal flag] Run the storage daemon instead of an Elvish shell")
	p.paths = fs.DaemonPaths()
}

func (p *Program) Run(fds [3]*os.File, args []string) error {
	if !p.run {
		return prog.ErrNextProgram
	}
	if len(args) > 0 {
		return prog.BadUsage("arguments are not allowed with -daemon")
	}

	// The stdout is redirected to a unique log file (see the spawn function),
	// so just use it for logging.
	logutil.SetOutput(fds[1])
	setUmaskForDaemon()
	exit := Serve(p.paths.Sock, p.paths.DB, p.serveOpts)
	return prog.Exit(exit)
}

// ServeOpts keeps options that can be passed to Serve.
type ServeOpts struct {
	// If not nil, will be closed when the daemon is ready to serve requests.
	Ready chan<- struct{}
	// Causes the daemon to abort if closed or sent any date. If nil, Serve will
	// set up its own signal channel by listening to SIGINT and SIGTERM.
	Signals <-chan os.Signal
	// If not nil, overrides the response of the Version RPC.
	Version *int
}

// Serve runs the daemon service, listening on the socket specified by sockpath
// and serving data from dbpath until all clients have exited. See doc for
// ServeChans for additional options.
func Serve(sockpath, dbpath string, opts ServeOpts) int {
	logger.Println("pid is", syscall.Getpid())
	logger.Println("going to listen", sockpath)
	listener, err := net.Listen("unix", sockpath)
	if err != nil {
		logger.Printf("failed to listen on %s: %v", sockpath, err)
		logger.Println("aborting")
		return 2
	}

	st, err := store.NewStore(dbpath)
	if err != nil {
		logger.Printf("failed to create storage: %v", err)
		logger.Printf("serving anyway")
	}

	server := rpc.NewServer()
	version := api.Version
	if opts.Version != nil {
		version = *opts.Version
	}
	server.RegisterName(api.ServiceName, &service{version, st, err})

	connCh := make(chan net.Conn, 10)
	listenErrCh := make(chan error, 1)
	go func() {
		for {
			conn, err := listener.Accept()
			if err != nil {
				listenErrCh <- err
				close(listenErrCh)
				return
			}
			connCh <- conn
		}
	}()

	sigCh := opts.Signals
	if sigCh == nil {
		ch := make(chan os.Signal, 1)
		signal.Notify(ch, syscall.SIGTERM, syscall.SIGINT)
		sigCh = ch
	}

	conns := make(map[net.Conn]struct{})
	connDoneCh := make(chan net.Conn, 10)

	interrupt := func() {
		if len(conns) == 0 {
			logger.Println("exiting since there are no clients")
		}
		logger.Printf("going to close %v active connections", len(conns))
		for conn := range conns {
			err := conn.Close()
			if err != nil {
				logger.Println("failed to close connection:", err)
			}
		}
	}

	if opts.Ready != nil {
		close(opts.Ready)
	}

loop:
	for {
		select {
		case sig := <-sigCh:
			logger.Printf("received signal %v", sig)
			interrupt()
			break loop
		case err := <-listenErrCh:
			logger.Println("could not listen:", err)
			if len(conns) == 0 {
				logger.Println("exiting since there are no clients")
				break loop
			}
			logger.Println("continuing to serve until all existing clients exit")
		case conn := <-connCh:
			conns[conn] = struct{}{}
			go func() {
				server.ServeConn(conn)
				connDoneCh <- conn
			}()
		case conn := <-connDoneCh:
			delete(conns, conn)
			if len(conns) == 0 {
				logger.Println("all clients disconnected, exiting")
				break loop
			}
		}
	}

	err = os.Remove(sockpath)
	if err != nil {
		logger.Printf("failed to remove socket %s: %v", sockpath, err)
	}
	if st != nil {
		err = st.Close()
		if err != nil {
			logger.Printf("failed to close storage: %v", err)
		}
	}
	err = listener.Close()
	if err != nil {
		logger.Printf("failed to close listener: %v", err)
	}
	// Ensure that the listener goroutine has exited before returning
	<-listenErrCh
	return 0
}
<commit_msg>pkg/daemon: Ignore error when closing connections on shutdown.<commit_after>// Package daemon implements a service for mediating access to the data store,
// and its client.
//
// Most RPCs exposed by the service correspond to the methods of Store in the
// store package and are not documented here.
package daemon

import (
	"net"
	"os"
	"os/signal"
	"syscall"

	"src.elv.sh/pkg/daemon/internal/api"
	"src.elv.sh/pkg/logutil"
	"src.elv.sh/pkg/prog"
	"src.elv.sh/pkg/rpc"
	"src.elv.sh/pkg/store"
)

var logger = logutil.GetLogger("[daemon] ")

// Program is the daemon subprogram.
type Program struct {
	run   bool
	paths *prog.DaemonPaths
	// Used in tests.
	serveOpts ServeOpts
}

func (p *Program) RegisterFlags(fs *prog.FlagSet) {
	fs.BoolVar(&p.run, "daemon", false,
		"[internal flag] Run the storage daemon instead of an Elvish shell")
	p.paths = fs.DaemonPaths()
}

func (p *Program) Run(fds [3]*os.File, args []string) error {
	if !p.run {
		return prog.ErrNextProgram
	}
	if len(args) > 0 {
		return prog.BadUsage("arguments are not allowed with -daemon")
	}

	// The stdout is redirected to a unique log file (see the spawn function),
	// so just use it for logging.
	logutil.SetOutput(fds[1])
	setUmaskForDaemon()
	exit := Serve(p.paths.Sock, p.paths.DB, p.serveOpts)
	return prog.Exit(exit)
}

// ServeOpts keeps options that can be passed to Serve.
type ServeOpts struct {
	// If not nil, will be closed when the daemon is ready to serve requests.
	Ready chan<- struct{}
	// Causes the daemon to abort if closed or sent any date. If nil, Serve will
	// set up its own signal channel by listening to SIGINT and SIGTERM.
	Signals <-chan os.Signal
	// If not nil, overrides the response of the Version RPC.
	Version *int
}

// Serve runs the daemon service, listening on the socket specified by sockpath
// and serving data from dbpath until all clients have exited. See doc for
// ServeChans for additional options.
func Serve(sockpath, dbpath string, opts ServeOpts) int {
	logger.Println("pid is", syscall.Getpid())
	logger.Println("going to listen", sockpath)
	listener, err := net.Listen("unix", sockpath)
	if err != nil {
		logger.Printf("failed to listen on %s: %v", sockpath, err)
		logger.Println("aborting")
		return 2
	}

	st, err := store.NewStore(dbpath)
	if err != nil {
		logger.Printf("failed to create storage: %v", err)
		logger.Printf("serving anyway")
	}

	server := rpc.NewServer()
	version := api.Version
	if opts.Version != nil {
		version = *opts.Version
	}
	server.RegisterName(api.ServiceName, &service{version, st, err})

	connCh := make(chan net.Conn, 10)
	listenErrCh := make(chan error, 1)
	go func() {
		for {
			conn, err := listener.Accept()
			if err != nil {
				listenErrCh <- err
				close(listenErrCh)
				return
			}
			connCh <- conn
		}
	}()

	sigCh := opts.Signals
	if sigCh == nil {
		ch := make(chan os.Signal, 1)
		signal.Notify(ch, syscall.SIGTERM, syscall.SIGINT)
		sigCh = ch
	}

	conns := make(map[net.Conn]struct{})
	connDoneCh := make(chan net.Conn, 10)

	interrupt := func() {
		if len(conns) == 0 {
			logger.Println("exiting since there are no clients")
		}
		logger.Printf("going to close %v active connections", len(conns))
		for conn := range conns {
			// Ignore the error - if we can't close the connection it's because
			// the client has closed it. There is nothing we can do anyway.
			conn.Close()
		}
	}

	if opts.Ready != nil {
		close(opts.Ready)
	}

loop:
	for {
		select {
		case sig := <-sigCh:
			logger.Printf("received signal %v", sig)
			interrupt()
			break loop
		case err := <-listenErrCh:
			logger.Println("could not listen:", err)
			if len(conns) == 0 {
				logger.Println("exiting since there are no clients")
				break loop
			}
			logger.Println("continuing to serve until all existing clients exit")
		case conn := <-connCh:
			conns[conn] = struct{}{}
			go func() {
				server.ServeConn(conn)
				connDoneCh <- conn
			}()
		case conn := <-connDoneCh:
			delete(conns, conn)
			if len(conns) == 0 {
				logger.Println("all clients disconnected, exiting")
				break loop
			}
		}
	}

	err = os.Remove(sockpath)
	if err != nil {
		logger.Printf("failed to remove socket %s: %v", sockpath, err)
	}
	if st != nil {
		err = st.Close()
		if err != nil {
			logger.Printf("failed to close storage: %v", err)
		}
	}
	err = listener.Close()
	if err != nil {
		logger.Printf("failed to close listener: %v", err)
	}
	// Ensure that the listener goroutine has exited before returning
	<-listenErrCh
	return 0
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"os"
	"strings"

	"github.com/andrew-d/go-termutil"
	"github.com/codegangsta/cli"
	"github.com/mgutz/ansi"
)

// os.Exit forcely kills process, so let me share this global variable to terminate at the last
var exitCode = 0

func main() {
	app := cli.NewApp()
	app.Name = "lltsv"
	app.Version = Version
	app.Usage = `List specified keys of LTSV (Labeled Tab Separated Values)

	Example1 $ echo "foo:aaa\tbar:bbb" | lltsv -k foo,bar
	foo:aaa   bar:bbb

	The output is colorized as default when you outputs to a terminal.
	The coloring is disabled if you pipe or redirect outputs.

	Example2 $ echo "foo:aaa\tbar:bbb" | lltsv -k foo,bar -K
	aaa       bbb

	You may eliminate labels with "-K" option`
	app.Author = "sonots"
	app.Email = "sonots@gmail.com"
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "key, k",
			Usage: "keys to output (multiple keys separated by ,)",
		},
		cli.BoolFlag{
			Name:  "no-key, K",
			Usage: "output without keys (and without color)",
		},
	}
	app.Action = doMain
	app.Run(os.Args)
	os.Exit(exitCode)
}

func doMain(c *cli.Context) {
	keys := make([]string, 0, 0) // slice with length 0
	if c.String("key") != "" {
		keys = strings.Split(c.String("key"), ",")
	}
	no_key := c.Bool("no-key")
	funcAppend := getFuncAppend(no_key)

	var file *os.File
	if len(c.Args()) > 0 {
		filename := c.Args()[0]
		var err error
		file, err = os.Open(filename)
		if err != nil {
			os.Stderr.WriteString("failed to open and read " + filename)
			exitCode = 1
			return
		}
	} else {
		file = os.Stdin
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		line := scanner.Text()
		lvs := parseLtsv(line)
		ltsv := restructLtsv(keys, lvs, funcAppend)
		os.Stdout.WriteString(ltsv + "\n")
	}
	if err := scanner.Err(); err != nil {
		os.Stderr.WriteString("reading standard input errored")
		exitCode = 1
		return
	}
}

func restructLtsv(keys []string, lvs map[string]string, funcAppend func([]string, string, string) []string) string {
	// specified keys or all keys
	orders := keys
	if len(keys) == 0 {
		orders = keysInMap(lvs)
	}
	// make slice with enough capacity so that append does not newly create object
	// cf. http://golang.org/pkg/builtin/#append
	selected := make([]string, 0, len(orders))
	for _, label := range orders {
		value := lvs[label]
		selected = funcAppend(selected, label, value)
	}
	return strings.Join(selected, "\t")
}

func parseLtsv(line string) map[string]string {
	columns := strings.Split(line, "\t")
	lvs := make(map[string]string)
	for _, column := range columns {
		l_v := strings.SplitN(column, ":", 2)
		if len(l_v) < 2 {
			continue
		}
		label, value := l_v[0], l_v[1]
		lvs[label] = value
	}
	return lvs
}

// Return function pointer to avoid `if` evaluation occurs in each iteration
func getFuncAppend(no_key bool) func([]string, string, string) []string {
	if no_key {
		return func(selected []string, label string, value string) []string {
			return append(selected, value)
		}
	} else {
		if termutil.Isatty(os.Stdout.Fd()) {
			return func(selected []string, label string, value string) []string {
				return append(selected, ansi.Color(label, "green")+":"+ansi.Color(value, "magenta"))
			}
		} else {
			// if pipe or redirect
			return func(selected []string, label string, value string) []string {
				return append(selected, label+":"+value)
			}
		}
	}
}

func keysInMap(m map[string]string) []string {
	keys := make([]string, 0, len(m))
	for key := range m {
		keys = append(keys, key)
	}
	return keys
}

func stringInSlice(a string, list []string) bool {
	for _, b := range list {
		if b == a {
			return true
		}
	}
	return false
}
<commit_msg>add github url<commit_after>package main

import (
	"bufio"
	"os"
	"strings"

	"github.com/andrew-d/go-termutil"
	"github.com/codegangsta/cli"
	"github.com/mgutz/ansi"
)

// os.Exit forcely kills process, so let me share this global variable to terminate at the last
var exitCode = 0

func main() {
	app := cli.NewApp()
	app.Name = "lltsv"
	app.Version = Version
	app.Usage = `List specified keys of LTSV (Labeled Tab Separated Values)

	Example1 $ echo "foo:aaa\tbar:bbb" | lltsv -k foo,bar
	foo:aaa   bar:bbb

	The output is colorized as default when you outputs to a terminal.
	The coloring is disabled if you pipe or redirect outputs.

	Example2 $ echo "foo:aaa\tbar:bbb" | lltsv -k foo,bar -K
	aaa       bbb

	You may eliminate labels with "-K" option.

	https://github.com/sonots/lltsv`
	app.Author = "sonots"
	app.Email = "sonots@gmail.com"
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "key, k",
			Usage: "keys to output (multiple keys separated by ,)",
		},
		cli.BoolFlag{
			Name:  "no-key, K",
			Usage: "output without keys (and without color)",
		},
	}
	app.Action = doMain
	app.Run(os.Args)
	os.Exit(exitCode)
}

func doMain(c *cli.Context) {
	keys := make([]string, 0, 0) // slice with length 0
	if c.String("key") != "" {
		keys = strings.Split(c.String("key"), ",")
	}
	no_key := c.Bool("no-key")
	funcAppend := getFuncAppend(no_key)

	var file *os.File
	if len(c.Args()) > 0 {
		filename := c.Args()[0]
		var err error
		file, err = os.Open(filename)
		if err != nil {
			os.Stderr.WriteString("failed to open and read " + filename)
			exitCode = 1
			return
		}
	} else {
		file = os.Stdin
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		line := scanner.Text()
		lvs := parseLtsv(line)
		ltsv := restructLtsv(keys, lvs, funcAppend)
		os.Stdout.WriteString(ltsv + "\n")
	}
	if err := scanner.Err(); err != nil {
		os.Stderr.WriteString("reading standard input errored")
		exitCode = 1
		return
	}
}

func restructLtsv(keys []string, lvs map[string]string, funcAppend func([]string, string, string) []string) string {
	// specified keys or all keys
	orders := keys
	if len(keys) == 0 {
		orders = keysInMap(lvs)
	}
	// make slice with enough capacity so that append does not newly create object
	// cf. http://golang.org/pkg/builtin/#append
	selected := make([]string, 0, len(orders))
	for _, label := range orders {
		value := lvs[label]
		selected = funcAppend(selected, label, value)
	}
	return strings.Join(selected, "\t")
}

func parseLtsv(line string) map[string]string {
	columns := strings.Split(line, "\t")
	lvs := make(map[string]string)
	for _, column := range columns {
		l_v := strings.SplitN(column, ":", 2)
		if len(l_v) < 2 {
			continue
		}
		label, value := l_v[0], l_v[1]
		lvs[label] = value
	}
	return lvs
}

// Return function pointer to avoid `if` evaluation occurs in each iteration
func getFuncAppend(no_key bool) func([]string, string, string) []string {
	if no_key {
		return func(selected []string, label string, value string) []string {
			return append(selected, value)
		}
	} else {
		if termutil.Isatty(os.Stdout.Fd()) {
			return func(selected []string, label string, value string) []string {
				return append(selected, ansi.Color(label, "green")+":"+ansi.Color(value, "magenta"))
			}
		} else {
			// if pipe or redirect
			return func(selected []string, label string, value string) []string {
				return append(selected, label+":"+value)
			}
		}
	}
}

func keysInMap(m map[string]string) []string {
	keys := make([]string, 0, len(m))
	for key := range m {
		keys = append(keys, key)
	}
	return keys
}

func stringInSlice(a string, list []string) bool {
	for _, b := range list {
		if b == a {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>// Copyright 2016 Red Hat, Inc, and individual contributors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package kubesh

import (
	"fmt"
	"io"
	"os"
	"os/exec"
	"os/signal"
	"path"
	"strings"
	"syscall"
	"time"

	"github.com/bbrowning/readline"
	"github.com/spf13/cobra"

	"k8s.io/kubernetes/pkg/kubectl/cmd"
	cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"
	"k8s.io/kubernetes/pkg/util/homedir"
)

type kubesh struct {
	finder     ResourceFinder
	context    []string
	lineReader *readline.Instance
	progname   string
	factory    *cmdutil.Factory
	out        *NewlineEnsuringWriter
}

func NewKubesh() *kubesh {
	factory := cmdutil.NewFactory(nil)
	sh := kubesh{
		factory:  &factory,
		finder:   TimeoutFinder{Resourceful{&factory}, time.Second * 2},
		progname: os.Args[0],
		out:      &NewlineEnsuringWriter{delegate: os.Stdout},
	}
	completer := NewCompleter(sh.newRootCommand(), sh.finder, &sh.context)
	sh.lineReader, _ = readline.NewEx(&readline.Config{
		Prompt:       prompt([]string{}),
		AutoComplete: completer,
		HistoryFile:  path.Join(homedir.HomeDir(), ".kubesh_history"),
		Listener:     completer,
	})
	return &sh
}

func (sh *kubesh) Run() {
	defer sh.lineReader.Close()
	for {
		line, err := sh.lineReader.Readline()
		if err == readline.ErrInterrupt {
			if len(line) == 0 {
				break
			} else {
				continue
			}
		} else if err == io.EOF {
			break
		}
		args, err := tokenize(line)
		if err != nil {
			fmt.Println(err)
		} else if len(args) > 0 {
			cmd := sh.findCommand(args)
			// TODO: what do we do with an error here? do we care?
			args, _ = applyContext(sh.context, args, cmd)
			sh.runCommand(cmd, args)
		}
		// if the command output something w/o a trailing \n, it won't
		// show, so we make sure one exists
		sh.out.EnsureNewline()
	}
}

func (sh *kubesh) Execute(args []string) {
	cmd := sh.newRootCommand()
	cmd.SetArgs(args)
	cmd.Execute()
}

func (sh *kubesh) findCommand(args []string) Command {
	root := sh.newRootCommand()
	cmd, _, _ := root.Find(args)
	return KubectlCommand{cmd}
}

func (sh *kubesh) runCommand(cmd Command, args []string) {
	defer func() {
		// Ignore any panics from kubectl
		recover()
	}()
	switch cmd.Name() {
	case "proxy", "attach":
		sh.runExec(args)
	default:
		sh.Execute(args)
	}
}

func (sh *kubesh) runExec(args []string) {
	cmd := exec.Command(sh.progname, args...)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Start()

	signals := make(chan os.Signal, 1)
	signal.Notify(signals, syscall.SIGINT)
	defer signal.Stop(signals)
	go func() {
		<-signals
		fmt.Println("")
		syscall.Kill(cmd.Process.Pid, syscall.SIGINT)
	}()

	cmd.Wait()
}

func (sh *kubesh) newRootCommand() *cobra.Command {
	root := cmd.NewKubectlCommand(*sh.factory, os.Stdin, sh.out, os.Stderr)
	get, _, err := root.Find([]string{"get"})
	if err != nil {
		panic(err)
	}
	cmd := &cobra.Command{
		Use:          "pin",
		Short:        "Pin resources for use in subsequent commands",
		Long:         pin_long,
		Example:      pin_example,
		SilenceUsage: true,
		ValidArgs:    get.ValidArgs,
		Run: func(cmd *cobra.Command, args []string) {
			if cmdutil.GetFlagBool(cmd, "clear") {
				sh.context = []string{}
			} else {
				err := setContextCommand(sh, args)
				cmdutil.CheckErr(err)
			}
			sh.lineReader.SetPrompt(prompt(sh.context))
		},
	}
	cmd.Flags().BoolP("clear", "c", false, "Clears pinned resource")
	root.AddCommand(cmd)

	cmd = &cobra.Command{
		Use:   "exit",
		Short: "Exit kubesh",
		Run: func(cmd *cobra.Command, args []string) {
			fmt.Println("Bye!")
			os.Exit(0)
		},
	}
	root.AddCommand(cmd)
	return root
}

func prompt(context []string) string {
	path := ""
	if len(context) > 0 {
		path = fmt.Sprintf("[%v]", strings.Join(context, "/"))
	}

	return fmt.Sprintf(" kubesh%v> ", path)
}
<commit_msg>Sharing factories means flags surprisingly persist<commit_after>// Copyright 2016 Red Hat, Inc, and individual contributors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package kubesh

import (
	"fmt"
	"io"
	"os"
	"os/exec"
	"os/signal"
	"path"
	"strings"
	"syscall"
	"time"

	"github.com/bbrowning/readline"
	"github.com/spf13/cobra"

	"k8s.io/kubernetes/pkg/kubectl/cmd"
	cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"
	"k8s.io/kubernetes/pkg/util/homedir"
)

type kubesh struct {
	finder     ResourceFinder
	context    []string
	lineReader *readline.Instance
	progname   string
	out        *NewlineEnsuringWriter
}

func NewKubesh() *kubesh {
	factory := cmdutil.NewFactory(nil)
	sh := kubesh{
		finder:   TimeoutFinder{Resourceful{&factory}, time.Second * 2},
		progname: os.Args[0],
		out:      &NewlineEnsuringWriter{delegate: os.Stdout},
	}
	completer := NewCompleter(sh.newRootCommand(), sh.finder, &sh.context)
	sh.lineReader, _ = readline.NewEx(&readline.Config{
		Prompt:       prompt([]string{}),
		AutoComplete: completer,
		HistoryFile:  path.Join(homedir.HomeDir(), ".kubesh_history"),
		Listener:     completer,
	})
	return &sh
}

func (sh *kubesh) Run() {
	defer sh.lineReader.Close()
	for {
		line, err := sh.lineReader.Readline()
		if err == readline.ErrInterrupt {
			if len(line) == 0 {
				break
			} else {
				continue
			}
		} else if err == io.EOF {
			break
		}
		args, err := tokenize(line)
		if err != nil {
			fmt.Println(err)
		} else if len(args) > 0 {
			cmd := sh.findCommand(args)
			// TODO: what do we do with an error here? do we care?
			args, _ = applyContext(sh.context, args, cmd)
			sh.runCommand(cmd, args)
		}
		// if the command output something w/o a trailing \n, it won't
		// show, so we make sure one exists
		sh.out.EnsureNewline()
	}
}

func (sh *kubesh) Execute(args []string) {
	cmd := sh.newRootCommand()
	cmd.SetArgs(args)
	cmd.Execute()
}

func (sh *kubesh) findCommand(args []string) Command {
	root := sh.newRootCommand()
	cmd, _, _ := root.Find(args)
	return KubectlCommand{cmd}
}

func (sh *kubesh) runCommand(cmd Command, args []string) {
	defer func() {
		// Ignore any panics from kubectl
		recover()
	}()
	switch cmd.Name() {
	case "proxy", "attach":
		sh.runExec(args)
	default:
		sh.Execute(args)
	}
}

func (sh *kubesh) runExec(args []string) {
	cmd := exec.Command(sh.progname, args...)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Start()

	signals := make(chan os.Signal, 1)
	signal.Notify(signals, syscall.SIGINT)
	defer signal.Stop(signals)
	go func() {
		<-signals
		fmt.Println("")
		syscall.Kill(cmd.Process.Pid, syscall.SIGINT)
	}()

	cmd.Wait()
}

func (sh *kubesh) newRootCommand() *cobra.Command {
	root := cmd.NewKubectlCommand(cmdutil.NewFactory(nil), os.Stdin, sh.out, os.Stderr)
	get, _, err := root.Find([]string{"get"})
	if err != nil {
		panic(err)
	}

	user, err := root.Flags().GetString("user")
	if err != nil {
		fmt.Println("JC: ERR!", err)
	}
	fmt.Println("JC: USER=>", user)

	cmd := &cobra.Command{
		Use:          "pin",
		Short:        "Pin resources for use in subsequent commands",
		Long:         pin_long,
		Example:      pin_example,
		SilenceUsage: true,
		ValidArgs:    get.ValidArgs,
		Run: func(cmd *cobra.Command, args []string) {
			if cmdutil.GetFlagBool(cmd, "clear") {
				sh.context = []string{}
			} else {
				err := setContextCommand(sh, args)
				cmdutil.CheckErr(err)
			}
			sh.lineReader.SetPrompt(prompt(sh.context))
		},
	}
	cmd.Flags().BoolP("clear", "c", false, "Clears pinned resource")
	root.AddCommand(cmd)

	cmd = &cobra.Command{
		Use:   "exit",
		Short: "Exit kubesh",
		Run: func(cmd *cobra.Command, args []string) {
			fmt.Println("Bye!")
			os.Exit(0)
		},
	}
	root.AddCommand(cmd)
	return root
}

func prompt(context []string) string {
	path := ""
	if len(context) > 0 {
		path = fmt.Sprintf("[%v]", strings.Join(context, "/"))
	}

	return fmt.Sprintf(" kubesh%v> ", path)
}
<|endoftext|>


<commit_before>package ot

// extend.go
//
// Extending Oblivious Transfers Efficiently
// Yuval Ishai, Joe Kilian, Kobbi Nissim, Erez Petrank
// CRYPTO 2003
// http://link.springer.com/chapter/10.1007/978-3-540-45146-4_9
//
// Modified with preprocessing step

import (
	"crypto/rand"
	"encoding/binary"
	"fmt"
	"github.com/tjim/smpcc/runtime/bit"
	"golang.org/x/crypto/sha3"
	"io"
)

type ExtendSender struct {
	R            Receiver
	z0, z1       [][]byte
	m            int
	k            int
	otExtChan    chan []byte
	otExtSelChan chan Selector
	curPair      int
	started      bool
	sendCalls    int
}

type ExtendReceiver struct {
	S            Sender
	r            []byte
	m            int
	k            int
	otExtChan    chan []byte
	otExtSelChan chan Selector
	curPair      int
	T            *bit.Matrix8
}

func NewExtendSender(c chan []byte, otExtSelChan chan Selector, R Receiver, k, m int) Sender {
	if k%8 != 0 {
		panic("k must be a multiple of 8")
	}
	if m%8 != 0 {
		panic("m must be a multiple of 8")
	}
	sender := new(ExtendSender)
	sender.otExtSelChan = otExtSelChan
	sender.k = k
	sender.R = R
	sender.otExtChan = c
	sender.m = m
	sender.curPair = m
	sender.started = false
	sender.sendCalls = 0
	return sender
}

func NewExtendReceiver(c chan []byte, otExtSelChan chan Selector, S Sender, k, m int) Receiver {
	if k%8 != 0 {
		panic("k must be a multiple of 8")
	}
	if m%8 != 0 {
		panic("m must be a multiple of 8")
	}
	receiver := new(ExtendReceiver)
	receiver.otExtSelChan = otExtSelChan
	receiver.k = k
	receiver.S = S
	receiver.m = m
	receiver.curPair = m
	receiver.otExtChan = c
	return receiver
}

func (self *ExtendSender) preProcessSender(m int) {
	if m%8 != 0 {
		panic("m must be a multiple of 8")
	}
	self.started = true
	self.m = m
	self.curPair = 0
	s := make([]byte, self.k/8)
	randomBitVector(s)

	QT := bit.NewMatrix8(self.k, self.m)
	for i := 0; i < QT.NumRows; i++ {
		recvd := self.R.Receive(Selector(bit.GetBit(s, i)))
		if len(recvd) != self.m/8 {
			panic(fmt.Sprintf("Incorrect column length received: %d != %d", len(recvd), self.m/8))
		}
		QT.SetRow(i, recvd)
	}
	Q := QT.Transpose()
	self.z0 = make([][]byte, m)
	self.z1 = make([][]byte, m)
	temp := make([]byte, self.k/8)
	for j := 0; j < m; j++ {
		self.z0[j] = Q.GetRow(j)
		xorBytes(temp, Q.GetRow(j), s)
		self.z1[j] = make([]byte, len(temp))
		copy(self.z1[j], temp)
	}
}

func (self *ExtendReceiver) preProcessReceiver(m int) {
	self.curPair = 0
	self.m = m
	self.r = make([]byte, self.m/8)
	randomBitVector(self.r)
	T := bit.NewMatrix8(self.m, self.k)
	T.Randomize()
	self.T = T
	TT := T.Transpose()
	temp := make([]byte, self.m/8)
	for i := 0; i < self.k; i++ {
		xorBytes(temp, self.r, TT.GetRow(i))
		self.S.Send(TT.GetRow(i), temp)
	}
}

// hash function instantiating a random oracle
func RO(input []byte, outBits int) []byte {
	if outBits <= 0 {
		panic("output size <= 0")
	}
	if outBits%8 != 0 {
		panic("output size must be a multiple of 8")
	}
	output := make([]byte, outBits/8)
	sha3.ShakeSum256(output, input)
	return output
}

// Currently not used because of Footnote 10, page 13 of Ishai03:
// It is not hard to verify that as long as the receiver is honest,
// the protocol remains secure. The inclusion of j in the input to
// the random oracle slightly simplifies the analysis and is useful
// towards realizing the fully secure variant of this protocol.
func RO_j(curPair int, input []byte, outBits int) []byte {
	if outBits <= 0 {
		panic("output size <= 0")
	}
	if outBits%8 != 0 {
		panic("output size must be a multiple of 8")
	}
	curPair_bytes := make([]byte, 4)
	binary.LittleEndian.PutUint32(curPair_bytes, curPair)
	output := make([]byte, outBits/8)
	sha3.ShakeSum256(output, append(curPair_bytes, input))
	return output
}

func (self *ExtendSender) Send(m0, m1 Message) {
	if self.curPair == self.m {
		self.preProcessSender(self.m)
	}
	if len(m0) != len(m1) {
		panic("(*ot.ExtendSender).Send: messages have different lengths")
	}
	msglen := len(m0)
	y0 := make([]byte, msglen)
	y1 := make([]byte, msglen)
	smod := <-self.otExtSelChan
	if smod == 0 {
		xorBytes(y0, m0, RO(self.z0[self.curPair], 8*msglen))
		xorBytes(y1, m1, RO(self.z1[self.curPair], 8*msglen))
	} else if smod == 1 {
		xorBytes(y0, m1, RO(self.z0[self.curPair], 8*msglen))
		xorBytes(y1, m0, RO(self.z1[self.curPair], 8*msglen))
	} else {
		panic("Sender: unexpected smod value")
	}
	self.otExtChan <- y0
	self.otExtChan <- y1
	self.curPair++
	return
}

func (self *ExtendReceiver) Receive(s Selector) Message {
	if self.curPair == self.m {
		self.preProcessReceiver(self.m)
	}
	smod := Selector(byte(s) ^ bit.GetBit(self.r, self.curPair))
	self.otExtSelChan <- smod
	y0 := <-self.otExtChan
	y1 := <-self.otExtChan
	if len(y0) != len(y1) {
		panic("(*ot.ExtendReceiver).Receive: messages have different length")
	}
	msglen := len(y0)
	w := make([]byte, msglen)
	if bit.GetBit(self.r, self.curPair) == 0 {
		xorBytes(w, y0, RO(self.T.GetRow(self.curPair), 8*msglen))
	} else if bit.GetBit(self.r, self.curPair) == 1 {
		xorBytes(w, y1, RO(self.T.GetRow(self.curPair), 8*msglen))
	}
	self.curPair++
	return w
}

func randomBitVector(pool []byte) {
	n, err := io.ReadFull(rand.Reader, pool)
	if err != nil || n != len(pool) {
		panic("randomness allocation failed")
	}
}

// Send m message pairs in one call
func (S *ExtendSender) SendM(a, b []Message) {
	m := len(a)
	if m%8 != 0 {
		panic("SendM: must send a multiple of 8 messages at a time") // force compatibility with stream OT
	}
	if len(b) != m {
		panic("SendM: must send pairs of messages")
	}
	for i := range a {
		S.Send(a[i], b[i])
	}
}
func (R *ExtendReceiver) ReceiveM(r []byte) []Message { // r is a packed vector of selections
	result := make([]Message, 8*len(r))
	for i := range r {
		for bit := 0; bit < 8; bit++ {
			selector := Selector((r[i] >> uint(7-bit)) & 1)
			result[i+bit] = R.Receive(selector)
		}
	}
	return result
}

// Send m pairs of bits (1-bit messages) in one call
func (S *ExtendSender) SendMBits(a, b []byte) { // messages are packed in bytes
	m := 8 * len(a)
	if 8*len(b) != m {
		panic("SendMBits: must send pairs of messages")
	}
	for i := range a {
		for bit := 0; bit < 8; bit++ {
			mask := byte(0x80 >> uint(bit))
			S.Send([]byte{a[i] & mask}, []byte{b[i] & mask})
		}
	}
}
func (R *ExtendReceiver) ReceiveMBits(r []byte) []byte { // r is a packed vector of selections and result is packed as well
	result := make([]byte, len(r))
	for i := range r {
		for bit := 0; bit < 8; bit++ {
			mask := byte(0x80 >> uint(bit))
			selector := Selector((r[i] >> uint(7-bit)) & 1)
			result[i] |= mask & R.Receive(selector)[0]
		}
	}
	return result
}
<commit_msg>Fixed bugs in RO_j<commit_after>package ot

// extend.go
//
// Extending Oblivious Transfers Efficiently
// Yuval Ishai, Joe Kilian, Kobbi Nissim, Erez Petrank
// CRYPTO 2003
// http://link.springer.com/chapter/10.1007/978-3-540-45146-4_9
//
// Modified with preprocessing step

import (
	"crypto/rand"
	"encoding/binary"
	"fmt"
	"github.com/tjim/smpcc/runtime/bit"
	"golang.org/x/crypto/sha3"
	"io"
)

type ExtendSender struct {
	R            Receiver
	z0, z1       [][]byte
	m            int
	k            int
	otExtChan    chan []byte
	otExtSelChan chan Selector
	curPair      int
	started      bool
	sendCalls    int
}

type ExtendReceiver struct {
	S            Sender
	r            []byte
	m            int
	k            int
	otExtChan    chan []byte
	otExtSelChan chan Selector
	curPair      int
	T            *bit.Matrix8
}

func NewExtendSender(c chan []byte, otExtSelChan chan Selector, R Receiver, k, m int) Sender {
	if k%8 != 0 {
		panic("k must be a multiple of 8")
	}
	if m%8 != 0 {
		panic("m must be a multiple of 8")
	}
	sender := new(ExtendSender)
	sender.otExtSelChan = otExtSelChan
	sender.k = k
	sender.R = R
	sender.otExtChan = c
	sender.m = m
	sender.curPair = m
	sender.started = false
	sender.sendCalls = 0
	return sender
}

func NewExtendReceiver(c chan []byte, otExtSelChan chan Selector, S Sender, k, m int) Receiver {
	if k%8 != 0 {
		panic("k must be a multiple of 8")
	}
	if m%8 != 0 {
		panic("m must be a multiple of 8")
	}
	receiver := new(ExtendReceiver)
	receiver.otExtSelChan = otExtSelChan
	receiver.k = k
	receiver.S = S
	receiver.m = m
	receiver.curPair = m
	receiver.otExtChan = c
	return receiver
}

func (self *ExtendSender) preProcessSender(m int) {
	if m%8 != 0 {
		panic("m must be a multiple of 8")
	}
	self.started = true
	self.m = m
	self.curPair = 0
	s := make([]byte, self.k/8)
	randomBitVector(s)

	QT := bit.NewMatrix8(self.k, self.m)
	for i := 0; i < QT.NumRows; i++ {
		recvd := self.R.Receive(Selector(bit.GetBit(s, i)))
		if len(recvd) != self.m/8 {
			panic(fmt.Sprintf("Incorrect column length received: %d != %d", len(recvd), self.m/8))
		}
		QT.SetRow(i, recvd)
	}
	Q := QT.Transpose()
	self.z0 = make([][]byte, m)
	self.z1 = make([][]byte, m)
	temp := make([]byte, self.k/8)
	for j := 0; j < m; j++ {
		self.z0[j] = Q.GetRow(j)
		xorBytes(temp, Q.GetRow(j), s)
		self.z1[j] = make([]byte, len(temp))
		copy(self.z1[j], temp)
	}
}

func (self *ExtendReceiver) preProcessReceiver(m int) {
	self.curPair = 0
	self.m = m
	self.r = make([]byte, self.m/8)
	randomBitVector(self.r)
	T := bit.NewMatrix8(self.m, self.k)
	T.Randomize()
	self.T = T
	TT := T.Transpose()
	temp := make([]byte, self.m/8)
	for i := 0; i < self.k; i++ {
		xorBytes(temp, self.r, TT.GetRow(i))
		self.S.Send(TT.GetRow(i), temp)
	}
}

// hash function instantiating a random oracle
func RO(input []byte, outBits int) []byte {
	if outBits <= 0 {
		panic("output size <= 0")
	}
	if outBits%8 != 0 {
		panic("output size must be a multiple of 8")
	}
	output := make([]byte, outBits/8)
	sha3.ShakeSum256(output, input)
	return output
}

// Currently not used because of Footnote 10, page 13 of Ishai03:
// It is not hard to verify that as long as the receiver is honest,
// the protocol remains secure. The inclusion of j in the input to
// the random oracle slightly simplifies the analysis and is useful
// towards realizing the fully secure variant of this protocol.
func RO_j(curPair int, input []byte, outBits int) []byte {
	if outBits <= 0 {
		panic("output size <= 0")
	}
	if outBits%8 != 0 {
		panic("output size must be a multiple of 8")
	}
	curPair_bytes := make([]byte, 4)
	binary.LittleEndian.PutUint32(curPair_bytes, uint32(curPair))
	output := make([]byte, outBits/8)
	sha3.ShakeSum256(output, append(curPair_bytes, input...))
	return output
}

func (self *ExtendSender) Send(m0, m1 Message) {
	if self.curPair == self.m {
		self.preProcessSender(self.m)
	}
	if len(m0) != len(m1) {
		panic("(*ot.ExtendSender).Send: messages have different lengths")
	}
	msglen := len(m0)
	y0 := make([]byte, msglen)
	y1 := make([]byte, msglen)
	smod := <-self.otExtSelChan
	if smod == 0 {
		xorBytes(y0, m0, RO(self.z0[self.curPair], 8*msglen))
		xorBytes(y1, m1, RO(self.z1[self.curPair], 8*msglen))
	} else if smod == 1 {
		xorBytes(y0, m1, RO(self.z0[self.curPair], 8*msglen))
		xorBytes(y1, m0, RO(self.z1[self.curPair], 8*msglen))
	} else {
		panic("Sender: unexpected smod value")
	}
	self.otExtChan <- y0
	self.otExtChan <- y1
	self.curPair++
	return
}

func (self *ExtendReceiver) Receive(s Selector) Message {
	if self.curPair == self.m {
		self.preProcessReceiver(self.m)
	}
	smod := Selector(byte(s) ^ bit.GetBit(self.r, self.curPair))
	self.otExtSelChan <- smod
	y0 := <-self.otExtChan
	y1 := <-self.otExtChan
	if len(y0) != len(y1) {
		panic("(*ot.ExtendReceiver).Receive: messages have different length")
	}
	msglen := len(y0)
	w := make([]byte, msglen)
	if bit.GetBit(self.r, self.curPair) == 0 {
		xorBytes(w, y0, RO(self.T.GetRow(self.curPair), 8*msglen))
	} else if bit.GetBit(self.r, self.curPair) == 1 {
		xorBytes(w, y1, RO(self.T.GetRow(self.curPair), 8*msglen))
	}
	self.curPair++
	return w
}

func randomBitVector(pool []byte) {
	n, err := io.ReadFull(rand.Reader, pool)
	if err != nil || n != len(pool) {
		panic("randomness allocation failed")
	}
}

// Send m message pairs in one call
func (S *ExtendSender) SendM(a, b []Message) {
	m := len(a)
	if m%8 != 0 {
		panic("SendM: must send a multiple of 8 messages at a time") // force compatibility with stream OT
	}
	if len(b) != m {
		panic("SendM: must send pairs of messages")
	}
	for i := range a {
		S.Send(a[i], b[i])
	}
}
func (R *ExtendReceiver) ReceiveM(r []byte) []Message { // r is a packed vector of selections
	result := make([]Message, 8*len(r))
	for i := range r {
		for bit := 0; bit < 8; bit++ {
			selector := Selector((r[i] >> uint(7-bit)) & 1)
			result[i+bit] = R.Receive(selector)
		}
	}
	return result
}

// Send m pairs of bits (1-bit messages) in one call
func (S *ExtendSender) SendMBits(a, b []byte) { // messages are packed in bytes
	m := 8 * len(a)
	if 8*len(b) != m {
		panic("SendMBits: must send pairs of messages")
	}
	for i := range a {
		for bit := 0; bit < 8; bit++ {
			mask := byte(0x80 >> uint(bit))
			S.Send([]byte{a[i] & mask}, []byte{b[i] & mask})
		}
	}
}
func (R *ExtendReceiver) ReceiveMBits(r []byte) []byte { // r is a packed vector of selections and result is packed as well
	result := make([]byte, len(r))
	for i := range r {
		for bit := 0; bit < 8; bit++ {
			mask := byte(0x80 >> uint(bit))
			selector := Selector((r[i] >> uint(7-bit)) & 1)
			result[i] |= mask & R.Receive(selector)[0]
		}
	}
	return result
}
<|endoftext|>


<commit_before>// Copyright 2013 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package mapper

import (
	"fmt"
	"io/ioutil"
	"regexp"
	"sync"
	"time"

	"github.com/go-kit/log"
	"github.com/prometheus/client_golang/prometheus"
	"gopkg.in/yaml.v2"

	"github.com/prometheus/statsd_exporter/pkg/level"
	"github.com/prometheus/statsd_exporter/pkg/mapper/fsm"
)

var (
	// The first segment of a match cannot start with a number
	statsdMetricRE = `[a-zA-Z_]([a-zA-Z0-9_\-])*`
	// The subsequent segments of a match can start with a number
	// See https://github.com/prometheus/statsd_exporter/issues/328
	statsdMetricSubsequentRE = `[a-zA-Z0-9_]([a-zA-Z0-9_\-])*`
	templateReplaceRE        = `(\$\{?\d+\}?)`

	metricLineRE = regexp.MustCompile(`^(\*|` + statsdMetricRE + `)(\.\*|\.` + statsdMetricSubsequentRE + `)*$`)
	metricNameRE = regexp.MustCompile(`^([a-zA-Z_]|` + templateReplaceRE + `)([a-zA-Z0-9_]|` + templateReplaceRE + `)*$`)
	labelNameRE  = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]+$`)
)

type MetricMapper struct {
	Registerer prometheus.Registerer
	Defaults   mapperConfigDefaults `yaml:"defaults"`
	Mappings   []MetricMapping      `yaml:"mappings"`
	FSM        *fsm.FSM
	doFSM      bool
	doRegex    bool
	cache      MetricMapperCache
	mutex      sync.RWMutex

	MappingsCount prometheus.Gauge

	Logger log.Logger
}

type SummaryOptions struct {
	Quantiles  []metricObjective `yaml:"quantiles"`
	MaxAge     time.Duration     `yaml:"max_age"`
	AgeBuckets uint32            `yaml:"age_buckets"`
	BufCap     uint32            `yaml:"buf_cap"`
}

type HistogramOptions struct {
	Buckets []float64 `yaml:"buckets"`
}

type metricObjective struct {
	Quantile float64 `yaml:"quantile"`
	Error    float64 `yaml:"error"`
}

var defaultQuantiles = []metricObjective{
	{Quantile: 0.5, Error: 0.05},
	{Quantile: 0.9, Error: 0.01},
	{Quantile: 0.99, Error: 0.001},
}

func (m *MetricMapper) InitFromYAMLString(fileContents string) error {
	var n MetricMapper

	if err := yaml.Unmarshal([]byte(fileContents), &n); err != nil {
		return err
	}

	if len(n.Defaults.HistogramOptions.Buckets) == 0 {
		n.Defaults.HistogramOptions.Buckets = prometheus.DefBuckets
	}

	if len(n.Defaults.SummaryOptions.Quantiles) == 0 {
		n.Defaults.SummaryOptions.Quantiles = defaultQuantiles
	}

	if n.Defaults.MatchType == MatchTypeDefault {
		n.Defaults.MatchType = MatchTypeGlob
	}

	remainingMappingsCount := len(n.Mappings)

	n.FSM = fsm.NewFSM([]string{string(MetricTypeCounter), string(MetricTypeGauge), string(MetricTypeObserver)},
		remainingMappingsCount, n.Defaults.GlobDisableOrdering)

	for i := range n.Mappings {
		remainingMappingsCount--

		currentMapping := &n.Mappings[i]

		// check that label is correct
		for k := range currentMapping.Labels {
			if !labelNameRE.MatchString(k) {
				return fmt.Errorf("invalid label key: %s", k)
			}
		}

		if currentMapping.Name == "" {
			return fmt.Errorf("line %d: metric mapping didn't set a metric name", i)
		}

		if !metricNameRE.MatchString(currentMapping.Name) {
			return fmt.Errorf("metric name '%s' doesn't match regex '%s'", currentMapping.Name, metricNameRE)
		}

		if currentMapping.MatchType == "" {
			currentMapping.MatchType = n.Defaults.MatchType
		}

		if currentMapping.Action == "" {
			currentMapping.Action = ActionTypeMap
		}

		if currentMapping.MatchType == MatchTypeGlob {
			n.doFSM = true
			if !metricLineRE.MatchString(currentMapping.Match) {
				return fmt.Errorf("invalid match: %s", currentMapping.Match)
			}

			captureCount := n.FSM.AddState(currentMapping.Match, string(currentMapping.MatchMetricType),
				remainingMappingsCount, currentMapping)

			currentMapping.nameFormatter = fsm.NewTemplateFormatter(currentMapping.Name, captureCount)

			labelKeys := make([]string, len(currentMapping.Labels))
			labelFormatters := make([]*fsm.TemplateFormatter, len(currentMapping.Labels))
			labelIndex := 0
			for label, valueExpr := range currentMapping.Labels {
				labelKeys[labelIndex] = label
				labelFormatters[labelIndex] = fsm.NewTemplateFormatter(valueExpr, captureCount)
				labelIndex++
			}
			currentMapping.labelFormatters = labelFormatters
			currentMapping.labelKeys = labelKeys
		} else {
			if regex, err := regexp.Compile(currentMapping.Match); err != nil {
				return fmt.Errorf("invalid regex %s in mapping: %v", currentMapping.Match, err)
			} else {
				currentMapping.regex = regex
			}
			n.doRegex = true
		}

		if currentMapping.ObserverType == "" {
			currentMapping.ObserverType = n.Defaults.ObserverType
		}

		if currentMapping.LegacyQuantiles != nil &&
			(currentMapping.SummaryOptions == nil || currentMapping.SummaryOptions.Quantiles != nil) {
			level.Warn(m.Logger).Log("msg", "using the top level quantiles is deprecated.  Please use quantiles in the summary_options hierarchy")
		}

		if currentMapping.LegacyBuckets != nil &&
			(currentMapping.HistogramOptions == nil || currentMapping.HistogramOptions.Buckets != nil) {
			level.Warn(m.Logger).Log("msg", "using the top level buckets is deprecated.  Please use buckets in the histogram_options hierarchy")
		}

		if currentMapping.SummaryOptions != nil &&
			currentMapping.LegacyQuantiles != nil &&
			currentMapping.SummaryOptions.Quantiles != nil {
			return fmt.Errorf("cannot use quantiles in both the top level and summary options at the same time in %s", currentMapping.Match)
		}

		if currentMapping.HistogramOptions != nil &&
			currentMapping.LegacyBuckets != nil &&
			currentMapping.HistogramOptions.Buckets != nil {
			return fmt.Errorf("cannot use buckets in both the top level and histogram options at the same time in %s", currentMapping.Match)
		}

		if currentMapping.ObserverType == ObserverTypeHistogram {
			if currentMapping.SummaryOptions != nil {
				return fmt.Errorf("cannot use histogram observer and summary options at the same time")
			}
			if currentMapping.HistogramOptions == nil {
				currentMapping.HistogramOptions = &HistogramOptions{}
			}
			if currentMapping.LegacyBuckets != nil && len(currentMapping.LegacyBuckets) != 0 {
				currentMapping.HistogramOptions.Buckets = currentMapping.LegacyBuckets
			}
			if currentMapping.HistogramOptions.Buckets == nil || len(currentMapping.HistogramOptions.Buckets) == 0 {
				currentMapping.HistogramOptions.Buckets = n.Defaults.HistogramOptions.Buckets
			}
		}

		if currentMapping.ObserverType == ObserverTypeSummary {
			if currentMapping.HistogramOptions != nil {
				return fmt.Errorf("cannot use summary observer and histogram options at the same time")
			}
			if currentMapping.SummaryOptions == nil {
				currentMapping.SummaryOptions = &SummaryOptions{}
			}
			if currentMapping.LegacyQuantiles != nil && len(currentMapping.LegacyQuantiles) != 0 {
				currentMapping.SummaryOptions.Quantiles = currentMapping.LegacyQuantiles
			}
			if currentMapping.SummaryOptions.Quantiles == nil || len(currentMapping.SummaryOptions.Quantiles) == 0 {
				currentMapping.SummaryOptions.Quantiles = n.Defaults.SummaryOptions.Quantiles
			}
			if currentMapping.SummaryOptions.MaxAge == 0 {
				currentMapping.SummaryOptions.MaxAge = n.Defaults.SummaryOptions.MaxAge
			}
			if currentMapping.SummaryOptions.AgeBuckets == 0 {
				currentMapping.SummaryOptions.AgeBuckets = n.Defaults.SummaryOptions.AgeBuckets
			}
			if currentMapping.SummaryOptions.BufCap == 0 {
				currentMapping.SummaryOptions.BufCap = n.Defaults.SummaryOptions.BufCap
			}
		}

		if currentMapping.Ttl == 0 && n.Defaults.Ttl > 0 {
			currentMapping.Ttl = n.Defaults.Ttl
		}
	}

	m.mutex.Lock()
	defer m.mutex.Unlock()

	m.Defaults = n.Defaults
	m.Mappings = n.Mappings

	// Reset the cache since this function can be used to reload config
	if m.cache != nil {
		m.cache.Reset()
	}

	if n.doFSM {
		var mappings []string
		for _, mapping := range n.Mappings {
			if mapping.MatchType == MatchTypeGlob {
				mappings = append(mappings, mapping.Match)
			}
		}
		n.FSM.BacktrackingNeeded = fsm.TestIfNeedBacktracking(mappings, n.FSM.OrderingDisabled, m.Logger)

		m.FSM = n.FSM
		m.doRegex = n.doRegex
	}
	m.doFSM = n.doFSM

	if m.MappingsCount != nil {
		m.MappingsCount.Set(float64(len(n.Mappings)))
	}

	if m.Logger == nil {
		m.Logger = log.NewNopLogger()
	}

	return nil
}

func (m *MetricMapper) InitFromFile(fileName string) error {
	mappingStr, err := ioutil.ReadFile(fileName)
	if err != nil {
		return err
	}

	return m.InitFromYAMLString(string(mappingStr))
}

// UseCache tells the mapper to use a cache that implements the MetricMapperCache interface.
// This cache MUST be thread-safe!
func (m *MetricMapper) UseCache(cache MetricMapperCache) {
	m.mutex.Lock()
	defer m.mutex.Unlock()
	m.cache = cache
}

func (m *MetricMapper) GetMapping(statsdMetric string, statsdMetricType MetricType) (*MetricMapping, prometheus.Labels, bool) {
	m.mutex.RLock()
	defer m.mutex.RUnlock()

	// only use a cache if one is present
	if m.cache != nil {
		result, cached := m.cache.Get(formatKey(statsdMetric, statsdMetricType))
		if cached {
			r := result.(MetricMapperCacheResult)
			return r.Mapping, r.Labels, r.Matched
		}
	}

	// glob matching
	if m.doFSM {
		finalState, captures := m.FSM.GetMapping(statsdMetric, string(statsdMetricType))
		if finalState != nil && finalState.Result != nil {
			v := finalState.Result.(*MetricMapping)
			result := copyMetricMapping(v)
			result.Name = result.nameFormatter.Format(captures)

			labels := prometheus.Labels{}
			for index, formatter := range result.labelFormatters {
				labels[result.labelKeys[index]] = formatter.Format(captures)
			}

			r := MetricMapperCacheResult{
				Mapping: result,
				Matched: true,
				Labels:  labels,
			}
			// add match to cache
			if m.cache != nil {
				m.cache.Add(formatKey(statsdMetric, statsdMetricType), r)
			}

			return result, labels, true
		} else if !m.doRegex {
			// if there's no regex match type, return immediately
			// Add miss to cache
			if m.cache != nil {
				m.cache.Add(formatKey(statsdMetric, statsdMetricType), MetricMapperCacheResult{})
			}
			return nil, nil, false
		}
	}

	// regex matching
	for _, mapping := range m.Mappings {
		// if a rule don't have regex matching type, the regex field is unset
		if mapping.regex == nil {
			continue
		}
		matches := mapping.regex.FindStringSubmatchIndex(statsdMetric)
		if len(matches) == 0 {
			continue
		}

		mapping.Name = string(mapping.regex.ExpandString(
			[]byte{},
			mapping.Name,
			statsdMetric,
			matches,
		))

		if mt := mapping.MatchMetricType; mt != "" && mt != statsdMetricType {
			continue
		}

		labels := prometheus.Labels{}
		for label, valueExpr := range mapping.Labels {
			value := mapping.regex.ExpandString([]byte{}, valueExpr, statsdMetric, matches)
			labels[label] = string(value)
		}

		r := MetricMapperCacheResult{
			Mapping: &mapping,
			Matched: true,
			Labels:  labels,
		}
		// Add Match to cache
		if m.cache != nil {
			m.cache.Add(formatKey(statsdMetric, statsdMetricType), r)
		}

		return &mapping, labels, true
	}

	// Add Miss to cache
	if m.cache != nil {
		m.cache.Add(formatKey(statsdMetric, statsdMetricType), MetricMapperCacheResult{})
	}
	return nil, nil, false
}

// make a shallow copy so that we do not overwrite name
// as multiple names can be matched by same mapping
func copyMetricMapping(in *MetricMapping) *MetricMapping {
	out := *in
	return &out
}
<commit_msg>Remove ioutil<commit_after>// Copyright 2013 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package mapper

import (
	"fmt"
	"os"
	"regexp"
	"sync"
	"time"

	"github.com/go-kit/log"
	"github.com/prometheus/client_golang/prometheus"
	"gopkg.in/yaml.v2"

	"github.com/prometheus/statsd_exporter/pkg/level"
	"github.com/prometheus/statsd_exporter/pkg/mapper/fsm"
)

var (
	// The first segment of a match cannot start with a number
	statsdMetricRE = `[a-zA-Z_]([a-zA-Z0-9_\-])*`
	// The subsequent segments of a match can start with a number
	// See https://github.com/prometheus/statsd_exporter/issues/328
	statsdMetricSubsequentRE = `[a-zA-Z0-9_]([a-zA-Z0-9_\-])*`
	templateReplaceRE        = `(\$\{?\d+\}?)`

	metricLineRE = regexp.MustCompile(`^(\*|` + statsdMetricRE + `)(\.\*|\.` + statsdMetricSubsequentRE + `)*$`)
	metricNameRE = regexp.MustCompile(`^([a-zA-Z_]|` + templateReplaceRE + `)([a-zA-Z0-9_]|` + templateReplaceRE + `)*$`)
	labelNameRE  = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]+$`)
)

type MetricMapper struct {
	Registerer prometheus.Registerer
	Defaults   mapperConfigDefaults `yaml:"defaults"`
	Mappings   []MetricMapping      `yaml:"mappings"`
	FSM        *fsm.FSM
	doFSM      bool
	doRegex    bool
	cache      MetricMapperCache
	mutex      sync.RWMutex

	MappingsCount prometheus.Gauge

	Logger log.Logger
}

type SummaryOptions struct {
	Quantiles  []metricObjective `yaml:"quantiles"`
	MaxAge     time.Duration     `yaml:"max_age"`
	AgeBuckets uint32            `yaml:"age_buckets"`
	BufCap     uint32            `yaml:"buf_cap"`
}

type HistogramOptions struct {
	Buckets []float64 `yaml:"buckets"`
}

type metricObjective struct {
	Quantile float64 `yaml:"quantile"`
	Error    float64 `yaml:"error"`
}

var defaultQuantiles = []metricObjective{
	{Quantile: 0.5, Error: 0.05},
	{Quantile: 0.9, Error: 0.01},
	{Quantile: 0.99, Error: 0.001},
}

func (m *MetricMapper) InitFromYAMLString(fileContents string) error {
	var n MetricMapper

	if err := yaml.Unmarshal([]byte(fileContents), &n); err != nil {
		return err
	}

	if len(n.Defaults.HistogramOptions.Buckets) == 0 {
		n.Defaults.HistogramOptions.Buckets = prometheus.DefBuckets
	}

	if len(n.Defaults.SummaryOptions.Quantiles) == 0 {
		n.Defaults.SummaryOptions.Quantiles = defaultQuantiles
	}

	if n.Defaults.MatchType == MatchTypeDefault {
		n.Defaults.MatchType = MatchTypeGlob
	}

	remainingMappingsCount := len(n.Mappings)

	n.FSM = fsm.NewFSM([]string{string(MetricTypeCounter), string(MetricTypeGauge), string(MetricTypeObserver)},
		remainingMappingsCount, n.Defaults.GlobDisableOrdering)

	for i := range n.Mappings {
		remainingMappingsCount--

		currentMapping := &n.Mappings[i]

		// check that label is correct
		for k := range currentMapping.Labels {
			if !labelNameRE.MatchString(k) {
				return fmt.Errorf("invalid label key: %s", k)
			}
		}

		if currentMapping.Name == "" {
			return fmt.Errorf("line %d: metric mapping didn't set a metric name", i)
		}

		if !metricNameRE.MatchString(currentMapping.Name) {
			return fmt.Errorf("metric name '%s' doesn't match regex '%s'", currentMapping.Name, metricNameRE)
		}

		if currentMapping.MatchType == "" {
			currentMapping.MatchType = n.Defaults.MatchType
		}

		if currentMapping.Action == "" {
			currentMapping.Action = ActionTypeMap
		}

		if currentMapping.MatchType == MatchTypeGlob {
			n.doFSM = true
			if !metricLineRE.MatchString(currentMapping.Match) {
				return fmt.Errorf("invalid match: %s", currentMapping.Match)
			}

			captureCount := n.FSM.AddState(currentMapping.Match, string(currentMapping.MatchMetricType),
				remainingMappingsCount, currentMapping)

			currentMapping.nameFormatter = fsm.NewTemplateFormatter(currentMapping.Name, captureCount)

			labelKeys := make([]string, len(currentMapping.Labels))
			labelFormatters := make([]*fsm.TemplateFormatter, len(currentMapping.Labels))
			labelIndex := 0
			for label, valueExpr := range currentMapping.Labels {
				labelKeys[labelIndex] = label
				labelFormatters[labelIndex] = fsm.NewTemplateFormatter(valueExpr, captureCount)
				labelIndex++
			}
			currentMapping.labelFormatters = labelFormatters
			currentMapping.labelKeys = labelKeys
		} else {
			if regex, err := regexp.Compile(currentMapping.Match); err != nil {
				return fmt.Errorf("invalid regex %s in mapping: %v", currentMapping.Match, err)
			} else {
				currentMapping.regex = regex
			}
			n.doRegex = true
		}

		if currentMapping.ObserverType == "" {
			currentMapping.ObserverType = n.Defaults.ObserverType
		}

		if currentMapping.LegacyQuantiles != nil &&
			(currentMapping.SummaryOptions == nil || currentMapping.SummaryOptions.Quantiles != nil) {
			level.Warn(m.Logger).Log("msg", "using the top level quantiles is deprecated.  Please use quantiles in the summary_options hierarchy")
		}

		if currentMapping.LegacyBuckets != nil &&
			(currentMapping.HistogramOptions == nil || currentMapping.HistogramOptions.Buckets != nil) {
			level.Warn(m.Logger).Log("msg", "using the top level buckets is deprecated.  Please use buckets in the histogram_options hierarchy")
		}

		if currentMapping.SummaryOptions != nil &&
			currentMapping.LegacyQuantiles != nil &&
			currentMapping.SummaryOptions.Quantiles != nil {
			return fmt.Errorf("cannot use quantiles in both the top level and summary options at the same time in %s", currentMapping.Match)
		}

		if currentMapping.HistogramOptions != nil &&
			currentMapping.LegacyBuckets != nil &&
			currentMapping.HistogramOptions.Buckets != nil {
			return fmt.Errorf("cannot use buckets in both the top level and histogram options at the same time in %s", currentMapping.Match)
		}

		if currentMapping.ObserverType == ObserverTypeHistogram {
			if currentMapping.SummaryOptions != nil {
				return fmt.Errorf("cannot use histogram observer and summary options at the same time")
			}
			if currentMapping.HistogramOptions == nil {
				currentMapping.HistogramOptions = &HistogramOptions{}
			}
			if currentMapping.LegacyBuckets != nil && len(currentMapping.LegacyBuckets) != 0 {
				currentMapping.HistogramOptions.Buckets = currentMapping.LegacyBuckets
			}
			if currentMapping.HistogramOptions.Buckets == nil || len(currentMapping.HistogramOptions.Buckets) == 0 {
				currentMapping.HistogramOptions.Buckets = n.Defaults.HistogramOptions.Buckets
			}
		}

		if currentMapping.ObserverType == ObserverTypeSummary {
			if currentMapping.HistogramOptions != nil {
				return fmt.Errorf("cannot use summary observer and histogram options at the same time")
			}
			if currentMapping.SummaryOptions == nil {
				currentMapping.SummaryOptions = &SummaryOptions{}
			}
			if currentMapping.LegacyQuantiles != nil && len(currentMapping.LegacyQuantiles) != 0 {
				currentMapping.SummaryOptions.Quantiles = currentMapping.LegacyQuantiles
			}
			if currentMapping.SummaryOptions.Quantiles == nil || len(currentMapping.SummaryOptions.Quantiles) == 0 {
				currentMapping.SummaryOptions.Quantiles = n.Defaults.SummaryOptions.Quantiles
			}
			if currentMapping.SummaryOptions.MaxAge == 0 {
				currentMapping.SummaryOptions.MaxAge = n.Defaults.SummaryOptions.MaxAge
			}
			if currentMapping.SummaryOptions.AgeBuckets == 0 {
				currentMapping.SummaryOptions.AgeBuckets = n.Defaults.SummaryOptions.AgeBuckets
			}
			if currentMapping.SummaryOptions.BufCap == 0 {
				currentMapping.SummaryOptions.BufCap = n.Defaults.SummaryOptions.BufCap
			}
		}

		if currentMapping.Ttl == 0 && n.Defaults.Ttl > 0 {
			currentMapping.Ttl = n.Defaults.Ttl
		}
	}

	m.mutex.Lock()
	defer m.mutex.Unlock()

	m.Defaults = n.Defaults
	m.Mappings = n.Mappings

	// Reset the cache since this function can be used to reload config
	if m.cache != nil {
		m.cache.Reset()
	}

	if n.doFSM {
		var mappings []string
		for _, mapping := range n.Mappings {
			if mapping.MatchType == MatchTypeGlob {
				mappings = append(mappings, mapping.Match)
			}
		}
		n.FSM.BacktrackingNeeded = fsm.TestIfNeedBacktracking(mappings, n.FSM.OrderingDisabled, m.Logger)

		m.FSM = n.FSM
		m.doRegex = n.doRegex
	}
	m.doFSM = n.doFSM

	if m.MappingsCount != nil {
		m.MappingsCount.Set(float64(len(n.Mappings)))
	}

	if m.Logger == nil {
		m.Logger = log.NewNopLogger()
	}

	return nil
}

func (m *MetricMapper) InitFromFile(fileName string) error {
	mappingStr, err := os.ReadFile(fileName)
	if err != nil {
		return err
	}

	return m.InitFromYAMLString(string(mappingStr))
}

// UseCache tells the mapper to use a cache that implements the MetricMapperCache interface.
// This cache MUST be thread-safe!
func (m *MetricMapper) UseCache(cache MetricMapperCache) {
	m.mutex.Lock()
	defer m.mutex.Unlock()
	m.cache = cache
}

func (m *MetricMapper) GetMapping(statsdMetric string, statsdMetricType MetricType) (*MetricMapping, prometheus.Labels, bool) {
	m.mutex.RLock()
	defer m.mutex.RUnlock()

	// only use a cache if one is present
	if m.cache != nil {
		result, cached := m.cache.Get(formatKey(statsdMetric, statsdMetricType))
		if cached {
			r := result.(MetricMapperCacheResult)
			return r.Mapping, r.Labels, r.Matched
		}
	}

	// glob matching
	if m.doFSM {
		finalState, captures := m.FSM.GetMapping(statsdMetric, string(statsdMetricType))
		if finalState != nil && finalState.Result != nil {
			v := finalState.Result.(*MetricMapping)
			result := copyMetricMapping(v)
			result.Name = result.nameFormatter.Format(captures)

			labels := prometheus.Labels{}
			for index, formatter := range result.labelFormatters {
				labels[result.labelKeys[index]] = formatter.Format(captures)
			}

			r := MetricMapperCacheResult{
				Mapping: result,
				Matched: true,
				Labels:  labels,
			}
			// add match to cache
			if m.cache != nil {
				m.cache.Add(formatKey(statsdMetric, statsdMetricType), r)
			}

			return result, labels, true
		} else if !m.doRegex {
			// if there's no regex match type, return immediately
			// Add miss to cache
			if m.cache != nil {
				m.cache.Add(formatKey(statsdMetric, statsdMetricType), MetricMapperCacheResult{})
			}
			return nil, nil, false
		}
	}

	// regex matching
	for _, mapping := range m.Mappings {
		// if a rule don't have regex matching type, the regex field is unset
		if mapping.regex == nil {
			continue
		}
		matches := mapping.regex.FindStringSubmatchIndex(statsdMetric)
		if len(matches) == 0 {
			continue
		}

		mapping.Name = string(mapping.regex.ExpandString(
			[]byte{},
			mapping.Name,
			statsdMetric,
			matches,
		))

		if mt := mapping.MatchMetricType; mt != "" && mt != statsdMetricType {
			continue
		}

		labels := prometheus.Labels{}
		for label, valueExpr := range mapping.Labels {
			value := mapping.regex.ExpandString([]byte{}, valueExpr, statsdMetric, matches)
			labels[label] = string(value)
		}

		r := MetricMapperCacheResult{
			Mapping: &mapping,
			Matched: true,
			Labels:  labels,
		}
		// Add Match to cache
		if m.cache != nil {
			m.cache.Add(formatKey(statsdMetric, statsdMetricType), r)
		}

		return &mapping, labels, true
	}

	// Add Miss to cache
	if m.cache != nil {
		m.cache.Add(formatKey(statsdMetric, statsdMetricType), MetricMapperCacheResult{})
	}
	return nil, nil, false
}

// make a shallow copy so that we do not overwrite name
// as multiple names can be matched by same mapping
func copyMetricMapping(in *MetricMapping) *MetricMapping {
	out := *in
	return &out
}
<|endoftext|>


<commit_before>// Copyright 2015 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package pbutil defines interfaces for handling Protocol Buffer objects.
package pbutil

import "github.com/coreos/pkg/capnslog"

var (
	plog = capnslog.NewPackageLogger("github.com/coreos/etcd/pkg", "flags")
)

type Marshaler interface {
	Marshal() (data []byte, err error)
}

type Unmarshaler interface {
	Unmarshal(data []byte) error
}

func MustMarshal(m Marshaler) []byte {
	d, err := m.Marshal()
	if err != nil {
		plog.Panicf("marshal should never fail (%v)", err)
	}
	return d
}

func MustUnmarshal(um Unmarshaler, data []byte) {
	if err := um.Unmarshal(data); err != nil {
		plog.Panicf("unmarshal should never fail (%v)", err)
	}
}

func MaybeUnmarshal(um Unmarshaler, data []byte) bool {
	if err := um.Unmarshal(data); err != nil {
		return false
	}
	return true
}

func GetBool(v *bool) (vv bool, set bool) {
	if v == nil {
		return false, false
	}
	return *v, true
}

func Boolp(b bool) *bool { return &b }
<commit_msg>corrected the package name in logger<commit_after>// Copyright 2015 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package pbutil defines interfaces for handling Protocol Buffer objects.
package pbutil

import "github.com/coreos/pkg/capnslog"

var (
	plog = capnslog.NewPackageLogger("github.com/coreos/etcd/pkg", "pbutil")
)

type Marshaler interface {
	Marshal() (data []byte, err error)
}

type Unmarshaler interface {
	Unmarshal(data []byte) error
}

func MustMarshal(m Marshaler) []byte {
	d, err := m.Marshal()
	if err != nil {
		plog.Panicf("marshal should never fail (%v)", err)
	}
	return d
}

func MustUnmarshal(um Unmarshaler, data []byte) {
	if err := um.Unmarshal(data); err != nil {
		plog.Panicf("unmarshal should never fail (%v)", err)
	}
}

func MaybeUnmarshal(um Unmarshaler, data []byte) bool {
	if err := um.Unmarshal(data); err != nil {
		return false
	}
	return true
}

func GetBool(v *bool) (vv bool, set bool) {
	if v == nil {
		return false, false
	}
	return *v, true
}

func Boolp(b bool) *bool { return &b }
<|endoftext|>