<commit_before>package main

import (
	"crypto/md5"
	"encoding/base64"
	"encoding/hex"
	"errors"
	"fmt"
	"log"
	"os"
	"regexp"
	"strconv"
	"strings"
	"time"
)

type UnixtimeMacro struct {
	duration time.Duration
	format   string
}

var PksInputs map[string]string = make(map[string]string)
var Md5Inputs map[string]string = make(map[string]string)
var Base64Inputs map[string]string = make(map[string]string)
var UnixtimeMacros map[string]UnixtimeMacro
var PrintTimeMacros map[string]UnixtimeMacro
var CommandMacros = make(map[string][]string)
var Md5Macros = make(map[string][]string)
var Base64Macros = make(map[string][]string)
var PksMacros = make(map[string][]string)

var reArgs = regexp.MustCompile("{%ARGS\\[(\\d+)\\]}")

func arrayContains(arr []string, str string) bool {
	i := 0
	for i < len(arr) {
		if str == arr[i] {
			break
		}
		i += 1
	}
	return i < len(arr)

}

func addCommandMacro(cmd string, macro string) {
	if !arrayContains(CommandMacros[cmd], macro) {
		CommandMacros[cmd] = append(CommandMacros[cmd], macro)
	}
}

func InitSessionLogMacros(sessionLog string) {
	// this will create an entry for a command named "\nSessionLog",
	// the newline will prevent any command named "SessionLog" in an ini file from overwriting it
	InitMacros("\nSessionLog", sessionLog)
}

func InitMacros(cmd string, field string) {
	_, exists := CommandMacros[cmd]
	if !exists {
		CommandMacros[cmd] = make([]string, 0)
	}

	rx, _ := regexp.Compile("\\{%.*?\\}")
	rxenv, _ := regexp.Compile("\\{\\$.*?\\}")

	for _, macro := range rx.FindAllString(field, -1) {
		addCommandMacro(cmd, macro)
	}
	for _, macro := range rxenv.FindAllString(field, -1) {
		addCommandMacro(cmd, macro)
	}
}

func parseTimeModifier(arg string) (time.Duration, error) {
	if len(arg) == 0 {
		return time.Duration(0), nil
	}

	rx := regexp.MustCompile("([\\+\\-]\\d+)(.*)")
	parsed := rx.FindStringSubmatch(arg)
	if parsed == nil {
		return time.Duration(0), errors.New(fmt.Sprintf("time modifier %s is not supported", arg))
	} else {
		mult, _ := strconv.Atoi(parsed[1]) // e.g -4, +4, 4
		if parsed[2] == "MONTH" || parsed[2] == "MONTHS" {
			return time.Duration(mult*30*24) * time.Hour, nil
		} else if parsed[2] == "DAY" || parsed[2] == "DAYS" {
			return time.Duration(mult) * 24 * time.Hour, nil
		} else if parsed[2] == "HOUR" || parsed[2] == "HOURS" {
			return time.Duration(mult) * time.Hour, nil
		} else if parsed[2] == "MINUTE" || parsed[2] == "MINUTES" {
			return time.Duration(mult) * time.Minute, nil
		} else if parsed[2] == "SECOND" || parsed[2] == "SECONDS" {
			return time.Duration(mult) * time.Second, nil
		} else {
			return time.Duration(0), errors.New(fmt.Sprintf("time modifier %s is not supported", arg))
		}
	}
}

func initUnixtimeMacro(macro string) {
	declaration := macro[2 : len(macro)-1]
	if strings.HasPrefix(declaration, "UNIXTIME") {
		arg := declaration[8:]
		rx1, _ := regexp.Compile("%(\\d+)?x")
		fmtmatch := rx1.FindString(arg)
		format := "%d"
		if len(fmtmatch) > 0 {
			format = fmtmatch
			arg = strings.Replace(arg, fmtmatch, "", -1)
		}
		duration, err := parseTimeModifier(arg)
		if err != nil {
			log.Fatalf("UNIXTIME macro %s: %s", declaration, err.Error())
		} else {
			UnixtimeMacros[macro] = UnixtimeMacro{duration, format}
		}
	}
}

func initPrintTimeMacro(macro string) {
	declaration := macro[2 : len(macro)-1]
	if strings.HasPrefix(declaration, "TIME") {
		format := "2006-01-02 15:04:05"
		arg := declaration[4:]
		duration, err := parseTimeModifier(arg)
		if err != nil {
			log.Fatalf("TIME macro %s: %s", declaration, err.Error())
		} else {
			PrintTimeMacros[macro] = UnixtimeMacro{duration, format}
		}
	}
}

func InitUnixtimeMacros() {
	PrintTimeMacros = make(map[string]UnixtimeMacro)
	UnixtimeMacros = make(map[string]UnixtimeMacro)
	for _, macros := range CommandMacros {
		for _, macro := range macros {
			initUnixtimeMacro(macro)
			initPrintTimeMacro(macro)
		}
	}
	for _, macros := range Base64Macros {
		for _, macro := range macros {
			initUnixtimeMacro(macro)
			initPrintTimeMacro(macro)
		}
	}
	for _, macros := range Md5Macros {
		for _, macro := range macros {
			initUnixtimeMacro(macro)
			initPrintTimeMacro(macro)
		}
	}
}

func addPksMacro(cmd string, macro string) {
	if !arrayContains(PksMacros[cmd], macro) {
		PksMacros[cmd] = append(PksMacros[cmd], macro)
	}
}

func InitPksMacro(cmd string, pksInput string) {
	if len(pksInput) == 0 {
		return
	}
	PksInputs[cmd] = pksInput
	PksMacros[cmd] = make([]string, 0)

	rx, _ := regexp.Compile("\\{%.*?\\}")
	rxenv, _ := regexp.Compile("\\{\\$.*?\\}")

	for _, macro := range rx.FindAllString(pksInput, -1) {
		addPksMacro(cmd, macro)
	}
	for _, macro := range rxenv.FindAllString(pksInput, -1) {
		addPksMacro(cmd, macro)
	}
} //InitPKSMacro
/*
func InitFuncMacro(cmd string, funcInput string) {
	if len(funcInput) == 0 {
		return
	}
	PksMacros[cmd] = make([]string, 0)

	rx, _ := regexp.Compile("\\{%.*?\\}")
	rxenv, _ := regexp.Compile("\\{\\$.*?\\}")

	for _, macro := range rx.FindAllString(funcInput, -1) {
		addPksMacro(cmd, macro)
	}
	for _, macro := range rxenv.FindAllString(funcInput, -1) {
		addPksMacro(cmd, macro)
	}
} //InitFuncMacro
*/
func addMd5Macro(cmd string, macro string) {
	if !arrayContains(Md5Macros[cmd], macro) {
		Md5Macros[cmd] = append(Md5Macros[cmd], macro)
	}
}

func InitMd5Macro(cmd string, md5Input string) {
	if len(md5Input) == 0 {
		return
	}

	Md5Inputs[cmd] = md5Input
	Md5Macros[cmd] = make([]string, 0)

	rx, _ := regexp.Compile("\\{%.*?\\}")
	rxenv, _ := regexp.Compile("\\{\\$.*?\\}")

	for _, macro := range rx.FindAllString(md5Input, -1) {
		addMd5Macro(cmd, macro)
	}
	for _, macro := range rxenv.FindAllString(md5Input, -1) {
		addMd5Macro(cmd, macro)
	}
}

func addBase64Macro(cmd string, macro string) {
	if !arrayContains(Base64Macros[cmd], macro) {
		Base64Macros[cmd] = append(Base64Macros[cmd], macro)
	}
}

func InitBase64Macro(cmd string, base64Input string) {
	if len(base64Input) == 0 {
		return
	}

	Base64Inputs[cmd] = base64Input
	Base64Macros[cmd] = make([]string, 0)

	rx, _ := regexp.Compile("\\{%.*?\\}")
	rxenv, _ := regexp.Compile("\\{\\$.*?\\}")

	for _, macro := range rx.FindAllString(base64Input, -1) {
		addBase64Macro(cmd, macro)
	}
	for _, macro := range rxenv.FindAllString(base64Input, -1) {
		addBase64Macro(cmd, macro)
	}
}

func _runnerMacro(command string, declaration string, inputData string, sessionVars map[string]string, reqTime time.Time) string {
	if !(strings.HasPrefix(declaration, "{%") || strings.HasPrefix(declaration, "{$")) || !strings.HasSuffix(declaration, "}") {
		return ""
	}
	//This func processes the token and returns the string
	uxt, ok := UnixtimeMacros[declaration]
	prt, ok1 := PrintTimeMacros[declaration]
	if ok {
		timestamp := reqTime.Add(uxt.duration).UnixNano() / (int64(time.Millisecond) / int64(time.Nanosecond)) //why not use now instead of reqTime?
		rx, _ := regexp.Compile("%(\\d+)x")
		fmtdigits := rx.FindStringSubmatch(uxt.format)
		if len(fmtdigits) == 0 {
			return fmt.Sprintf(uxt.format, timestamp)
		} else {
			fmtnum, _ := strconv.Atoi(fmtdigits[1])
			if fmtnum >= 12 {
				return fmt.Sprintf(uxt.format, timestamp)
			} else {
				tmp := fmt.Sprintf("%012x", timestamp)
				return tmp[0:fmtnum]
			}
		}
	} else if ok1 {
		return reqTime.Add(prt.duration).Format(prt.format)
	} else if declaration == "{%MD5SUM}" {
		testMd5 := Md5Inputs[command]
		for _, macro := range Md5Macros[command] {
			testMd5 = strings.Replace(testMd5, macro, runnerMacro(command, macro, inputData, sessionVars, reqTime), -1)
		}
		return strings.ToUpper(fmt.Sprintf("%x", md5.Sum([]byte(testMd5))))
	} else if declaration == "{%PKSENC}" {
		pksInput := PksInputs[command]
		for _, macro := range PksMacros[command] {
			pksInput = strings.Replace(pksInput, macro, runnerMacro(command, macro, inputData, sessionVars, reqTime), -1)
		}

		// that's dirty, waiting for proper func management
		inputs := strings.Split(pksInput, ",")
		if len(inputs) != 3 {
			return "invalid PKSInput format. Must be pwd,key,keyexp"
		}
		encryptor, err := NewPKSEncryptor(inputs[1], inputs[2], inputs[0])
		if err != nil {
			return err.Error()
		}
		res, err := encryptor.Encrypt()
		if err != nil {
			return err.Error()
		}
		return hex.EncodeToString(res)
	} else if declaration == "{%BASE64ENC}" {
		base64In := Base64Inputs[command]
		for _, macro := range Base64Macros[command] {
			base64In = strings.Replace(base64In, macro, runnerMacro(command, macro, inputData, sessionVars, reqTime), -1)
		}
		return base64.StdEncoding.EncodeToString([]byte(base64In))
	} else if strings.HasPrefix(declaration, "{$") {
		// an env var macro like {$SECRET}
		return os.Getenv(declaration[2 : len(declaration)-1])
	} else {
		// Check if it match {%ARGS[X]}
		argsIndex := reArgs.FindStringSubmatch(declaration) // regexp.MustCompile("{%ARGS\\[(\\d+)\\]}")
		if len(argsIndex) > 0 {
			i, _ := strconv.Atoi(argsIndex[1])
			arr := strings.Split(inputData, delimeter)
			if i >= len(arr) {
				// TODO : print error ?
				return ""
			}
			return arr[i]
		}
		// Check if it match a session var
		if declaration[1] == '%' {
			session_var := declaration[2 : len(declaration)-1]
			val, ok := sessionVars[session_var]
			if ok {
				return val
			}
		}
	}
	return ""
} //_runnerMacro replaces the variable reference like {%X} or {$X} with the value stored in the hash table

func runnerMacro(command string, declaration string, inputData string, sessionVars map[string]string, reqTime time.Time) string {
	if !(strings.HasPrefix(declaration, "{%") || strings.HasPrefix(declaration, "{$")) || !strings.HasSuffix(declaration, "}") {
		return ""
	} //This functions gets passed in a {%X} and {$X} variable reference, looks it up the hash table and returns the string value

	ssrx, _ := regexp.Compile("\\[(\\d+):(\\d+)\\]}")
	declSubstr := ssrx.FindStringSubmatch(declaration)
	if len(declSubstr) == 0 {
		return _runnerMacro(command, declaration, inputData, sessionVars, reqTime)
	} else {
		declaration = strings.Replace(declaration, declSubstr[0], "}", 1)
		result := _runnerMacro(command, declaration, inputData, sessionVars, reqTime)
		ss0, _ := strconv.Atoi(declSubstr[1])
		ss1, _ := strconv.Atoi(declSubstr[2])
		return result[ss0:ss1]
	}
}

func RunnerMacros(command string, inputData string, sessionVars map[string]string, reqTime time.Time, field string) string {
	for _, macro := range CommandMacros[command] {
		field = strings.Replace(field, macro, runnerMacro(command, macro, inputData, sessionVars, reqTime), -1)
	}
	return field
}

func RunnerMacrosRegexp(command string, inputData string, sessionVars map[string]string, reqTime time.Time, field string) string {
	for _, macro := range CommandMacros[command] {
		replacement := regexp.QuoteMeta(runnerMacro(command, macro, inputData, sessionVars, reqTime))
		field = strings.Replace(field, macro, replacement, -1)
	}
	return field
}

func SessionLogMacros(inputData string, sessionVars map[string]string, logTime time.Time, initial string) string {
	return RunnerMacros("\nSessionLog", inputData, sessionVars, logTime, initial)
}
<commit_msg>adding javascript macro support cleanup big comment<commit_after>package main

import (
	"crypto/md5"
	"encoding/base64"
	"encoding/hex"
	"errors"
	"fmt"
	"log"
	"os"
	"regexp"
	"strconv"
	"strings"
	"time"
)

type UnixtimeMacro struct {
	duration time.Duration
	format   string
}

var PksInputs map[string]string = make(map[string]string)
var Md5Inputs map[string]string = make(map[string]string)
var Base64Inputs map[string]string = make(map[string]string)
var UnixtimeMacros map[string]UnixtimeMacro
var PrintTimeMacros map[string]UnixtimeMacro
var CommandMacros = make(map[string][]string)
var Md5Macros = make(map[string][]string)
var Base64Macros = make(map[string][]string)
var PksMacros = make(map[string][]string)

var reArgs = regexp.MustCompile("{%ARGS\\[(\\d+)\\]}")

func arrayContains(arr []string, str string) bool {
	i := 0
	for i < len(arr) {
		if str == arr[i] {
			break
		}
		i += 1
	}
	return i < len(arr)

}

func addCommandMacro(cmd string, macro string) {
	if !arrayContains(CommandMacros[cmd], macro) {
		CommandMacros[cmd] = append(CommandMacros[cmd], macro)
	}
}

func InitSessionLogMacros(sessionLog string) {
	// this will create an entry for a command named "\nSessionLog",
	// the newline will prevent any command named "SessionLog" in an ini file from overwriting it
	InitMacros("\nSessionLog", sessionLog)
}

func InitMacros(cmd string, field string) {
	_, exists := CommandMacros[cmd]
	if !exists {
		CommandMacros[cmd] = make([]string, 0)
	}

	rx, _ := regexp.Compile("\\{%.*?\\}")
	rxenv, _ := regexp.Compile("\\{\\$.*?\\}")

	for _, macro := range rx.FindAllString(field, -1) {
		addCommandMacro(cmd, macro)
	}
	for _, macro := range rxenv.FindAllString(field, -1) {
		addCommandMacro(cmd, macro)
	}
}

func parseTimeModifier(arg string) (time.Duration, error) {
	if len(arg) == 0 {
		return time.Duration(0), nil
	}

	rx := regexp.MustCompile("([\\+\\-]\\d+)(.*)")
	parsed := rx.FindStringSubmatch(arg)
	if parsed == nil {
		return time.Duration(0), errors.New(fmt.Sprintf("time modifier %s is not supported", arg))
	} else {
		mult, _ := strconv.Atoi(parsed[1]) // e.g -4, +4, 4
		if parsed[2] == "MONTH" || parsed[2] == "MONTHS" {
			return time.Duration(mult*30*24) * time.Hour, nil
		} else if parsed[2] == "DAY" || parsed[2] == "DAYS" {
			return time.Duration(mult) * 24 * time.Hour, nil
		} else if parsed[2] == "HOUR" || parsed[2] == "HOURS" {
			return time.Duration(mult) * time.Hour, nil
		} else if parsed[2] == "MINUTE" || parsed[2] == "MINUTES" {
			return time.Duration(mult) * time.Minute, nil
		} else if parsed[2] == "SECOND" || parsed[2] == "SECONDS" {
			return time.Duration(mult) * time.Second, nil
		} else {
			return time.Duration(0), errors.New(fmt.Sprintf("time modifier %s is not supported", arg))
		}
	}
}

func initUnixtimeMacro(macro string) {
	declaration := macro[2 : len(macro)-1]
	if strings.HasPrefix(declaration, "UNIXTIME") {
		arg := declaration[8:]
		rx1, _ := regexp.Compile("%(\\d+)?x")
		fmtmatch := rx1.FindString(arg)
		format := "%d"
		if len(fmtmatch) > 0 {
			format = fmtmatch
			arg = strings.Replace(arg, fmtmatch, "", -1)
		}
		duration, err := parseTimeModifier(arg)
		if err != nil {
			log.Fatalf("UNIXTIME macro %s: %s", declaration, err.Error())
		} else {
			UnixtimeMacros[macro] = UnixtimeMacro{duration, format}
		}
	}
}

func initPrintTimeMacro(macro string) {
	declaration := macro[2 : len(macro)-1]
	if strings.HasPrefix(declaration, "TIME") {
		format := "2006-01-02 15:04:05"
		arg := declaration[4:]
		duration, err := parseTimeModifier(arg)
		if err != nil {
			log.Fatalf("TIME macro %s: %s", declaration, err.Error())
		} else {
			PrintTimeMacros[macro] = UnixtimeMacro{duration, format}
		}
	}
}

func InitUnixtimeMacros() {
	PrintTimeMacros = make(map[string]UnixtimeMacro)
	UnixtimeMacros = make(map[string]UnixtimeMacro)
	for _, macros := range CommandMacros {
		for _, macro := range macros {
			initUnixtimeMacro(macro)
			initPrintTimeMacro(macro)
		}
	}
	for _, macros := range Base64Macros {
		for _, macro := range macros {
			initUnixtimeMacro(macro)
			initPrintTimeMacro(macro)
		}
	}
	for _, macros := range Md5Macros {
		for _, macro := range macros {
			initUnixtimeMacro(macro)
			initPrintTimeMacro(macro)
		}
	}
}

func addPksMacro(cmd string, macro string) {
	if !arrayContains(PksMacros[cmd], macro) {
		PksMacros[cmd] = append(PksMacros[cmd], macro)
	}
}

func InitPksMacro(cmd string, pksInput string) {
	if len(pksInput) == 0 {
		return
	}
	PksInputs[cmd] = pksInput
	PksMacros[cmd] = make([]string, 0)

	rx, _ := regexp.Compile("\\{%.*?\\}")
	rxenv, _ := regexp.Compile("\\{\\$.*?\\}")

	for _, macro := range rx.FindAllString(pksInput, -1) {
		addPksMacro(cmd, macro)
	}
	for _, macro := range rxenv.FindAllString(pksInput, -1) {
		addPksMacro(cmd, macro)
	}
} //InitPKSMacro
func addMd5Macro(cmd string, macro string) {
	if !arrayContains(Md5Macros[cmd], macro) {
		Md5Macros[cmd] = append(Md5Macros[cmd], macro)
	}
}

func InitMd5Macro(cmd string, md5Input string) {
	if len(md5Input) == 0 {
		return
	}

	Md5Inputs[cmd] = md5Input
	Md5Macros[cmd] = make([]string, 0)

	rx, _ := regexp.Compile("\\{%.*?\\}")
	rxenv, _ := regexp.Compile("\\{\\$.*?\\}")

	for _, macro := range rx.FindAllString(md5Input, -1) {
		addMd5Macro(cmd, macro)
	}
	for _, macro := range rxenv.FindAllString(md5Input, -1) {
		addMd5Macro(cmd, macro)
	}
}

func addBase64Macro(cmd string, macro string) {
	if !arrayContains(Base64Macros[cmd], macro) {
		Base64Macros[cmd] = append(Base64Macros[cmd], macro)
	}
}

func InitBase64Macro(cmd string, base64Input string) {
	if len(base64Input) == 0 {
		return
	}

	Base64Inputs[cmd] = base64Input
	Base64Macros[cmd] = make([]string, 0)

	rx, _ := regexp.Compile("\\{%.*?\\}")
	rxenv, _ := regexp.Compile("\\{\\$.*?\\}")

	for _, macro := range rx.FindAllString(base64Input, -1) {
		addBase64Macro(cmd, macro)
	}
	for _, macro := range rxenv.FindAllString(base64Input, -1) {
		addBase64Macro(cmd, macro)
	}
}

func _runnerMacro(command string, declaration string, inputData string, sessionVars map[string]string, reqTime time.Time) string {
	if !(strings.HasPrefix(declaration, "{%") || strings.HasPrefix(declaration, "{$")) || !strings.HasSuffix(declaration, "}") {
		return ""
	}
	//This func processes the token and returns the string
	uxt, ok := UnixtimeMacros[declaration]
	prt, ok1 := PrintTimeMacros[declaration]
	if ok {
		timestamp := reqTime.Add(uxt.duration).UnixNano() / (int64(time.Millisecond) / int64(time.Nanosecond)) //why not use now instead of reqTime?
		rx, _ := regexp.Compile("%(\\d+)x")
		fmtdigits := rx.FindStringSubmatch(uxt.format)
		if len(fmtdigits) == 0 {
			return fmt.Sprintf(uxt.format, timestamp)
		} else {
			fmtnum, _ := strconv.Atoi(fmtdigits[1])
			if fmtnum >= 12 {
				return fmt.Sprintf(uxt.format, timestamp)
			} else {
				tmp := fmt.Sprintf("%012x", timestamp)
				return tmp[0:fmtnum]
			}
		}
	} else if ok1 {
		return reqTime.Add(prt.duration).Format(prt.format)
	} else if declaration == "{%MD5SUM}" {
		testMd5 := Md5Inputs[command]
		for _, macro := range Md5Macros[command] {
			testMd5 = strings.Replace(testMd5, macro, runnerMacro(command, macro, inputData, sessionVars, reqTime), -1)
		}
		return strings.ToUpper(fmt.Sprintf("%x", md5.Sum([]byte(testMd5))))
	} else if declaration == "{%PKSENC}" {
		pksInput := PksInputs[command]
		for _, macro := range PksMacros[command] {
			pksInput = strings.Replace(pksInput, macro, runnerMacro(command, macro, inputData, sessionVars, reqTime), -1)
		}

		// that's dirty, waiting for proper func management
		inputs := strings.Split(pksInput, ",")
		if len(inputs) != 3 {
			return "invalid PKSInput format. Must be pwd,key,keyexp"
		}
		encryptor, err := NewPKSEncryptor(inputs[1], inputs[2], inputs[0])
		if err != nil {
			return err.Error()
		}
		res, err := encryptor.Encrypt()
		if err != nil {
			return err.Error()
		}
		return hex.EncodeToString(res)
	} else if declaration == "{%BASE64ENC}" {
		base64In := Base64Inputs[command]
		for _, macro := range Base64Macros[command] {
			base64In = strings.Replace(base64In, macro, runnerMacro(command, macro, inputData, sessionVars, reqTime), -1)
		}
		return base64.StdEncoding.EncodeToString([]byte(base64In))
	} else if strings.HasPrefix(declaration, "{$") {
		// an env var macro like {$SECRET}
		return os.Getenv(declaration[2 : len(declaration)-1])
	} else {
		// Check if it match {%ARGS[X]}
		argsIndex := reArgs.FindStringSubmatch(declaration) // regexp.MustCompile("{%ARGS\\[(\\d+)\\]}")
		if len(argsIndex) > 0 {
			i, _ := strconv.Atoi(argsIndex[1])
			arr := strings.Split(inputData, delimeter)
			if i >= len(arr) {
				// TODO : print error ?
				return ""
			}
			return arr[i]
		}
		// Check if it match a session var
		if declaration[1] == '%' {
			session_var := declaration[2 : len(declaration)-1]
			val, ok := sessionVars[session_var]
			if ok {
				return val
			}
		}
	}
	return ""
} //_runnerMacro replaces the variable reference like {%X} or {$X} with the value stored in the hash table

func runnerMacro(command string, declaration string, inputData string, sessionVars map[string]string, reqTime time.Time) string {
	if !(strings.HasPrefix(declaration, "{%") || strings.HasPrefix(declaration, "{$")) || !strings.HasSuffix(declaration, "}") {
		return ""
	} //This functions gets passed in a {%X} and {$X} variable reference, looks it up the hash table and returns the string value

	ssrx, _ := regexp.Compile("\\[(\\d+):(\\d+)\\]}")
	declSubstr := ssrx.FindStringSubmatch(declaration)
	if len(declSubstr) == 0 {
		return _runnerMacro(command, declaration, inputData, sessionVars, reqTime)
	} else {
		declaration = strings.Replace(declaration, declSubstr[0], "}", 1)
		result := _runnerMacro(command, declaration, inputData, sessionVars, reqTime)
		ss0, _ := strconv.Atoi(declSubstr[1])
		ss1, _ := strconv.Atoi(declSubstr[2])
		return result[ss0:ss1]
	}
}

func RunnerMacros(command string, inputData string, sessionVars map[string]string, reqTime time.Time, field string) string {
	for _, macro := range CommandMacros[command] {
		field = strings.Replace(field, macro, runnerMacro(command, macro, inputData, sessionVars, reqTime), -1)
	}
	return field
}

func RunnerMacrosRegexp(command string, inputData string, sessionVars map[string]string, reqTime time.Time, field string) string {
	for _, macro := range CommandMacros[command] {
		replacement := regexp.QuoteMeta(runnerMacro(command, macro, inputData, sessionVars, reqTime))
		field = strings.Replace(field, macro, replacement, -1)
	}
	return field
}

func SessionLogMacros(inputData string, sessionVars map[string]string, logTime time.Time, initial string) string {
	return RunnerMacros("\nSessionLog", inputData, sessionVars, logTime, initial)
}
<|endoftext|>


<commit_before>package jwtauth_test

import (
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/http/httptest"
	"testing"
	"time"

	"github.com/dgrijalva/jwt-go"
	"github.com/goware/jwtauth"
	"github.com/pressly/chi"
	"golang.org/x/net/context"
)

var (
	TokenAuth   *jwtauth.JwtAuth
	TokenSecret = []byte("secretpass")
)

func init() {
	TokenAuth = jwtauth.New("HS256", []byte("secretpass"), nil)
}

//
// Tests
//

func TestSimple(t *testing.T) {
	r := chi.NewRouter()

	r.Use(TokenAuth.Verifier)
	r.Use(jwtauth.Authenticator)

	r.Get("/", func(ctx context.Context, w http.ResponseWriter, r *http.Request) {
		w.Write([]byte("welcome"))
	})

	ts := httptest.NewServer(r)
	defer ts.Close()

	// sending unauthorized requests
	if resp := testRequest(t, ts, "GET", "/", nil, nil); resp != "Unauthorized\n" {
		t.Fatalf(resp)
	}

	h := http.Header{}
	h.Set("Authorization", "BEARER "+newJwtToken([]byte("wrong"), map[string]interface{}{}))
	if resp := testRequest(t, ts, "GET", "/", h, nil); resp != "Unauthorized\n" {
		t.Fatalf(resp)
	}
	h.Set("Authorization", "BEARER asdf")
	if resp := testRequest(t, ts, "GET", "/", h, nil); resp != "Unauthorized\n" {
		t.Fatalf(resp)
	}

	// sending authorized requests
	if resp := testRequest(t, ts, "GET", "/", newAuthHeader(), nil); resp != "welcome" {
		t.Fatalf(resp)
	}
}

func TestMore(t *testing.T) {
	r := chi.NewRouter()

	// Protected routes
	r.Group(func(r chi.Router) {
		r.Use(TokenAuth.Verifier)

		authenticator := func(next chi.Handler) chi.Handler {
			return chi.HandlerFunc(func(ctx context.Context, w http.ResponseWriter, r *http.Request) {
				if jwtErr, ok := ctx.Value("jwt.err").(error); ok {
					switch jwtErr {
					default:
						log.Println("...we're expired... I think..:", jwtErr)
						http.Error(w, http.StatusText(401), 401)
						return
					case jwtauth.ErrExpired:
						http.Error(w, "expired", 401)
						return
					case jwtauth.ErrUnauthorized:
						http.Error(w, http.StatusText(401), 401)
						return
					case nil:
						// no error
					}
				}

				jwtToken, ok := ctx.Value("jwt").(*jwt.Token)
				if !ok || jwtToken == nil || !jwtToken.Valid {
					log.Println("jwt token..........?", jwtToken)
					http.Error(w, http.StatusText(401), 401)
					return
				}

				// Token is authenticated, pass it through
				next.ServeHTTPC(ctx, w, r)
			})
		}
		r.Use(authenticator)

		r.Get("/admin", func(ctx context.Context, w http.ResponseWriter, r *http.Request) {
			w.Write([]byte("protected"))
		})
	})

	// Public routes
	r.Group(func(r chi.Router) {
		r.Get("/", func(ctx context.Context, w http.ResponseWriter, r *http.Request) {
			w.Write([]byte("welcome"))
		})
	})

	ts := httptest.NewServer(r)
	defer ts.Close()

	// sending unauthorized requests
	if resp := testRequest(t, ts, "GET", "/admin", nil, nil); resp != "Unauthorized\n" {
		t.Fatalf(resp)
	}

	h := http.Header{}
	h.Set("Authorization", "BEARER "+newJwtToken([]byte("wrong"), map[string]interface{}{}))
	if resp := testRequest(t, ts, "GET", "/admin", h, nil); resp != "Unauthorized\n" {
		t.Fatalf(resp)
	}
	h.Set("Authorization", "BEARER asdf")
	if resp := testRequest(t, ts, "GET", "/admin", h, nil); resp != "Unauthorized\n" {
		t.Fatalf(resp)
	}

	h = newAuthHeader((jwtauth.Claims{}).Set("exp", jwtauth.EpochNow()-1000))
	if resp := testRequest(t, ts, "GET", "/admin", h, nil); resp != "expired\n" {
		t.Fatalf(resp)
	}

	// sending authorized requests
	if resp := testRequest(t, ts, "GET", "/", nil, nil); resp != "welcome" {
		t.Fatalf(resp)
	}

	h = newAuthHeader((jwtauth.Claims{}).SetExpiryIn(5 * time.Minute))
	if resp := testRequest(t, ts, "GET", "/admin", h, nil); resp != "protected" {
		t.Fatalf(resp)
	}
}

//
// Test helper functions
//

func testRequest(t *testing.T, ts *httptest.Server, method, path string, header http.Header, body io.Reader) string {
	req, err := http.NewRequest(method, ts.URL+path, body)
	if err != nil {
		t.Fatal(err)
		return ""
	}

	if header != nil {
		for k, v := range header {
			req.Header.Set(k, v[0])
		}
	}

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		t.Fatal(err)
		return ""
	}

	respBody, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		t.Fatal(err)
		return ""
	}
	defer resp.Body.Close()

	return string(respBody)
}

func newJwtToken(secret []byte, claims ...jwtauth.Claims) string {
	token := jwt.New(jwt.GetSigningMethod("HS256"))
	if len(claims) > 0 {
		for k, v := range claims[0] {
			token.Claims[k] = v
		}
	}
	tokenStr, err := token.SignedString(secret)
	if err != nil {
		log.Fatal(err)
	}
	return tokenStr
}

func newAuthHeader(claims ...jwtauth.Claims) http.Header {
	h := http.Header{}
	h.Set("Authorization", "BEARER "+newJwtToken(TokenSecret, claims...))
	return h
}
<commit_msg>Remove debug statements<commit_after>package jwtauth_test

import (
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/http/httptest"
	"testing"
	"time"

	"github.com/dgrijalva/jwt-go"
	"github.com/goware/jwtauth"
	"github.com/pressly/chi"
	"golang.org/x/net/context"
)

var (
	TokenAuth   *jwtauth.JwtAuth
	TokenSecret = []byte("secretpass")
)

func init() {
	TokenAuth = jwtauth.New("HS256", []byte("secretpass"), nil)
}

//
// Tests
//

func TestSimple(t *testing.T) {
	r := chi.NewRouter()

	r.Use(TokenAuth.Verifier)
	r.Use(jwtauth.Authenticator)

	r.Get("/", func(ctx context.Context, w http.ResponseWriter, r *http.Request) {
		w.Write([]byte("welcome"))
	})

	ts := httptest.NewServer(r)
	defer ts.Close()

	// sending unauthorized requests
	if resp := testRequest(t, ts, "GET", "/", nil, nil); resp != "Unauthorized\n" {
		t.Fatalf(resp)
	}

	h := http.Header{}
	h.Set("Authorization", "BEARER "+newJwtToken([]byte("wrong"), map[string]interface{}{}))
	if resp := testRequest(t, ts, "GET", "/", h, nil); resp != "Unauthorized\n" {
		t.Fatalf(resp)
	}
	h.Set("Authorization", "BEARER asdf")
	if resp := testRequest(t, ts, "GET", "/", h, nil); resp != "Unauthorized\n" {
		t.Fatalf(resp)
	}

	// sending authorized requests
	if resp := testRequest(t, ts, "GET", "/", newAuthHeader(), nil); resp != "welcome" {
		t.Fatalf(resp)
	}
}

func TestMore(t *testing.T) {
	r := chi.NewRouter()

	// Protected routes
	r.Group(func(r chi.Router) {
		r.Use(TokenAuth.Verifier)

		authenticator := func(next chi.Handler) chi.Handler {
			return chi.HandlerFunc(func(ctx context.Context, w http.ResponseWriter, r *http.Request) {
				if jwtErr, ok := ctx.Value("jwt.err").(error); ok {
					switch jwtErr {
					default:
						http.Error(w, http.StatusText(401), 401)
						return
					case jwtauth.ErrExpired:
						http.Error(w, "expired", 401)
						return
					case jwtauth.ErrUnauthorized:
						http.Error(w, http.StatusText(401), 401)
						return
					case nil:
						// no error
					}
				}

				jwtToken, ok := ctx.Value("jwt").(*jwt.Token)
				if !ok || jwtToken == nil || !jwtToken.Valid {
					http.Error(w, http.StatusText(401), 401)
					return
				}

				// Token is authenticated, pass it through
				next.ServeHTTPC(ctx, w, r)
			})
		}
		r.Use(authenticator)

		r.Get("/admin", func(ctx context.Context, w http.ResponseWriter, r *http.Request) {
			w.Write([]byte("protected"))
		})
	})

	// Public routes
	r.Group(func(r chi.Router) {
		r.Get("/", func(ctx context.Context, w http.ResponseWriter, r *http.Request) {
			w.Write([]byte("welcome"))
		})
	})

	ts := httptest.NewServer(r)
	defer ts.Close()

	// sending unauthorized requests
	if resp := testRequest(t, ts, "GET", "/admin", nil, nil); resp != "Unauthorized\n" {
		t.Fatalf(resp)
	}

	h := http.Header{}
	h.Set("Authorization", "BEARER "+newJwtToken([]byte("wrong"), map[string]interface{}{}))
	if resp := testRequest(t, ts, "GET", "/admin", h, nil); resp != "Unauthorized\n" {
		t.Fatalf(resp)
	}
	h.Set("Authorization", "BEARER asdf")
	if resp := testRequest(t, ts, "GET", "/admin", h, nil); resp != "Unauthorized\n" {
		t.Fatalf(resp)
	}

	h = newAuthHeader((jwtauth.Claims{}).Set("exp", jwtauth.EpochNow()-1000))
	if resp := testRequest(t, ts, "GET", "/admin", h, nil); resp != "expired\n" {
		t.Fatalf(resp)
	}

	// sending authorized requests
	if resp := testRequest(t, ts, "GET", "/", nil, nil); resp != "welcome" {
		t.Fatalf(resp)
	}

	h = newAuthHeader((jwtauth.Claims{}).SetExpiryIn(5 * time.Minute))
	if resp := testRequest(t, ts, "GET", "/admin", h, nil); resp != "protected" {
		t.Fatalf(resp)
	}
}

//
// Test helper functions
//

func testRequest(t *testing.T, ts *httptest.Server, method, path string, header http.Header, body io.Reader) string {
	req, err := http.NewRequest(method, ts.URL+path, body)
	if err != nil {
		t.Fatal(err)
		return ""
	}

	if header != nil {
		for k, v := range header {
			req.Header.Set(k, v[0])
		}
	}

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		t.Fatal(err)
		return ""
	}

	respBody, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		t.Fatal(err)
		return ""
	}
	defer resp.Body.Close()

	return string(respBody)
}

func newJwtToken(secret []byte, claims ...jwtauth.Claims) string {
	token := jwt.New(jwt.GetSigningMethod("HS256"))
	if len(claims) > 0 {
		for k, v := range claims[0] {
			token.Claims[k] = v
		}
	}
	tokenStr, err := token.SignedString(secret)
	if err != nil {
		log.Fatal(err)
	}
	return tokenStr
}

func newAuthHeader(claims ...jwtauth.Claims) http.Header {
	h := http.Header{}
	h.Set("Authorization", "BEARER "+newJwtToken(TokenSecret, claims...))
	return h
}
<|endoftext|>


<commit_before>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package app implements a server that runs a set of active
// components.  This includes replication controllers, service endpoints and
// nodes.
package app

import (
	"net"
	"net/http"
	"strconv"
	"time"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/resource"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider"
	nodeControllerPkg "github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider/controller"
	replicationControllerPkg "github.com/GoogleCloudPlatform/kubernetes/pkg/controller"
	_ "github.com/GoogleCloudPlatform/kubernetes/pkg/healthz"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/master/ports"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/resourcequota"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/service"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"

	"github.com/golang/glog"
	"github.com/spf13/pflag"
)

// CMServer is the mail context object for the controller manager.
type CMServer struct {
	Port                    int
	Address                 util.IP
	ClientConfig            client.Config
	CloudProvider           string
	CloudConfigFile         string
	MinionRegexp            string
	NodeSyncPeriod          time.Duration
	ResourceQuotaSyncPeriod time.Duration
	RegisterRetryCount      int
	MachineList             util.StringList
	SyncNodeList            bool
	PodEvictionTimeout      time.Duration

	// TODO: Discover these by pinging the host machines, and rip out these params.
	NodeMilliCPU int64
	NodeMemory   resource.Quantity

	KubeletConfig client.KubeletConfig
}

// NewCMServer creates a new CMServer with a default config.
func NewCMServer() *CMServer {
	s := CMServer{
		Port:                    ports.ControllerManagerPort,
		Address:                 util.IP(net.ParseIP("127.0.0.1")),
		NodeSyncPeriod:          10 * time.Second,
		ResourceQuotaSyncPeriod: 10 * time.Second,
		RegisterRetryCount:      10,
		PodEvictionTimeout:      5 * time.Minute,
		NodeMilliCPU:            1000,
		NodeMemory:              resource.MustParse("3Gi"),
		SyncNodeList:            true,
		KubeletConfig: client.KubeletConfig{
			Port:        ports.KubeletPort,
			EnableHttps: false,
		},
	}
	return &s
}

// AddFlags adds flags for a specific CMServer to the specified FlagSet
func (s *CMServer) AddFlags(fs *pflag.FlagSet) {
	fs.IntVar(&s.Port, "port", s.Port, "The port that the controller-manager's http service runs on")
	fs.Var(&s.Address, "address", "The IP address to serve on (set to 0.0.0.0 for all interfaces)")
	client.BindClientConfigFlags(fs, &s.ClientConfig)
	fs.StringVar(&s.CloudProvider, "cloud_provider", s.CloudProvider, "The provider for cloud services.  Empty string for no provider.")
	fs.StringVar(&s.CloudConfigFile, "cloud_config", s.CloudConfigFile, "The path to the cloud provider configuration file.  Empty string for no configuration file.")
	fs.StringVar(&s.MinionRegexp, "minion_regexp", s.MinionRegexp, "If non empty, and --cloud_provider is specified, a regular expression for matching minion VMs.")
	fs.DurationVar(&s.NodeSyncPeriod, "node_sync_period", s.NodeSyncPeriod, ""+
		"The period for syncing nodes from cloudprovider. Longer periods will result in "+
		"fewer calls to cloud provider, but may delay addition of new nodes to cluster.")
	fs.DurationVar(&s.ResourceQuotaSyncPeriod, "resource_quota_sync_period", s.ResourceQuotaSyncPeriod, "The period for syncing quota usage status in the system")
	fs.DurationVar(&s.PodEvictionTimeout, "pod_eviction_timeout", s.PodEvictionTimeout, "The grace peroid for deleting pods on failed nodes.")
	fs.IntVar(&s.RegisterRetryCount, "register_retry_count", s.RegisterRetryCount, ""+
		"The number of retries for initial node registration.  Retry interval equals node_sync_period.")
	fs.Var(&s.MachineList, "machines", "List of machines to schedule onto, comma separated.")
	fs.BoolVar(&s.SyncNodeList, "sync_nodes", s.SyncNodeList, "If true, and --cloud_provider is specified, sync nodes from the cloud provider. Default true.")
	// TODO: Discover these by pinging the host machines, and rip out these flags.
	// TODO: in the meantime, use resource.QuantityFlag() instead of these
	fs.Int64Var(&s.NodeMilliCPU, "node_milli_cpu", s.NodeMilliCPU, "The amount of MilliCPU provisioned on each node")
	fs.Var(resource.NewQuantityFlagValue(&s.NodeMemory), "node_memory", "The amount of memory (in bytes) provisioned on each node")
	client.BindKubeletClientConfigFlags(fs, &s.KubeletConfig)
}

func (s *CMServer) verifyMinionFlags() {
	if !s.SyncNodeList && s.MinionRegexp != "" {
		glog.Info("--minion_regexp is ignored by --sync_nodes=false")
	}
	if s.CloudProvider == "" || s.MinionRegexp == "" {
		if len(s.MachineList) == 0 {
			glog.Info("No machines specified!")
		}
		return
	}
	if len(s.MachineList) != 0 {
		glog.Info("--machines is overwritten by --minion_regexp")
	}
}

// Run runs the CMServer.  This should never exit.
func (s *CMServer) Run(_ []string) error {
	s.verifyMinionFlags()

	if len(s.ClientConfig.Host) == 0 {
		glog.Fatal("usage: controller-manager --master <master>")
	}

	kubeClient, err := client.New(&s.ClientConfig)
	if err != nil {
		glog.Fatalf("Invalid API configuration: %v", err)
	}

	go http.ListenAndServe(net.JoinHostPort(s.Address.String(), strconv.Itoa(s.Port)), nil)

	endpoints := service.NewEndpointController(kubeClient)
	go util.Forever(func() { endpoints.SyncServiceEndpoints() }, time.Second*10)

	controllerManager := replicationControllerPkg.NewReplicationManager(kubeClient)
	controllerManager.Run(10 * time.Second)

	kubeletClient, err := client.NewKubeletClient(&s.KubeletConfig)
	if err != nil {
		glog.Fatalf("Failure to start kubelet client: %v", err)
	}

	cloud := cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile)
	nodeResources := &api.NodeResources{
		Capacity: api.ResourceList{
			api.ResourceCPU:    *resource.NewMilliQuantity(s.NodeMilliCPU, resource.DecimalSI),
			api.ResourceMemory: s.NodeMemory,
		},
	}

	nodeController := nodeControllerPkg.NewNodeController(cloud, s.MinionRegexp, s.MachineList, nodeResources,
		kubeClient, kubeletClient, s.RegisterRetryCount, s.PodEvictionTimeout)
	nodeController.Run(s.NodeSyncPeriod, s.SyncNodeList)

	resourceQuotaManager := resourcequota.NewResourceQuotaManager(kubeClient)
	resourceQuotaManager.Run(s.ResourceQuotaSyncPeriod)

	select {}
	return nil
}
<commit_msg>Fix typo in comment: mail -> main<commit_after>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package app implements a server that runs a set of active
// components.  This includes replication controllers, service endpoints and
// nodes.
package app

import (
	"net"
	"net/http"
	"strconv"
	"time"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/resource"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider"
	nodeControllerPkg "github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider/controller"
	replicationControllerPkg "github.com/GoogleCloudPlatform/kubernetes/pkg/controller"
	_ "github.com/GoogleCloudPlatform/kubernetes/pkg/healthz"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/master/ports"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/resourcequota"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/service"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"

	"github.com/golang/glog"
	"github.com/spf13/pflag"
)

// CMServer is the main context object for the controller manager.
type CMServer struct {
	Port                    int
	Address                 util.IP
	ClientConfig            client.Config
	CloudProvider           string
	CloudConfigFile         string
	MinionRegexp            string
	NodeSyncPeriod          time.Duration
	ResourceQuotaSyncPeriod time.Duration
	RegisterRetryCount      int
	MachineList             util.StringList
	SyncNodeList            bool
	PodEvictionTimeout      time.Duration

	// TODO: Discover these by pinging the host machines, and rip out these params.
	NodeMilliCPU int64
	NodeMemory   resource.Quantity

	KubeletConfig client.KubeletConfig
}

// NewCMServer creates a new CMServer with a default config.
func NewCMServer() *CMServer {
	s := CMServer{
		Port:                    ports.ControllerManagerPort,
		Address:                 util.IP(net.ParseIP("127.0.0.1")),
		NodeSyncPeriod:          10 * time.Second,
		ResourceQuotaSyncPeriod: 10 * time.Second,
		RegisterRetryCount:      10,
		PodEvictionTimeout:      5 * time.Minute,
		NodeMilliCPU:            1000,
		NodeMemory:              resource.MustParse("3Gi"),
		SyncNodeList:            true,
		KubeletConfig: client.KubeletConfig{
			Port:        ports.KubeletPort,
			EnableHttps: false,
		},
	}
	return &s
}

// AddFlags adds flags for a specific CMServer to the specified FlagSet
func (s *CMServer) AddFlags(fs *pflag.FlagSet) {
	fs.IntVar(&s.Port, "port", s.Port, "The port that the controller-manager's http service runs on")
	fs.Var(&s.Address, "address", "The IP address to serve on (set to 0.0.0.0 for all interfaces)")
	client.BindClientConfigFlags(fs, &s.ClientConfig)
	fs.StringVar(&s.CloudProvider, "cloud_provider", s.CloudProvider, "The provider for cloud services.  Empty string for no provider.")
	fs.StringVar(&s.CloudConfigFile, "cloud_config", s.CloudConfigFile, "The path to the cloud provider configuration file.  Empty string for no configuration file.")
	fs.StringVar(&s.MinionRegexp, "minion_regexp", s.MinionRegexp, "If non empty, and --cloud_provider is specified, a regular expression for matching minion VMs.")
	fs.DurationVar(&s.NodeSyncPeriod, "node_sync_period", s.NodeSyncPeriod, ""+
		"The period for syncing nodes from cloudprovider. Longer periods will result in "+
		"fewer calls to cloud provider, but may delay addition of new nodes to cluster.")
	fs.DurationVar(&s.ResourceQuotaSyncPeriod, "resource_quota_sync_period", s.ResourceQuotaSyncPeriod, "The period for syncing quota usage status in the system")
	fs.DurationVar(&s.PodEvictionTimeout, "pod_eviction_timeout", s.PodEvictionTimeout, "The grace peroid for deleting pods on failed nodes.")
	fs.IntVar(&s.RegisterRetryCount, "register_retry_count", s.RegisterRetryCount, ""+
		"The number of retries for initial node registration.  Retry interval equals node_sync_period.")
	fs.Var(&s.MachineList, "machines", "List of machines to schedule onto, comma separated.")
	fs.BoolVar(&s.SyncNodeList, "sync_nodes", s.SyncNodeList, "If true, and --cloud_provider is specified, sync nodes from the cloud provider. Default true.")
	// TODO: Discover these by pinging the host machines, and rip out these flags.
	// TODO: in the meantime, use resource.QuantityFlag() instead of these
	fs.Int64Var(&s.NodeMilliCPU, "node_milli_cpu", s.NodeMilliCPU, "The amount of MilliCPU provisioned on each node")
	fs.Var(resource.NewQuantityFlagValue(&s.NodeMemory), "node_memory", "The amount of memory (in bytes) provisioned on each node")
	client.BindKubeletClientConfigFlags(fs, &s.KubeletConfig)
}

func (s *CMServer) verifyMinionFlags() {
	if !s.SyncNodeList && s.MinionRegexp != "" {
		glog.Info("--minion_regexp is ignored by --sync_nodes=false")
	}
	if s.CloudProvider == "" || s.MinionRegexp == "" {
		if len(s.MachineList) == 0 {
			glog.Info("No machines specified!")
		}
		return
	}
	if len(s.MachineList) != 0 {
		glog.Info("--machines is overwritten by --minion_regexp")
	}
}

// Run runs the CMServer.  This should never exit.
func (s *CMServer) Run(_ []string) error {
	s.verifyMinionFlags()

	if len(s.ClientConfig.Host) == 0 {
		glog.Fatal("usage: controller-manager --master <master>")
	}

	kubeClient, err := client.New(&s.ClientConfig)
	if err != nil {
		glog.Fatalf("Invalid API configuration: %v", err)
	}

	go http.ListenAndServe(net.JoinHostPort(s.Address.String(), strconv.Itoa(s.Port)), nil)

	endpoints := service.NewEndpointController(kubeClient)
	go util.Forever(func() { endpoints.SyncServiceEndpoints() }, time.Second*10)

	controllerManager := replicationControllerPkg.NewReplicationManager(kubeClient)
	controllerManager.Run(10 * time.Second)

	kubeletClient, err := client.NewKubeletClient(&s.KubeletConfig)
	if err != nil {
		glog.Fatalf("Failure to start kubelet client: %v", err)
	}

	cloud := cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile)
	nodeResources := &api.NodeResources{
		Capacity: api.ResourceList{
			api.ResourceCPU:    *resource.NewMilliQuantity(s.NodeMilliCPU, resource.DecimalSI),
			api.ResourceMemory: s.NodeMemory,
		},
	}

	nodeController := nodeControllerPkg.NewNodeController(cloud, s.MinionRegexp, s.MachineList, nodeResources,
		kubeClient, kubeletClient, s.RegisterRetryCount, s.PodEvictionTimeout)
	nodeController.Run(s.NodeSyncPeriod, s.SyncNodeList)

	resourceQuotaManager := resourcequota.NewResourceQuotaManager(kubeClient)
	resourceQuotaManager.Run(s.ResourceQuotaSyncPeriod)

	select {}
	return nil
}
<|endoftext|>


<commit_before>package defaults

import (
	"testing"

	"github.com/goreleaser/goreleaser/config"
	"github.com/goreleaser/goreleaser/context"
	"github.com/goreleaser/goreleaser/internal/testlib"
	"github.com/stretchr/testify/assert"
)

func TestDescription(t *testing.T) {
	assert.NotEmpty(t, Pipe{}.String())
}

func TestFillBasicData(t *testing.T) {
	_, back := testlib.Mktmp(t)
	defer back()
	testlib.GitInit(t)
	testlib.GitRemoteAdd(t, "git@github.com:goreleaser/goreleaser.git")

	var ctx = &context.Context{
		Config: config.Project{},
	}

	assert.NoError(t, Pipe{}.Run(ctx))
	assert.Equal(t, "goreleaser", ctx.Config.Release.GitHub.Owner)
	assert.Equal(t, "goreleaser", ctx.Config.Release.GitHub.Name)
	assert.NotEmpty(t, ctx.Config.Builds)
	assert.Equal(t, "goreleaser", ctx.Config.Builds[0].Binary)
	assert.Equal(t, ".", ctx.Config.Builds[0].Main)
	assert.Contains(t, ctx.Config.Builds[0].Goos, "darwin")
	assert.Contains(t, ctx.Config.Builds[0].Goos, "linux")
	assert.Contains(t, ctx.Config.Builds[0].Goarch, "386")
	assert.Contains(t, ctx.Config.Builds[0].Goarch, "amd64")
	assert.Equal(t, "tar.gz", ctx.Config.Archive.Format)
	assert.Contains(t, ctx.Config.Brew.Install, "bin.install \"goreleaser\"")
	assert.Empty(t, ctx.Config.Dockers)
	assert.Equal(t, "https://github.com", ctx.Config.GitHubURLs.Download)
	assert.NotEmpty(
		t,
		ctx.Config.Archive.NameTemplate,
		ctx.Config.Builds[0].Ldflags,
		ctx.Config.Archive.Files,
		ctx.Config.Dist,
	)
}

func TestFillPartial(t *testing.T) {
	_, back := testlib.Mktmp(t)
	defer back()
	testlib.GitInit(t)
	testlib.GitRemoteAdd(t, "git@github.com:goreleaser/goreleaser.git")

	var ctx = &context.Context{
		Config: config.Project{
			GitHubURLs: config.GitHubURLs{
				Download: "https://github.company.com",
			},
			Dist: "disttt",
			Release: config.Release{
				GitHub: config.Repo{
					Owner: "goreleaser",
					Name:  "test",
				},
			},
			Archive: config.Archive{
				Files: []string{
					"glob/*",
				},
			},
			Builds: []config.Build{
				{Binary: "testreleaser"},
				{Goos: []string{"linux"}},
				{
					Binary: "another",
					Ignore: []config.IgnoredBuild{
						{Goos: "darwin", Goarch: "amd64"},
					},
				},
			},
			Dockers: []config.Docker{
				{Image: "a/b"},
			},
		},
	}
	assert.NoError(t, Pipe{}.Run(ctx))
	assert.Len(t, ctx.Config.Archive.Files, 1)
	assert.Equal(t, `bin.install "testreleaser"`, ctx.Config.Brew.Install)
	assert.NotEmpty(t, ctx.Config.Dockers[0].Binary)
	assert.NotEmpty(t, ctx.Config.Dockers[0].Goos)
	assert.NotEmpty(t, ctx.Config.Dockers[0].Goarch)
	assert.NotEmpty(t, ctx.Config.Dockers[0].Dockerfile)
	assert.Empty(t, ctx.Config.Dockers[0].Goarm)
	assert.Equal(t, "disttt", ctx.Config.Dist)
	assert.NotEqual(t, "https://github.com", ctx.Config.GitHubURLs.Download)
}
<commit_msg>test: Use multiple calls to assert.NotEmpty()<commit_after>package defaults

import (
	"testing"

	"github.com/goreleaser/goreleaser/config"
	"github.com/goreleaser/goreleaser/context"
	"github.com/goreleaser/goreleaser/internal/testlib"
	"github.com/stretchr/testify/assert"
)

func TestDescription(t *testing.T) {
	assert.NotEmpty(t, Pipe{}.String())
}

func TestFillBasicData(t *testing.T) {
	_, back := testlib.Mktmp(t)
	defer back()
	testlib.GitInit(t)
	testlib.GitRemoteAdd(t, "git@github.com:goreleaser/goreleaser.git")

	var ctx = &context.Context{
		Config: config.Project{},
	}

	assert.NoError(t, Pipe{}.Run(ctx))
	assert.Equal(t, "goreleaser", ctx.Config.Release.GitHub.Owner)
	assert.Equal(t, "goreleaser", ctx.Config.Release.GitHub.Name)
	assert.NotEmpty(t, ctx.Config.Builds)
	assert.Equal(t, "goreleaser", ctx.Config.Builds[0].Binary)
	assert.Equal(t, ".", ctx.Config.Builds[0].Main)
	assert.Contains(t, ctx.Config.Builds[0].Goos, "darwin")
	assert.Contains(t, ctx.Config.Builds[0].Goos, "linux")
	assert.Contains(t, ctx.Config.Builds[0].Goarch, "386")
	assert.Contains(t, ctx.Config.Builds[0].Goarch, "amd64")
	assert.Equal(t, "tar.gz", ctx.Config.Archive.Format)
	assert.Contains(t, ctx.Config.Brew.Install, "bin.install \"goreleaser\"")
	assert.Empty(t, ctx.Config.Dockers)
	assert.Equal(t, "https://github.com", ctx.Config.GitHubURLs.Download)
	assert.NotEmpty(t, ctx.Config.Archive.NameTemplate)
	assert.NotEmpty(t, ctx.Config.Builds[0].Ldflags)
	assert.NotEmpty(t, ctx.Config.Archive.Files)
	assert.NotEmpty(t, ctx.Config.Dist)
}

func TestFillPartial(t *testing.T) {
	_, back := testlib.Mktmp(t)
	defer back()
	testlib.GitInit(t)
	testlib.GitRemoteAdd(t, "git@github.com:goreleaser/goreleaser.git")

	var ctx = &context.Context{
		Config: config.Project{
			GitHubURLs: config.GitHubURLs{
				Download: "https://github.company.com",
			},
			Dist: "disttt",
			Release: config.Release{
				GitHub: config.Repo{
					Owner: "goreleaser",
					Name:  "test",
				},
			},
			Archive: config.Archive{
				Files: []string{
					"glob/*",
				},
			},
			Builds: []config.Build{
				{Binary: "testreleaser"},
				{Goos: []string{"linux"}},
				{
					Binary: "another",
					Ignore: []config.IgnoredBuild{
						{Goos: "darwin", Goarch: "amd64"},
					},
				},
			},
			Dockers: []config.Docker{
				{Image: "a/b"},
			},
		},
	}
	assert.NoError(t, Pipe{}.Run(ctx))
	assert.Len(t, ctx.Config.Archive.Files, 1)
	assert.Equal(t, `bin.install "testreleaser"`, ctx.Config.Brew.Install)
	assert.NotEmpty(t, ctx.Config.Dockers[0].Binary)
	assert.NotEmpty(t, ctx.Config.Dockers[0].Goos)
	assert.NotEmpty(t, ctx.Config.Dockers[0].Goarch)
	assert.NotEmpty(t, ctx.Config.Dockers[0].Dockerfile)
	assert.Empty(t, ctx.Config.Dockers[0].Goarm)
	assert.Equal(t, "disttt", ctx.Config.Dist)
	assert.NotEqual(t, "https://github.com", ctx.Config.GitHubURLs.Download)
}
<|endoftext|>


<commit_before>package storage

import (
	"archive/zip"
	"fmt"
	"github.com/viant/toolbox"
	"io"
	"path"
	"strings"
)

type CopyHandler func(sourceObject Object, source io.Reader, destinationService Service, destinationURL string) error
type ModificationHandler func(reader io.ReadCloser) (io.ReadCloser, error)

func urlPath(URL string) string {
	var result = URL
	schemaPosition := strings.Index(URL, "://")
	if schemaPosition != -1 {
		result = string(URL[schemaPosition+3:])
	}
	pathRoot := strings.Index(result, "/")
	if pathRoot > 0 {
		result = string(result[pathRoot:])
	}
	if strings.HasSuffix(result, "/") {
		result = string(result[:len(result)-1])
	}

	return result
}

func copyData(sourceService Service, sourceURL string, destinationService Service, destinationURL string, modifyContentHandler ModificationHandler, subPath string, copyHandler CopyHandler) error {
	sourceListURL := sourceURL
	if subPath != "" {
		sourceListURL = toolbox.URLPathJoin(sourceURL, subPath)
	}
	objects, err := sourceService.List(sourceListURL)
	var objectRelativePath string
	sourceURLPath := urlPath(sourceURL)
	for _, object := range objects {
		var objectURLPath = urlPath(object.URL())
		if object.IsFolder() {

			if sourceURLPath == objectURLPath {
				continue
			}
			if subPath != "" && objectURLPath == toolbox.URLPathJoin(sourceURLPath, subPath) {
				continue
			}
		}
		if len(objectURLPath) > len(sourceURLPath) {
			objectRelativePath = objectURLPath[len(sourceURLPath):]
			if strings.HasPrefix(objectRelativePath, "/") {
				objectRelativePath = string(objectRelativePath[1:])
			}
		}
		var destinationObjectURL = destinationURL
		if objectRelativePath != "" {
			destinationObjectURL = toolbox.URLPathJoin(destinationURL, objectRelativePath)
		}
		if object.IsContent() {
			reader, err := sourceService.Download(object)
			if err != nil {
				err = fmt.Errorf("unable download, %v -> %v, %v", object.URL(), destinationObjectURL, err)
				return err
			}
			defer reader.Close()
			if modifyContentHandler != nil {
				reader, err = modifyContentHandler(reader)
				if err != nil {
					err = fmt.Errorf("unable modify content, %v %v %v", object.URL(), destinationObjectURL, err)
					return err
				}
			}
			if subPath == "" {
				_, sourceName := path.Split(object.URL())
				_, destinationName := path.Split(destinationURL)
				if strings.HasSuffix(destinationObjectURL, "/") {
					destinationObjectURL = toolbox.URLPathJoin(destinationObjectURL, sourceName)
				} else {
					destinationObject, _ := destinationService.StorageObject(destinationObjectURL)
					if destinationObject != nil && destinationObject.IsFolder() {
						destinationObjectURL = toolbox.URLPathJoin(destinationObjectURL, sourceName)
					} else if destinationName != sourceName {
						if !strings.Contains(destinationName, ".") {
							destinationObjectURL = toolbox.URLPathJoin(destinationURL, sourceName)
						}

					}
				}
			}
			err = copyHandler(object, reader, destinationService, destinationObjectURL)
			if err != nil {
				return err
			}

		} else {
			err = copyData(sourceService, sourceURL, destinationService, destinationURL, modifyContentHandler, objectRelativePath, copyHandler)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

func copySourceToDestination(sourceObject Object, reader io.Reader, destinationService Service, destinationURL string) error {
	err := destinationService.Upload(destinationURL, reader)
	if err != nil {
		err = fmt.Errorf("unable upload, %v %v %v", sourceObject.URL(), destinationURL, err)
	}
	return err
}

func addPathIfNeeded(directories map[string]bool, path string, archive zip.Writer) {
	if path == "" {
		return
	}
	if _, has := directories[path]; has {
		return
	}

}

func getArchiveCopyHandler(archive zip.Writer, parentURL string) CopyHandler {
	var directories = make(map[string]bool)
	return func(sourceObject Object, reader io.Reader, destinationService Service, destinationURL string) error {
		var _, relativePath = toolbox.URLSplit(destinationURL)
		if destinationURL != parentURL {
			relativePath = strings.Replace(destinationURL, parentURL, "", 1)
			var parent, _ = path.Split(relativePath)
			addPathIfNeeded(directories, parent, archive)
		}
		header, err := zip.FileInfoHeader(sourceObject.FileInfo())
		if err != nil {
			return err
		}
		header.Method = zip.Deflate
		header.Name = relativePath
		writer, err := archive.CreateHeader(header)
		if err != nil {
			return err
		}
		_, err = io.Copy(writer, reader)
		return err
	}
}

//Copy downloads objects from source URL to upload them to destination URL.
func Copy(sourceService Service, sourceURL string, destinationService Service, destinationURL string, modifyContentHandler ModificationHandler, copyHandler CopyHandler) (err error) {
	if copyHandler == nil {
		copyHandler = copySourceToDestination
	}
	err = copyData(sourceService, sourceURL, destinationService, destinationURL, modifyContentHandler, "", copyHandler)
	if err != nil {
		err = fmt.Errorf("failed to copy %v -> %v: %v", sourceURL, destinationURL, err)
	}
	return err
}
<commit_msg>change name<commit_after>package storage

import (
	"archive/zip"
	"fmt"
	"github.com/viant/toolbox"
	"io"
	"path"
	"strings"
)

type CopyHandler func(sourceObject Object, source io.Reader, destinationService Service, destinationURL string) error
type ModificationHandler func(reader io.ReadCloser) (io.ReadCloser, error)

func urlPath(URL string) string {
	var result = URL
	schemaPosition := strings.Index(URL, "://")
	if schemaPosition != -1 {
		result = string(URL[schemaPosition+3:])
	}
	pathRoot := strings.Index(result, "/")
	if pathRoot > 0 {
		result = string(result[pathRoot:])
	}
	if strings.HasSuffix(result, "/") {
		result = string(result[:len(result)-1])
	}

	return result
}

func copyStorageContent(sourceService Service, sourceURL string, destinationService Service, destinationURL string, modifyContentHandler ModificationHandler, subPath string, copyHandler CopyHandler) error {
	sourceListURL := sourceURL
	if subPath != "" {
		sourceListURL = toolbox.URLPathJoin(sourceURL, subPath)
	}
	objects, err := sourceService.List(sourceListURL)
	var objectRelativePath string
	sourceURLPath := urlPath(sourceURL)
	for _, object := range objects {
		var objectURLPath = urlPath(object.URL())
		if object.IsFolder() {

			if sourceURLPath == objectURLPath {
				continue
			}
			if subPath != "" && objectURLPath == toolbox.URLPathJoin(sourceURLPath, subPath) {
				continue
			}
		}
		if len(objectURLPath) > len(sourceURLPath) {
			objectRelativePath = objectURLPath[len(sourceURLPath):]
			if strings.HasPrefix(objectRelativePath, "/") {
				objectRelativePath = string(objectRelativePath[1:])
			}
		}
		var destinationObjectURL = destinationURL
		if objectRelativePath != "" {
			destinationObjectURL = toolbox.URLPathJoin(destinationURL, objectRelativePath)
		}
		if object.IsContent() {
			reader, err := sourceService.Download(object)
			if err != nil {
				err = fmt.Errorf("unable download, %v -> %v, %v", object.URL(), destinationObjectURL, err)
				return err
			}
			defer reader.Close()
			if modifyContentHandler != nil {
				reader, err = modifyContentHandler(reader)
				if err != nil {
					err = fmt.Errorf("unable modify content, %v %v %v", object.URL(), destinationObjectURL, err)
					return err
				}
			}
			if subPath == "" {
				_, sourceName := path.Split(object.URL())
				_, destinationName := path.Split(destinationURL)
				if strings.HasSuffix(destinationObjectURL, "/") {
					destinationObjectURL = toolbox.URLPathJoin(destinationObjectURL, sourceName)
				} else {
					destinationObject, _ := destinationService.StorageObject(destinationObjectURL)
					if destinationObject != nil && destinationObject.IsFolder() {
						destinationObjectURL = toolbox.URLPathJoin(destinationObjectURL, sourceName)
					} else if destinationName != sourceName {
						if !strings.Contains(destinationName, ".") {
							destinationObjectURL = toolbox.URLPathJoin(destinationURL, sourceName)
						}

					}
				}
			}
			err = copyHandler(object, reader, destinationService, destinationObjectURL)
			if err != nil {
				return err
			}

		} else {
			err = copyStorageContent(sourceService, sourceURL, destinationService, destinationURL, modifyContentHandler, objectRelativePath, copyHandler)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

func copySourceToDestination(sourceObject Object, reader io.Reader, destinationService Service, destinationURL string) error {
	err := destinationService.Upload(destinationURL, reader)
	if err != nil {
		err = fmt.Errorf("unable upload, %v %v %v", sourceObject.URL(), destinationURL, err)
	}
	return err
}

func addPathIfNeeded(directories map[string]bool, path string, archive zip.Writer) {
	if path == "" {
		return
	}
	if _, has := directories[path]; has {
		return
	}

}

func getArchiveCopyHandler(archive zip.Writer, parentURL string) CopyHandler {
	var directories = make(map[string]bool)
	return func(sourceObject Object, reader io.Reader, destinationService Service, destinationURL string) error {
		var _, relativePath = toolbox.URLSplit(destinationURL)
		if destinationURL != parentURL {
			relativePath = strings.Replace(destinationURL, parentURL, "", 1)
			var parent, _ = path.Split(relativePath)
			addPathIfNeeded(directories, parent, archive)
		}
		header, err := zip.FileInfoHeader(sourceObject.FileInfo())
		if err != nil {
			return err
		}
		header.Method = zip.Deflate
		header.Name = relativePath
		writer, err := archive.CreateHeader(header)
		if err != nil {
			return err
		}
		_, err = io.Copy(writer, reader)
		return err
	}
}

//Copy downloads objects from source URL to upload them to destination URL.
func Copy(sourceService Service, sourceURL string, destinationService Service, destinationURL string, modifyContentHandler ModificationHandler, copyHandler CopyHandler) (err error) {
	if copyHandler == nil {
		copyHandler = copySourceToDestination
	}
	err = copyStorageContent(sourceService, sourceURL, destinationService, destinationURL, modifyContentHandler, "", copyHandler)
	if err != nil {
		err = fmt.Errorf("failed to copy %v -> %v: %v", sourceURL, destinationURL, err)
	}
	return err
}
<|endoftext|>


<commit_before>/**
 * Copyright 2016 Confluent Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// kafka client.
// This package implements high-level Apache Kafka producer and consumers
// using bindings on-top of the C librdkafka library.
package kafka

import (
	"fmt"
	"sync"
	"unsafe"
)

/*
#include <librdkafka/rdkafka.h>
#include <stdlib.h>
*/
import "C"

type Handle interface {
	get_handle() *handle
}

// Common instance handle for both Producer and Consumer
type handle struct {
	rk  *C.rd_kafka_t
	rkq *C.rd_kafka_queue_t

	// Termination of background go-routines
	terminated_chan chan string // string is go-routine name

	// Topic <-> rkt caches
	rkt_cache_lock sync.Mutex
	// topic name -> rkt cache
	rkt_cache map[string]*C.rd_kafka_topic_t
	// rkt -> topic name cache
	rkt_name_cache map[*C.rd_kafka_topic_t]string

	//
	// cgo map
	// Maps C callbacks based on cgoid back to its Go object
	cgo_lock   sync.Mutex
	cgoid_next uintptr
	cgomap     map[uintptr]cgoif

	//
	// producer
	//
	p *Producer

	// Forward delivery reports on Producer.Events channel
	fwd_dr bool

	//
	// consumer
	//
	c *Consumer

	// Forward rebalancing ack responsibility to application (current setting)
	curr_app_rebalance_enable bool
}

func (h *handle) String() string {
	return C.GoString(C.rd_kafka_name(h.rk))
}

func (h *handle) setup() {
	h.rkt_cache = make(map[string]*C.rd_kafka_topic_t)
	h.rkt_name_cache = make(map[*C.rd_kafka_topic_t]string)

	h.terminated_chan = make(chan string, 10)
}

func (h *handle) cleanup() {
	for _, c_rkt := range h.rkt_cache {
		C.rd_kafka_topic_destroy(c_rkt)
	}

	if h.rkq != nil {
		C.rd_kafka_queue_destroy(h.rkq)
	}
}

// wait_terminated waits termination of background go-routines.
// term_cnt is the number of goroutines expected to signal termination completion
// on h.terminated_chan
func (h *handle) wait_terminated(term_cnt int) {
	// Wait for term_cnt termination-done events from goroutines
	for ; term_cnt > 0; term_cnt -= 1 {
		_ = <-h.terminated_chan
	}
}

// get_rkt0 finds or creates and returns a C topic_t object from the local cache.
func (h *handle) get_rkt0(topic string, c_topic *C.char, do_lock bool) (c_rkt *C.rd_kafka_topic_t) {
	if do_lock {
		h.rkt_cache_lock.Lock()
	}
	c_rkt, ok := h.rkt_cache[topic]
	if ok {
		if do_lock {
			h.rkt_cache_lock.Unlock()
		}
		return c_rkt
	}

	if c_topic == nil {
		c_topic = C.CString(topic)
		defer C.free(unsafe.Pointer(c_topic))
	}

	c_rkt = C.rd_kafka_topic_new(h.rk, c_topic, nil)
	if c_rkt == nil {
		panic(fmt.Sprintf("Unable to create now C topic \"%s\": %s",
			topic, C.GoString(C.rd_kafka_err2str(C.rd_kafka_last_error()))))
	}

	h.rkt_cache[topic] = c_rkt
	h.rkt_name_cache[c_rkt] = topic

	if do_lock {
		h.rkt_cache_lock.Unlock()
	}

	return c_rkt
}

// get_rkt finds or creates and returns a C topic_t object from the local cache.
func (h *handle) get_rkt(topic string) (c_rkt *C.rd_kafka_topic_t) {
	return h.get_rkt0(topic, nil, true)
}

// get_topic_name_from_rkt returns the topic name for a C topic_t object, preferably
// using the local cache to avoid a cgo call.
func (h *handle) get_topic_name_from_rkt(c_rkt *C.rd_kafka_topic_t) (topic string) {
	h.rkt_cache_lock.Lock()
	topic, ok := h.rkt_name_cache[c_rkt]
	if ok {
		h.rkt_cache_lock.Unlock()
		return topic
	}

	// we need our own copy/refcount of the c_rkt
	c_topic := C.rd_kafka_topic_name(c_rkt)
	topic = C.GoString(c_topic)

	c_rkt = h.get_rkt0(topic, c_topic, false /* dont lock */)
	h.rkt_cache_lock.Unlock()

	return topic
}

// cgoif is a generic interface for holding Go state passed as opaque
// value to the C code.
// Since pointers to complex Go types cannot be passed to C we instead create
// a cgoif object, generate a unique id that is added to the cgomap,
// and then pass that id to the C code. When the C code callback is called we
// use the id to look up the cgoif object in the cgomap.
type cgoif interface{}

// delivery report cgoif container
type cgo_dr struct {
	delivery_chan chan Event
	opaque        *interface{}
}

// cgo_put adds object cg to the handle's cgo map and returns a
// unique id for the added entry.
// Thread-safe.
// FIXME: the uniquity of the id is questionable over time.
func (h *handle) cgo_put(cg cgoif) (cgoid uintptr) {
	h.cgo_lock.Lock()
	h.cgoid_next += 1
	if h.cgoid_next == 0 {
		h.cgoid_next += 1
	}
	cgoid = h.cgoid_next
	h.cgomap[cgoid] = cg
	h.cgo_lock.Unlock()
	return cgoid
}

// cgo_get looks up cgoid in the cgo map, deletes the reference from the map
// and returns the object, if found. Else returns nil, false.
// Thread-safe.
func (h *handle) cgo_get(cgoid uintptr) (cg cgoif, found bool) {
	if cgoid == 0 {
		return nil, false
	}

	h.cgo_lock.Lock()
	cg, found = h.cgomap[cgoid]
	if found {
		delete(h.cgomap, cgoid)
	}
	h.cgo_lock.Unlock()
	return cg, found
}
<commit_msg>Use defer to Unlock<commit_after>/**
 * Copyright 2016 Confluent Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// kafka client.
// This package implements high-level Apache Kafka producer and consumers
// using bindings on-top of the C librdkafka library.
package kafka

import (
	"fmt"
	"sync"
	"unsafe"
)

/*
#include <librdkafka/rdkafka.h>
#include <stdlib.h>
*/
import "C"

type Handle interface {
	get_handle() *handle
}

// Common instance handle for both Producer and Consumer
type handle struct {
	rk  *C.rd_kafka_t
	rkq *C.rd_kafka_queue_t

	// Termination of background go-routines
	terminated_chan chan string // string is go-routine name

	// Topic <-> rkt caches
	rkt_cache_lock sync.Mutex
	// topic name -> rkt cache
	rkt_cache map[string]*C.rd_kafka_topic_t
	// rkt -> topic name cache
	rkt_name_cache map[*C.rd_kafka_topic_t]string

	//
	// cgo map
	// Maps C callbacks based on cgoid back to its Go object
	cgo_lock   sync.Mutex
	cgoid_next uintptr
	cgomap     map[uintptr]cgoif

	//
	// producer
	//
	p *Producer

	// Forward delivery reports on Producer.Events channel
	fwd_dr bool

	//
	// consumer
	//
	c *Consumer

	// Forward rebalancing ack responsibility to application (current setting)
	curr_app_rebalance_enable bool
}

func (h *handle) String() string {
	return C.GoString(C.rd_kafka_name(h.rk))
}

func (h *handle) setup() {
	h.rkt_cache = make(map[string]*C.rd_kafka_topic_t)
	h.rkt_name_cache = make(map[*C.rd_kafka_topic_t]string)

	h.terminated_chan = make(chan string, 10)
}

func (h *handle) cleanup() {
	for _, c_rkt := range h.rkt_cache {
		C.rd_kafka_topic_destroy(c_rkt)
	}

	if h.rkq != nil {
		C.rd_kafka_queue_destroy(h.rkq)
	}
}

// wait_terminated waits termination of background go-routines.
// term_cnt is the number of goroutines expected to signal termination completion
// on h.terminated_chan
func (h *handle) wait_terminated(term_cnt int) {
	// Wait for term_cnt termination-done events from goroutines
	for ; term_cnt > 0; term_cnt -= 1 {
		_ = <-h.terminated_chan
	}
}

// get_rkt0 finds or creates and returns a C topic_t object from the local cache.
func (h *handle) get_rkt0(topic string, c_topic *C.char, do_lock bool) (c_rkt *C.rd_kafka_topic_t) {
	if do_lock {
		h.rkt_cache_lock.Lock()
		defer h.rkt_cache_lock.Unlock()
	}
	c_rkt, ok := h.rkt_cache[topic]
	if ok {
		return c_rkt
	}

	if c_topic == nil {
		c_topic = C.CString(topic)
		defer C.free(unsafe.Pointer(c_topic))
	}

	c_rkt = C.rd_kafka_topic_new(h.rk, c_topic, nil)
	if c_rkt == nil {
		panic(fmt.Sprintf("Unable to create now C topic \"%s\": %s",
			topic, C.GoString(C.rd_kafka_err2str(C.rd_kafka_last_error()))))
	}

	h.rkt_cache[topic] = c_rkt
	h.rkt_name_cache[c_rkt] = topic

	return c_rkt
}

// get_rkt finds or creates and returns a C topic_t object from the local cache.
func (h *handle) get_rkt(topic string) (c_rkt *C.rd_kafka_topic_t) {
	return h.get_rkt0(topic, nil, true)
}

// get_topic_name_from_rkt returns the topic name for a C topic_t object, preferably
// using the local cache to avoid a cgo call.
func (h *handle) get_topic_name_from_rkt(c_rkt *C.rd_kafka_topic_t) (topic string) {
	h.rkt_cache_lock.Lock()
	defer h.rkt_cache_lock.Unlock()

	topic, ok := h.rkt_name_cache[c_rkt]
	if ok {
		return topic
	}

	// we need our own copy/refcount of the c_rkt
	c_topic := C.rd_kafka_topic_name(c_rkt)
	topic = C.GoString(c_topic)

	c_rkt = h.get_rkt0(topic, c_topic, false /* dont lock */)

	return topic
}

// cgoif is a generic interface for holding Go state passed as opaque
// value to the C code.
// Since pointers to complex Go types cannot be passed to C we instead create
// a cgoif object, generate a unique id that is added to the cgomap,
// and then pass that id to the C code. When the C code callback is called we
// use the id to look up the cgoif object in the cgomap.
type cgoif interface{}

// delivery report cgoif container
type cgo_dr struct {
	delivery_chan chan Event
	opaque        *interface{}
}

// cgo_put adds object cg to the handle's cgo map and returns a
// unique id for the added entry.
// Thread-safe.
// FIXME: the uniquity of the id is questionable over time.
func (h *handle) cgo_put(cg cgoif) (cgoid uintptr) {
	h.cgo_lock.Lock()
	defer h.cgo_lock.Unlock()

	h.cgoid_next += 1
	if h.cgoid_next == 0 {
		h.cgoid_next += 1
	}
	cgoid = h.cgoid_next
	h.cgomap[cgoid] = cg
	return cgoid
}

// cgo_get looks up cgoid in the cgo map, deletes the reference from the map
// and returns the object, if found. Else returns nil, false.
// Thread-safe.
func (h *handle) cgo_get(cgoid uintptr) (cg cgoif, found bool) {
	if cgoid == 0 {
		return nil, false
	}

	h.cgo_lock.Lock()
	defer h.cgo_lock.Unlock()
	cg, found = h.cgomap[cgoid]
	if found {
		delete(h.cgomap, cgoid)
	}

	return cg, found
}
<|endoftext|>


<commit_before>package operation

type Manifest struct {
	Applications []AppManifest `yaml:"applications"`
}

type AppManifest struct {
	Name                    string              `yaml:"name"`
	Buildpacks              []string            `yaml:"buildpacks,omitempty"`
	Command                 string              `yaml:"command,omitempty"`
	DiskQuota               string              `yaml:"disk_quota,omitempty"`
	Docker                  *AppManifestDocker  `yaml:"docker,omitempty"`
	Env                     map[string]string   `yaml:"env,omitempty"`
	HealthCheckType         string              `yaml:"health-check-type,omitempty"`
	HealthCheckHTTPEndpoint string              `yaml:"health-check-http-endpoint,omitempty"`
	Instances               int                 `yaml:"instances,omitempty"`
	LogRateLimit            string              `yaml:"log-rate-limit,omitempty"`
	Memory                  string              `yaml:"memory,omitempty"`
	NoRoute                 bool                `yaml:"no-route,omitempty"`
	Routes                  []AppManifestRoutes `yaml:"routes,omitempty"`
	Services                []string            `yaml:"services,omitempty"`
	Stack                   string              `yaml:"stack,omitempty"`
	Timeout                 int                 `yaml:"timeout,omitempty"`
}

type AppManifestDocker struct {
	Image    string `yaml:"image,omitempty"`
	Username string `yaml:"username,omitempty"`
}

type AppManifestRoutes struct {
	Route string `yaml:"route,omitempty"`
}
<commit_msg>Add pointer type ot apps array<commit_after>package operation

type Manifest struct {
	Applications []*AppManifest `yaml:"applications"`
}

type AppManifest struct {
	Name                    string              `yaml:"name"`
	Buildpacks              []string            `yaml:"buildpacks,omitempty"`
	Command                 string              `yaml:"command,omitempty"`
	DiskQuota               string              `yaml:"disk_quota,omitempty"`
	Docker                  *AppManifestDocker  `yaml:"docker,omitempty"`
	Env                     map[string]string   `yaml:"env,omitempty"`
	HealthCheckType         string              `yaml:"health-check-type,omitempty"`
	HealthCheckHTTPEndpoint string              `yaml:"health-check-http-endpoint,omitempty"`
	Instances               int                 `yaml:"instances,omitempty"`
	LogRateLimit            string              `yaml:"log-rate-limit,omitempty"`
	Memory                  string              `yaml:"memory,omitempty"`
	NoRoute                 bool                `yaml:"no-route,omitempty"`
	Routes                  []AppManifestRoutes `yaml:"routes,omitempty"`
	Services                []string            `yaml:"services,omitempty"`
	Stack                   string              `yaml:"stack,omitempty"`
	Timeout                 int                 `yaml:"timeout,omitempty"`
}

type AppManifestDocker struct {
	Image    string `yaml:"image,omitempty"`
	Username string `yaml:"username,omitempty"`
}

type AppManifestRoutes struct {
	Route string `yaml:"route,omitempty"`
}
<|endoftext|>


<commit_before><commit_msg>Add ZEROXDEL constant<commit_after><|endoftext|>


<commit_before>package main

import (
	"github.com/codegangsta/cli"
	"os"
	"path/filepath"
	"runtime"
)

func homepath(p string) string {
	if runtime.GOOS == "windows" {
		return filepath.Join(os.Getenv("USERPROFILE"), p)
	}
	return filepath.Join(os.Getenv("HOME"), p)
}

func getDiscovery(c *cli.Context) string {
	if len(c.Args()) == 1 {
		return c.Args()[0]
	}
	return os.Getenv("SWARM_DISCOVERY")
}

var (
	flStore = cli.StringFlag{
		Name:  "rootdir",
		Value: homepath(".swarm"),
		Usage: "",
	}
	flAddr = cli.StringFlag{
		Name:   "addr",
		Value:  "127.0.0.1:4243",
		Usage:  "ip to advertise",
		EnvVar: "SWARM_ADDR",
	}
	flHosts = cli.StringSliceFlag{
		Name:   "host, H",
		Value:  &cli.StringSlice{"tcp://127.0.0.1:2375"},
		Usage:  "ip/socket to listen on",
		EnvVar: "SWARM_HOST",
	}
	flHeartBeat = cli.IntFlag{
		Name:  "heartbeat, hb",
		Value: 25,
		Usage: "time in second between each heartbeat",
	}
	flEnableCors = cli.BoolFlag{
		Name:  "api-enable-cors, cors",
		Usage: "enable CORS headers in the remote API",
	}
	flTls = cli.BoolFlag{
		Name:  "tls",
		Usage: "use TLS; implied by --tlsverify=true",
	}
	flTlsCaCert = cli.StringFlag{
		Name:  "tlscacert",
		Usage: "trust only remotes providing a certificate signed by the CA given here",
	}
	flTlsCert = cli.StringFlag{
		Name:  "tlscert",
		Usage: "path to TLS certificate file",
	}
	flTlsKey = cli.StringFlag{
		Name:  "tlskey",
		Usage: "path to TLS key file",
	}
	flTlsVerify = cli.BoolFlag{
		Name:  "tlsverify",
		Usage: "use TLS and verify the remote",
	}
	flOverCommit = cli.Float64Flag{
		Name:  "overcommit, oc",
		Usage: "overcommit to apply on resources",
		Value: 0.05,
	}
	flStrategy = cli.StringFlag{
		Name:  "strategy",
		Usage: "placement strategy to use [binpacking, random]",
		Value: "binpacking",
	}
	flFilter = cli.StringSliceFlag{
		Name:  "filter, f",
		Usage: "filter to use [constraint, affinity, health, port]",
		Value: &cli.StringSlice{"constraint", "affinity", "health", "port"},
	}
)
<commit_msg>flags: Cleanup the homepath() function.<commit_after>package main

import (
	"github.com/codegangsta/cli"
	"os"
	"path/filepath"
	"runtime"
)

func homepath(p string) string {
	home := os.Getenv("HOME")
	if runtime.GOOS == "windows" {
		home = os.Getenv("USERPROFILE")
	}
	return filepath.Join(home, p)
}

func getDiscovery(c *cli.Context) string {
	if len(c.Args()) == 1 {
		return c.Args()[0]
	}
	return os.Getenv("SWARM_DISCOVERY")
}

var (
	flStore = cli.StringFlag{
		Name:  "rootdir",
		Value: homepath(".swarm"),
		Usage: "",
	}
	flAddr = cli.StringFlag{
		Name:   "addr",
		Value:  "127.0.0.1:4243",
		Usage:  "ip to advertise",
		EnvVar: "SWARM_ADDR",
	}
	flHosts = cli.StringSliceFlag{
		Name:   "host, H",
		Value:  &cli.StringSlice{"tcp://127.0.0.1:2375"},
		Usage:  "ip/socket to listen on",
		EnvVar: "SWARM_HOST",
	}
	flHeartBeat = cli.IntFlag{
		Name:  "heartbeat, hb",
		Value: 25,
		Usage: "time in second between each heartbeat",
	}
	flEnableCors = cli.BoolFlag{
		Name:  "api-enable-cors, cors",
		Usage: "enable CORS headers in the remote API",
	}
	flTls = cli.BoolFlag{
		Name:  "tls",
		Usage: "use TLS; implied by --tlsverify=true",
	}
	flTlsCaCert = cli.StringFlag{
		Name:  "tlscacert",
		Usage: "trust only remotes providing a certificate signed by the CA given here",
	}
	flTlsCert = cli.StringFlag{
		Name:  "tlscert",
		Usage: "path to TLS certificate file",
	}
	flTlsKey = cli.StringFlag{
		Name:  "tlskey",
		Usage: "path to TLS key file",
	}
	flTlsVerify = cli.BoolFlag{
		Name:  "tlsverify",
		Usage: "use TLS and verify the remote",
	}
	flOverCommit = cli.Float64Flag{
		Name:  "overcommit, oc",
		Usage: "overcommit to apply on resources",
		Value: 0.05,
	}
	flStrategy = cli.StringFlag{
		Name:  "strategy",
		Usage: "placement strategy to use [binpacking, random]",
		Value: "binpacking",
	}
	flFilter = cli.StringSliceFlag{
		Name:  "filter, f",
		Usage: "filter to use [constraint, affinity, health, port]",
		Value: &cli.StringSlice{"constraint", "affinity", "health", "port"},
	}
)
<|endoftext|>


<commit_before>package mount // import "github.com/docker/docker/pkg/mount"

import (
	"fmt"
	"strings"
)

var flags = map[string]struct {
	clear bool
	flag  int
}{
	"defaults":      {false, 0},
	"ro":            {false, RDONLY},
	"rw":            {true, RDONLY},
	"suid":          {true, NOSUID},
	"nosuid":        {false, NOSUID},
	"dev":           {true, NODEV},
	"nodev":         {false, NODEV},
	"exec":          {true, NOEXEC},
	"noexec":        {false, NOEXEC},
	"sync":          {false, SYNCHRONOUS},
	"async":         {true, SYNCHRONOUS},
	"dirsync":       {false, DIRSYNC},
	"remount":       {false, REMOUNT},
	"mand":          {false, MANDLOCK},
	"nomand":        {true, MANDLOCK},
	"atime":         {true, NOATIME},
	"noatime":       {false, NOATIME},
	"diratime":      {true, NODIRATIME},
	"nodiratime":    {false, NODIRATIME},
	"bind":          {false, BIND},
	"rbind":         {false, RBIND},
	"unbindable":    {false, UNBINDABLE},
	"runbindable":   {false, RUNBINDABLE},
	"private":       {false, PRIVATE},
	"rprivate":      {false, RPRIVATE},
	"shared":        {false, SHARED},
	"rshared":       {false, RSHARED},
	"slave":         {false, SLAVE},
	"rslave":        {false, RSLAVE},
	"relatime":      {false, RELATIME},
	"norelatime":    {true, RELATIME},
	"strictatime":   {false, STRICTATIME},
	"nostrictatime": {true, STRICTATIME},
}

var validFlags = map[string]bool{
	"":          true,
	"size":      true,
	"mode":      true,
	"uid":       true,
	"gid":       true,
	"nr_inodes": true,
	"nr_blocks": true,
	"mpol":      true,
}

var propagationFlags = map[string]bool{
	"bind":        true,
	"rbind":       true,
	"unbindable":  true,
	"runbindable": true,
	"private":     true,
	"rprivate":    true,
	"shared":      true,
	"rshared":     true,
	"slave":       true,
	"rslave":      true,
}

// MergeTmpfsOptions merge mount options to make sure there is no duplicate.
func MergeTmpfsOptions(options []string) ([]string, error) {
	// We use collisions maps to remove duplicates.
	// For flag, the key is the flag value (the key for propagation flag is -1)
	// For data=value, the key is the data
	flagCollisions := map[int]bool{}
	dataCollisions := map[string]bool{}

	var newOptions []string
	// We process in reverse order
	for i := len(options) - 1; i >= 0; i-- {
		option := options[i]
		if option == "defaults" {
			continue
		}
		if f, ok := flags[option]; ok && f.flag != 0 {
			// There is only one propagation mode
			key := f.flag
			if propagationFlags[option] {
				key = -1
			}
			// Check to see if there is collision for flag
			if !flagCollisions[key] {
				// We prepend the option and add to collision map
				newOptions = append([]string{option}, newOptions...)
				flagCollisions[key] = true
			}
			continue
		}
		opt := strings.SplitN(option, "=", 2)
		if len(opt) != 2 || !validFlags[opt[0]] {
			return nil, fmt.Errorf("Invalid tmpfs option %q", opt)
		}
		if !dataCollisions[opt[0]] {
			// We prepend the option and add to collision map
			newOptions = append([]string{option}, newOptions...)
			dataCollisions[opt[0]] = true
		}
	}

	return newOptions, nil
}

// Parse fstab type mount options into mount() flags
// and device specific data
func parseOptions(options string) (int, string) {
	var (
		flag int
		data []string
	)

	for _, o := range strings.Split(options, ",") {
		// If the option does not exist in the flags table or the flag
		// is not supported on the platform,
		// then it is a data value for a specific fs type
		if f, exists := flags[o]; exists && f.flag != 0 {
			if f.clear {
				flag &= ^f.flag
			} else {
				flag |= f.flag
			}
		} else {
			data = append(data, o)
		}
	}
	return flag, strings.Join(data, ",")
}

// ParseTmpfsOptions parse fstab type mount options into flags and data
func ParseTmpfsOptions(options string) (int, string, error) {
	flags, data := parseOptions(options)
	for _, o := range strings.Split(data, ",") {
		opt := strings.SplitN(o, "=", 2)
		if !validFlags[opt[0]] {
			return 0, "", fmt.Errorf("Invalid tmpfs option %q", opt)
		}
	}
	return flags, data, nil
}
<commit_msg>pkg/mount: remove unused ParseTmpfsOptions<commit_after>package mount // import "github.com/docker/docker/pkg/mount"

import (
	"fmt"
	"strings"
)

var flags = map[string]struct {
	clear bool
	flag  int
}{
	"defaults":      {false, 0},
	"ro":            {false, RDONLY},
	"rw":            {true, RDONLY},
	"suid":          {true, NOSUID},
	"nosuid":        {false, NOSUID},
	"dev":           {true, NODEV},
	"nodev":         {false, NODEV},
	"exec":          {true, NOEXEC},
	"noexec":        {false, NOEXEC},
	"sync":          {false, SYNCHRONOUS},
	"async":         {true, SYNCHRONOUS},
	"dirsync":       {false, DIRSYNC},
	"remount":       {false, REMOUNT},
	"mand":          {false, MANDLOCK},
	"nomand":        {true, MANDLOCK},
	"atime":         {true, NOATIME},
	"noatime":       {false, NOATIME},
	"diratime":      {true, NODIRATIME},
	"nodiratime":    {false, NODIRATIME},
	"bind":          {false, BIND},
	"rbind":         {false, RBIND},
	"unbindable":    {false, UNBINDABLE},
	"runbindable":   {false, RUNBINDABLE},
	"private":       {false, PRIVATE},
	"rprivate":      {false, RPRIVATE},
	"shared":        {false, SHARED},
	"rshared":       {false, RSHARED},
	"slave":         {false, SLAVE},
	"rslave":        {false, RSLAVE},
	"relatime":      {false, RELATIME},
	"norelatime":    {true, RELATIME},
	"strictatime":   {false, STRICTATIME},
	"nostrictatime": {true, STRICTATIME},
}

var validFlags = map[string]bool{
	"":          true,
	"size":      true,
	"mode":      true,
	"uid":       true,
	"gid":       true,
	"nr_inodes": true,
	"nr_blocks": true,
	"mpol":      true,
}

var propagationFlags = map[string]bool{
	"bind":        true,
	"rbind":       true,
	"unbindable":  true,
	"runbindable": true,
	"private":     true,
	"rprivate":    true,
	"shared":      true,
	"rshared":     true,
	"slave":       true,
	"rslave":      true,
}

// MergeTmpfsOptions merge mount options to make sure there is no duplicate.
func MergeTmpfsOptions(options []string) ([]string, error) {
	// We use collisions maps to remove duplicates.
	// For flag, the key is the flag value (the key for propagation flag is -1)
	// For data=value, the key is the data
	flagCollisions := map[int]bool{}
	dataCollisions := map[string]bool{}

	var newOptions []string
	// We process in reverse order
	for i := len(options) - 1; i >= 0; i-- {
		option := options[i]
		if option == "defaults" {
			continue
		}
		if f, ok := flags[option]; ok && f.flag != 0 {
			// There is only one propagation mode
			key := f.flag
			if propagationFlags[option] {
				key = -1
			}
			// Check to see if there is collision for flag
			if !flagCollisions[key] {
				// We prepend the option and add to collision map
				newOptions = append([]string{option}, newOptions...)
				flagCollisions[key] = true
			}
			continue
		}
		opt := strings.SplitN(option, "=", 2)
		if len(opt) != 2 || !validFlags[opt[0]] {
			return nil, fmt.Errorf("Invalid tmpfs option %q", opt)
		}
		if !dataCollisions[opt[0]] {
			// We prepend the option and add to collision map
			newOptions = append([]string{option}, newOptions...)
			dataCollisions[opt[0]] = true
		}
	}

	return newOptions, nil
}

// Parse fstab type mount options into mount() flags
// and device specific data
func parseOptions(options string) (int, string) {
	var (
		flag int
		data []string
	)

	for _, o := range strings.Split(options, ",") {
		// If the option does not exist in the flags table or the flag
		// is not supported on the platform,
		// then it is a data value for a specific fs type
		if f, exists := flags[o]; exists && f.flag != 0 {
			if f.clear {
				flag &= ^f.flag
			} else {
				flag |= f.flag
			}
		} else {
			data = append(data, o)
		}
	}
	return flag, strings.Join(data, ",")
}
<|endoftext|>


<commit_before>package flenv

import (
	"errors"
	"flag"
	"net/url"
	"os"
	"reflect"
	"strconv"
	"strings"
	"time"
)

var ErrNotStruct = errors.New("Attempted to decode into a value that was not a pointer to struct")
var ErrUnsupportedFlagType = errors.New("Unsupported flag type")

func DecodeArgs(result interface{}) (*flag.FlagSet, error) {
	return Decode(result, os.Args)
}

func Decode(result interface{}, args []string) (*flag.FlagSet, error) {
	flagSet := flag.NewFlagSet("", flag.ContinueOnError)

	val := reflect.ValueOf(result)
	if val.Kind() != reflect.Ptr {
		return nil, ErrNotStruct
	}

	st := val.Elem()
	if st.Kind() != reflect.Struct {
		return nil, ErrNotStruct
	}

	typ := val.Elem().Type()

	for i := 0; i < st.NumField(); i++ {
		field := st.Field(i)
		typeField := typ.Field(i)

		// Get env tag.
		envName := typeField.Tag.Get("env")
		defaultValue := typeField.Tag.Get("default")

		// Set the value to the default value
		if err := setValue(field, envName, defaultValue); err != nil {
			return nil, err
		}

		// If there's a flag associated, we need to set that up, possibly with a default.
		flagNames := decodeFlagTag(typeField.Tag.Get("flag"))
		flagHelp := typeField.Tag.Get("help")
		if len(flagNames) > 0 {
			addFlag(flagSet, field, flagNames, flagHelp)
		}
	}

	return flagSet, flagSet.Parse(args)
}

func decodeFlagTag(tag string) (names []string) {
	bits := strings.Split(tag, ",")
	out := make([]string, 0, len(bits))
	for _, bit := range bits {
		trimmed := strings.TrimLeft(bit, "-")
		if len(trimmed) > 0 {
			out = append(out, trimmed)
		}
	}

	return out
}

func addFlag(flagSet *flag.FlagSet, field reflect.Value, names []string, help string) error {
	for _, name := range names {
		switch field.Kind() {
		case reflect.Bool:
			flagSet.BoolVar(field.Addr().Interface().(*bool), name, field.Bool(), help)
		case reflect.Float32, reflect.Float64:
			flagSet.Float64Var(field.Addr().Interface().(*float64), name, field.Float(), help)
		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32:
			flagSet.IntVar(field.Addr().Interface().(*int), name, int(field.Int()), help)
		case reflect.Int64:
			if t := field.Type(); t.PkgPath() == "time" && t.Name() == "Duration" {
				flagSet.DurationVar(field.Addr().Interface().(*time.Duration), name, time.Duration(field.Int()), help)
			} else {
				flagSet.Int64Var(field.Addr().Interface().(*int64), name, field.Int(), help)
			}
		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
			flagSet.Uint64Var(field.Addr().Interface().(*uint64), name, field.Uint(), help)
		case reflect.String:
			flagSet.StringVar(field.Addr().Interface().(*string), name, field.String(), help)
		default:
			// TODO: We can make a URL parser, but not right now.
			return ErrUnsupportedFlagType
		}
	}
	return nil
}

func setValue(field reflect.Value, name, def string) error {
	tmp := os.Getenv(name)
	if tmp == "" {
		tmp = def
	}

	if tmp == "" {
		return nil
	}

	switch field.Kind() {
	case reflect.Bool:
		v, err := strconv.ParseBool(tmp)
		if err != nil {
			return err
		}
		field.SetBool(v)

	case reflect.Float32, reflect.Float64:
		bits := field.Type().Bits()
		v, err := strconv.ParseFloat(tmp, bits)
		if err != nil {
			return err
		}
		field.SetFloat(v)

	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		if t := field.Type(); t.PkgPath() == "time" && t.Name() == "Duration" {
			v, err := time.ParseDuration(tmp)
			if err != nil {
				return err
			}
			field.SetInt(int64(v))

		} else {
			bits := field.Type().Bits()
			v, err := strconv.ParseInt(tmp, 0, bits)
			if err != nil {
				return err
			}
			field.SetInt(v)
		}
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		bits := field.Type().Bits()
		v, err := strconv.ParseUint(tmp, 0, bits)
		if err != nil {
			return err
		}
		field.SetUint(v)

	case reflect.String:
		field.SetString(tmp)

	case reflect.Ptr:
		if t := field.Type().Elem(); t.Kind() == reflect.Struct && t.PkgPath() == "net/url" && t.Name() == "URL" {
			v, err := url.Parse(tmp)
			if err != nil {
				return err
			}

			field.Set(reflect.ValueOf(v))
		}
	}
	return nil
}
<commit_msg>Add documentation to Decode<commit_after>package flenv

import (
	"errors"
	"flag"
	"net/url"
	"os"
	"reflect"
	"strconv"
	"strings"
	"time"
)

// ErrNotStruct is an error
var ErrNotStruct = errors.New("Attempted to decode into a value that was not a pointer to struct")

// ErrNotStruct is an error
var ErrUnsupportedFlagType = errors.New("Unsupported flag type")

// DecodeArgs runs decode with os.Args
func DecodeArgs(result interface{}) (*flag.FlagSet, error) {
	return Decode(result, os.Args)
}

// Decode attempts to populate result, based on struct tags from args and the environment
//
// `result` should be a struct with at least one of the following tags
//
//    env:"ENVIRONMENT_VARIABLE"
//    flag:"--environment-variable"
//
// Additionally, the following tags are also allowed
//
//    help:"help text for flag parsing"
//    default:"default value"
func Decode(result interface{}, args []string) (*flag.FlagSet, error) {
	flagSet := flag.NewFlagSet("", flag.ContinueOnError)

	val := reflect.ValueOf(result)
	if val.Kind() != reflect.Ptr {
		return nil, ErrNotStruct
	}

	st := val.Elem()
	if st.Kind() != reflect.Struct {
		return nil, ErrNotStruct
	}

	typ := val.Elem().Type()

	for i := 0; i < st.NumField(); i++ {
		field := st.Field(i)
		typeField := typ.Field(i)

		// Get env tag.
		envName := typeField.Tag.Get("env")
		defaultValue := typeField.Tag.Get("default")

		// Set the value to the default value
		if err := setValue(field, envName, defaultValue); err != nil {
			return nil, err
		}

		// If there's a flag associated, we need to set that up, possibly with a default.
		flagNames := decodeFlagTag(typeField.Tag.Get("flag"))
		flagHelp := typeField.Tag.Get("help")
		if len(flagNames) > 0 {
			addFlag(flagSet, field, flagNames, flagHelp)
		}
	}

	return flagSet, flagSet.Parse(args)
}

func decodeFlagTag(tag string) (names []string) {
	bits := strings.Split(tag, ",")
	out := make([]string, 0, len(bits))
	for _, bit := range bits {
		trimmed := strings.TrimLeft(bit, "-")
		if len(trimmed) > 0 {
			out = append(out, trimmed)
		}
	}

	return out
}

func addFlag(flagSet *flag.FlagSet, field reflect.Value, names []string, help string) error {
	for _, name := range names {
		switch field.Kind() {
		case reflect.Bool:
			flagSet.BoolVar(field.Addr().Interface().(*bool), name, field.Bool(), help)
		case reflect.Float32, reflect.Float64:
			flagSet.Float64Var(field.Addr().Interface().(*float64), name, field.Float(), help)
		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32:
			flagSet.IntVar(field.Addr().Interface().(*int), name, int(field.Int()), help)
		case reflect.Int64:
			if t := field.Type(); t.PkgPath() == "time" && t.Name() == "Duration" {
				flagSet.DurationVar(field.Addr().Interface().(*time.Duration), name, time.Duration(field.Int()), help)
			} else {
				flagSet.Int64Var(field.Addr().Interface().(*int64), name, field.Int(), help)
			}
		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
			flagSet.Uint64Var(field.Addr().Interface().(*uint64), name, field.Uint(), help)
		case reflect.String:
			flagSet.StringVar(field.Addr().Interface().(*string), name, field.String(), help)
		default:
			// TODO: We can make a URL parser, but not right now.
			return ErrUnsupportedFlagType
		}
	}
	return nil
}

func setValue(field reflect.Value, name, def string) error {
	tmp := os.Getenv(name)
	if tmp == "" {
		tmp = def
	}

	if tmp == "" {
		return nil
	}

	switch field.Kind() {
	case reflect.Bool:
		v, err := strconv.ParseBool(tmp)
		if err != nil {
			return err
		}
		field.SetBool(v)

	case reflect.Float32, reflect.Float64:
		bits := field.Type().Bits()
		v, err := strconv.ParseFloat(tmp, bits)
		if err != nil {
			return err
		}
		field.SetFloat(v)

	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		if t := field.Type(); t.PkgPath() == "time" && t.Name() == "Duration" {
			v, err := time.ParseDuration(tmp)
			if err != nil {
				return err
			}
			field.SetInt(int64(v))

		} else {
			bits := field.Type().Bits()
			v, err := strconv.ParseInt(tmp, 0, bits)
			if err != nil {
				return err
			}
			field.SetInt(v)
		}
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		bits := field.Type().Bits()
		v, err := strconv.ParseUint(tmp, 0, bits)
		if err != nil {
			return err
		}
		field.SetUint(v)

	case reflect.String:
		field.SetString(tmp)

	case reflect.Ptr:
		if t := field.Type().Elem(); t.Kind() == reflect.Struct && t.PkgPath() == "net/url" && t.Name() == "URL" {
			v, err := url.Parse(tmp)
			if err != nil {
				return err
			}

			field.Set(reflect.ValueOf(v))
		}
	}
	return nil
}
<|endoftext|>