<commit_before>package enry

import (
	"bufio"
	"bytes"
	"path/filepath"
	"regexp"
	"strings"

	"gopkg.in/src-d/enry.v1/data"
)

// OtherLanguage is used as a zero value when a function can not return a specific language.
const OtherLanguage = ""

// Strategy type fix the signature for the functions that can be used as a strategy.
type Strategy func(filename string, content []byte, candidates []string) (languages []string)

// DefaultStrategies is the strategies' sequence GetLanguage uses to detect languages.
var DefaultStrategies = []Strategy{
	GetLanguagesByModeline,
	GetLanguagesByFilename,
	GetLanguagesByShebang,
	GetLanguagesByExtension,
	GetLanguagesByContent,
	GetLanguagesByClassifier,
}

var DefaultClassifier Classifier = &classifier{
	languagesLogProbabilities: data.LanguagesLogProbabilities,
	tokensLogProbabilities:    data.TokensLogProbabilities,
	tokensTotal:               data.TokensTotal,
}

// GetLanguage applies a sequence of strategies based on the given filename and content
// to find out the most probably language to return.
func GetLanguage(filename string, content []byte) (language string) {
	languages := GetLanguages(filename, content)
	return firstLanguage(languages)
}

func firstLanguage(languages []string) string {
	if len(languages) == 0 {
		return OtherLanguage
	}

	return languages[0]
}

// GetLanguageByModeline returns detected language. If there are more than one possibles languages
// it returns the first language by alphabetically order and safe to false.
func GetLanguageByModeline(content []byte) (language string, safe bool) {
	return getLanguageByStrategy(GetLanguagesByModeline, "", content, nil)
}

// GetLanguageByEmacsModeline returns detected language. If there are more than one possibles languages
// it returns the first language by alphabetically order and safe to false.
func GetLanguageByEmacsModeline(content []byte) (language string, safe bool) {
	return getLanguageByStrategy(GetLanguagesByEmacsModeline, "", content, nil)
}

// GetLanguageByVimModeline returns detected language. If there are more than one possibles languages
// it returns the first language by alphabetically order and safe to false.
func GetLanguageByVimModeline(content []byte) (language string, safe bool) {
	return getLanguageByStrategy(GetLanguagesByVimModeline, "", content, nil)
}

// GetLanguageByFilename returns detected language. If there are more than one possibles languages
// it returns the first language by alphabetically order and safe to false.
func GetLanguageByFilename(filename string) (language string, safe bool) {
	return getLanguageByStrategy(GetLanguagesByFilename, filename, nil, nil)
}

// GetLanguageByShebang returns detected language. If there are more than one possibles languages
// it returns the first language by alphabetically order and safe to false.
func GetLanguageByShebang(content []byte) (language string, safe bool) {
	return getLanguageByStrategy(GetLanguagesByShebang, "", content, nil)
}

// GetLanguageByExtension returns detected language. If there are more than one possibles languages
// it returns the first language by alphabetically order and safe to false.
func GetLanguageByExtension(filename string) (language string, safe bool) {
	return getLanguageByStrategy(GetLanguagesByExtension, filename, nil, nil)
}

// GetLanguageByContent returns detected language. If there are more than one possibles languages
// it returns the first language by alphabetically order and safe to false.
func GetLanguageByContent(filename string, content []byte) (language string, safe bool) {
	return getLanguageByStrategy(GetLanguagesByContent, filename, content, nil)
}

// GetLanguageByClassifier returns the most probably language detected for the given content. It uses
// DefaultClassifier, if no candidates are provided it returns OtherLanguage.
func GetLanguageByClassifier(content []byte, candidates []string) (language string, safe bool) {
	return getLanguageByStrategy(GetLanguagesByClassifier, "", content, candidates)
}

func getLanguageByStrategy(strategy Strategy, filename string, content []byte, candidates []string) (string, bool) {
	languages := strategy(filename, content, candidates)
	return getFirstLanguageAndSafe(languages)
}

func getFirstLanguageAndSafe(languages []string) (language string, safe bool) {
	language = firstLanguage(languages)
	safe = len(languages) == 1
	return
}

// GetLanguageBySpecificClassifier returns the most probably language for the given content using
// classifier to detect language.
func GetLanguageBySpecificClassifier(content []byte, candidates []string, classifier Classifier) (language string, safe bool) {
	languages := GetLanguagesBySpecificClassifier(content, candidates, classifier)
	return getFirstLanguageAndSafe(languages)
}

// GetLanguages applies a sequence of strategies based on the given filename and content
// to find out the most probably languages to return.
func GetLanguages(filename string, content []byte) []string {
	if IsBinary(content) {
		return nil
	}

	var languages []string
	candidates := []string{}
	for _, strategy := range DefaultStrategies {
		languages = strategy(filename, content, candidates)
		if len(languages) == 1 {
			return languages
		}

		if len(languages) > 0 {
			candidates = append(candidates, languages...)
		}
	}

	return languages
}

// GetLanguagesByModeline returns a slice of possible languages for the given content.
// It complies with the signature to be a Strategy type.
func GetLanguagesByModeline(_ string, content []byte, candidates []string) []string {
	headFoot := getHeaderAndFooter(content)
	var languages []string
	for _, getLang := range modelinesFunc {
		languages = getLang("", headFoot, candidates)
		if len(languages) > 0 {
			break
		}
	}

	return languages
}

var modelinesFunc = []Strategy{
	GetLanguagesByEmacsModeline,
	GetLanguagesByVimModeline,
}

func getHeaderAndFooter(content []byte) []byte {
	const searchScope = 5

	if len(content) == 0 {
		return content
	}

	if bytes.Count(content, []byte("\n")) < 2*searchScope {
		return content
	}

	header := headScope(content, searchScope)
	footer := footScope(content, searchScope)
	headerAndFooter := make([]byte, 0, len(content[:header])+len(content[footer:]))
	headerAndFooter = append(headerAndFooter, content[:header]...)
	headerAndFooter = append(headerAndFooter, content[footer:]...)
	return headerAndFooter
}

func headScope(content []byte, scope int) (index int) {
	for i := 0; i < scope; i++ {
		eol := bytes.IndexAny(content, "\n")
		content = content[eol+1:]
		index += eol
	}

	return index + scope - 1
}

func footScope(content []byte, scope int) (index int) {
	for i := 0; i < scope; i++ {
		index = bytes.LastIndexAny(content, "\n")
		content = content[:index]
	}

	return index + 1
}

var (
	reEmacsModeline = regexp.MustCompile(`.*-\*-\s*(.+?)\s*-\*-.*(?m:$)`)
	reEmacsLang     = regexp.MustCompile(`.*(?i:mode)\s*:\s*([^\s;]+)\s*;*.*`)
	reVimModeline   = regexp.MustCompile(`(?:(?m:\s|^)vi(?:m[<=>]?\d+|m)?|[\t\x20]*ex)\s*[:]\s*(.*)(?m:$)`)
	reVimLang       = regexp.MustCompile(`(?i:filetype|ft|syntax)\s*=(\w+)(?:\s|:|$)`)
)

// GetLanguagesByEmacsModeline returns a slice of possible languages for the given content.
// It complies with the signature to be a Strategy type.
func GetLanguagesByEmacsModeline(_ string, content []byte, _ []string) []string {
	matched := reEmacsModeline.FindAllSubmatch(content, -1)
	if matched == nil {
		return nil
	}

	// only take the last matched line, discard previous lines
	lastLineMatched := matched[len(matched)-1][1]
	matchedAlias := reEmacsLang.FindSubmatch(lastLineMatched)
	var alias string
	if matchedAlias != nil {
		alias = string(matchedAlias[1])
	} else {
		alias = string(lastLineMatched)
	}

	language, ok := GetLanguageByAlias(alias)
	if !ok {
		return nil
	}

	return []string{language}
}

// GetLanguagesByVimModeline returns a slice of possible languages for the given content.
// It complies with the signature to be a Strategy type.
func GetLanguagesByVimModeline(_ string, content []byte, _ []string) []string {
	matched := reVimModeline.FindAllSubmatch(content, -1)
	if matched == nil {
		return nil
	}

	// only take the last matched line, discard previous lines
	lastLineMatched := matched[len(matched)-1][1]
	matchedAlias := reVimLang.FindAllSubmatch(lastLineMatched, -1)
	if matchedAlias == nil {
		return nil
	}

	alias := string(matchedAlias[0][1])
	if len(matchedAlias) > 1 {
		// cases:
		// matchedAlias = [["syntax=ruby " "ruby"] ["ft=python " "python"] ["filetype=perl " "perl"]] returns OtherLanguage;
		// matchedAlias = [["syntax=python " "python"] ["ft=python " "python"] ["filetype=python " "python"]] returns "Python";
		for _, match := range matchedAlias {
			otherAlias := string(match[1])
			if otherAlias != alias {
				return nil
			}
		}
	}

	language, ok := GetLanguageByAlias(alias)
	if !ok {
		return nil
	}

	return []string{language}
}

// GetLanguagesByFilename returns a slice of possible languages for the given filename.
// It complies with the signature to be a Strategy type.
func GetLanguagesByFilename(filename string, _ []byte, _ []string) []string {
	return data.LanguagesByFilename[filepath.Base(filename)]
}

// GetLanguagesByShebang returns a slice of possible languages for the given content.
// It complies with the signature to be a Strategy type.
func GetLanguagesByShebang(_ string, content []byte, _ []string) (languages []string) {
	interpreter := getInterpreter(content)
	return data.LanguagesByInterpreter[interpreter]
}

var (
	shebangExecHack = regexp.MustCompile(`exec (\w+).+\$0.+\$@`)
	pythonVersion   = regexp.MustCompile(`python\d\.\d+`)
)

func getInterpreter(data []byte) (interpreter string) {
	line := getFirstLine(data)
	if !hasShebang(line) {
		return ""
	}

	// skip shebang
	line = bytes.TrimSpace(line[2:])
	splitted := bytes.Fields(line)
	if len(splitted) == 0 {
		return ""
	}

	if bytes.Contains(splitted[0], []byte("env")) {
		if len(splitted) > 1 {
			interpreter = string(splitted[1])
		}
	} else {
		splittedPath := bytes.Split(splitted[0], []byte{'/'})
		interpreter = string(splittedPath[len(splittedPath)-1])
	}

	if interpreter == "sh" {
		interpreter = lookForMultilineExec(data)
	}

	if pythonVersion.MatchString(interpreter) {
		interpreter = interpreter[:strings.Index(interpreter, `.`)]
	}

	return
}

func getFirstLine(data []byte) []byte {
	buf := bufio.NewScanner(bytes.NewReader(data))
	buf.Scan()
	line := buf.Bytes()
	if err := buf.Err(); err != nil {
		return nil
	}

	return line
}

func hasShebang(line []byte) bool {
	const shebang = `#!`
	prefix := []byte(shebang)
	return bytes.HasPrefix(line, prefix)
}

func lookForMultilineExec(data []byte) string {
	const magicNumOfLines = 5
	interpreter := "sh"

	buf := bufio.NewScanner(bytes.NewReader(data))
	for i := 0; i < magicNumOfLines && buf.Scan(); i++ {
		line := buf.Bytes()
		if shebangExecHack.Match(line) {
			interpreter = shebangExecHack.FindStringSubmatch(string(line))[1]
			break
		}
	}

	if err := buf.Err(); err != nil {
		return interpreter
	}

	return interpreter
}

// GetLanguagesByExtension returns a slice of possible languages for the given filename.
// It complies with the signature to be a Strategy type.
func GetLanguagesByExtension(filename string, _ []byte, _ []string) []string {
	if !strings.Contains(filename, ".") {
		return nil
	}

	filename = strings.ToLower(filename)
	dots := getDotIndexes(filename)
	for _, dot := range dots {
		ext := filename[dot:]
		languages, ok := data.LanguagesByExtension[ext]
		if ok {
			return languages
		}
	}

	return nil
}

func getDotIndexes(filename string) []int {
	dots := make([]int, 0, 2)
	for i, letter := range filename {
		if letter == rune('.') {
			dots = append(dots, i)
		}
	}

	return dots
}

// GetLanguagesByContent returns a slice of possible languages for the given content.
// It complies with the signature to be a Strategy type.
func GetLanguagesByContent(filename string, content []byte, _ []string) []string {
	ext := strings.ToLower(filepath.Ext(filename))
	fnMatcher, ok := data.ContentMatchers[ext]
	if !ok {
		return nil
	}

	return fnMatcher(content)
}

// GetLanguagesByClassifier uses DefaultClassifier as a Classifier and returns a sorted slice of possible languages ordered by
// decreasing language's probability. If there are not candidates it returns nil. It complies with the signature to be a Strategy type.
func GetLanguagesByClassifier(filename string, content []byte, candidates []string) (languages []string) {
	if len(candidates) == 0 {
		return nil
	}

	return GetLanguagesBySpecificClassifier(content, candidates, DefaultClassifier)
}

// GetLanguagesBySpecificClassifier returns a slice of possible languages. It takes in a Classifier to be used.
func GetLanguagesBySpecificClassifier(content []byte, candidates []string, classifier Classifier) (languages []string) {
	mapCandidates := make(map[string]float64)
	for _, candidate := range candidates {
		mapCandidates[candidate]++
	}

	return classifier.Classify(content, mapCandidates)
}

// GetLanguageExtensions returns the different extensions being used by the language.
func GetLanguageExtensions(language string) []string {
	return data.ExtensionsByLanguage[language]
}

// Type represent language's type. Either data, programming, markup, prose, or unknown.
type Type int

// Type's values.
const (
	Unknown Type = iota
	Data
	Programming
	Markup
	Prose
)

// GetLanguageType returns the type of the given language.
func GetLanguageType(language string) (langType Type) {
	intType, ok := data.LanguagesType[language]
	langType = Type(intType)
	if !ok {
		langType = Unknown
	}
	return langType
}

// GetLanguageByAlias returns either the language related to the given alias and ok set to true
// or Otherlanguage and ok set to false if the alias is not recognized.
func GetLanguageByAlias(alias string) (lang string, ok bool) {
	a := strings.Split(alias, `,`)[0]
	a = strings.ToLower(a)
	lang, ok = data.LanguagesByAlias[a]
	if !ok {
		lang = OtherLanguage
	}

	return
}
<commit_msg>common: Return nil on empty filenames<commit_after>package enry

import (
	"bufio"
	"bytes"
	"path/filepath"
	"regexp"
	"strings"

	"gopkg.in/src-d/enry.v1/data"
)

// OtherLanguage is used as a zero value when a function can not return a specific language.
const OtherLanguage = ""

// Strategy type fix the signature for the functions that can be used as a strategy.
type Strategy func(filename string, content []byte, candidates []string) (languages []string)

// DefaultStrategies is the strategies' sequence GetLanguage uses to detect languages.
var DefaultStrategies = []Strategy{
	GetLanguagesByModeline,
	GetLanguagesByFilename,
	GetLanguagesByShebang,
	GetLanguagesByExtension,
	GetLanguagesByContent,
	GetLanguagesByClassifier,
}

var DefaultClassifier Classifier = &classifier{
	languagesLogProbabilities: data.LanguagesLogProbabilities,
	tokensLogProbabilities:    data.TokensLogProbabilities,
	tokensTotal:               data.TokensTotal,
}

// GetLanguage applies a sequence of strategies based on the given filename and content
// to find out the most probably language to return.
func GetLanguage(filename string, content []byte) (language string) {
	languages := GetLanguages(filename, content)
	return firstLanguage(languages)
}

func firstLanguage(languages []string) string {
	if len(languages) == 0 {
		return OtherLanguage
	}

	return languages[0]
}

// GetLanguageByModeline returns detected language. If there are more than one possibles languages
// it returns the first language by alphabetically order and safe to false.
func GetLanguageByModeline(content []byte) (language string, safe bool) {
	return getLanguageByStrategy(GetLanguagesByModeline, "", content, nil)
}

// GetLanguageByEmacsModeline returns detected language. If there are more than one possibles languages
// it returns the first language by alphabetically order and safe to false.
func GetLanguageByEmacsModeline(content []byte) (language string, safe bool) {
	return getLanguageByStrategy(GetLanguagesByEmacsModeline, "", content, nil)
}

// GetLanguageByVimModeline returns detected language. If there are more than one possibles languages
// it returns the first language by alphabetically order and safe to false.
func GetLanguageByVimModeline(content []byte) (language string, safe bool) {
	return getLanguageByStrategy(GetLanguagesByVimModeline, "", content, nil)
}

// GetLanguageByFilename returns detected language. If there are more than one possibles languages
// it returns the first language by alphabetically order and safe to false.
func GetLanguageByFilename(filename string) (language string, safe bool) {
	return getLanguageByStrategy(GetLanguagesByFilename, filename, nil, nil)
}

// GetLanguageByShebang returns detected language. If there are more than one possibles languages
// it returns the first language by alphabetically order and safe to false.
func GetLanguageByShebang(content []byte) (language string, safe bool) {
	return getLanguageByStrategy(GetLanguagesByShebang, "", content, nil)
}

// GetLanguageByExtension returns detected language. If there are more than one possibles languages
// it returns the first language by alphabetically order and safe to false.
func GetLanguageByExtension(filename string) (language string, safe bool) {
	return getLanguageByStrategy(GetLanguagesByExtension, filename, nil, nil)
}

// GetLanguageByContent returns detected language. If there are more than one possibles languages
// it returns the first language by alphabetically order and safe to false.
func GetLanguageByContent(filename string, content []byte) (language string, safe bool) {
	return getLanguageByStrategy(GetLanguagesByContent, filename, content, nil)
}

// GetLanguageByClassifier returns the most probably language detected for the given content. It uses
// DefaultClassifier, if no candidates are provided it returns OtherLanguage.
func GetLanguageByClassifier(content []byte, candidates []string) (language string, safe bool) {
	return getLanguageByStrategy(GetLanguagesByClassifier, "", content, candidates)
}

func getLanguageByStrategy(strategy Strategy, filename string, content []byte, candidates []string) (string, bool) {
	languages := strategy(filename, content, candidates)
	return getFirstLanguageAndSafe(languages)
}

func getFirstLanguageAndSafe(languages []string) (language string, safe bool) {
	language = firstLanguage(languages)
	safe = len(languages) == 1
	return
}

// GetLanguageBySpecificClassifier returns the most probably language for the given content using
// classifier to detect language.
func GetLanguageBySpecificClassifier(content []byte, candidates []string, classifier Classifier) (language string, safe bool) {
	languages := GetLanguagesBySpecificClassifier(content, candidates, classifier)
	return getFirstLanguageAndSafe(languages)
}

// GetLanguages applies a sequence of strategies based on the given filename and content
// to find out the most probably languages to return.
func GetLanguages(filename string, content []byte) []string {
	if IsBinary(content) {
		return nil
	}

	var languages []string
	candidates := []string{}
	for _, strategy := range DefaultStrategies {
		languages = strategy(filename, content, candidates)
		if len(languages) == 1 {
			return languages
		}

		if len(languages) > 0 {
			candidates = append(candidates, languages...)
		}
	}

	return languages
}

// GetLanguagesByModeline returns a slice of possible languages for the given content.
// It complies with the signature to be a Strategy type.
func GetLanguagesByModeline(_ string, content []byte, candidates []string) []string {
	headFoot := getHeaderAndFooter(content)
	var languages []string
	for _, getLang := range modelinesFunc {
		languages = getLang("", headFoot, candidates)
		if len(languages) > 0 {
			break
		}
	}

	return languages
}

var modelinesFunc = []Strategy{
	GetLanguagesByEmacsModeline,
	GetLanguagesByVimModeline,
}

func getHeaderAndFooter(content []byte) []byte {
	const searchScope = 5

	if len(content) == 0 {
		return content
	}

	if bytes.Count(content, []byte("\n")) < 2*searchScope {
		return content
	}

	header := headScope(content, searchScope)
	footer := footScope(content, searchScope)
	headerAndFooter := make([]byte, 0, len(content[:header])+len(content[footer:]))
	headerAndFooter = append(headerAndFooter, content[:header]...)
	headerAndFooter = append(headerAndFooter, content[footer:]...)
	return headerAndFooter
}

func headScope(content []byte, scope int) (index int) {
	for i := 0; i < scope; i++ {
		eol := bytes.IndexAny(content, "\n")
		content = content[eol+1:]
		index += eol
	}

	return index + scope - 1
}

func footScope(content []byte, scope int) (index int) {
	for i := 0; i < scope; i++ {
		index = bytes.LastIndexAny(content, "\n")
		content = content[:index]
	}

	return index + 1
}

var (
	reEmacsModeline = regexp.MustCompile(`.*-\*-\s*(.+?)\s*-\*-.*(?m:$)`)
	reEmacsLang     = regexp.MustCompile(`.*(?i:mode)\s*:\s*([^\s;]+)\s*;*.*`)
	reVimModeline   = regexp.MustCompile(`(?:(?m:\s|^)vi(?:m[<=>]?\d+|m)?|[\t\x20]*ex)\s*[:]\s*(.*)(?m:$)`)
	reVimLang       = regexp.MustCompile(`(?i:filetype|ft|syntax)\s*=(\w+)(?:\s|:|$)`)
)

// GetLanguagesByEmacsModeline returns a slice of possible languages for the given content.
// It complies with the signature to be a Strategy type.
func GetLanguagesByEmacsModeline(_ string, content []byte, _ []string) []string {
	matched := reEmacsModeline.FindAllSubmatch(content, -1)
	if matched == nil {
		return nil
	}

	// only take the last matched line, discard previous lines
	lastLineMatched := matched[len(matched)-1][1]
	matchedAlias := reEmacsLang.FindSubmatch(lastLineMatched)
	var alias string
	if matchedAlias != nil {
		alias = string(matchedAlias[1])
	} else {
		alias = string(lastLineMatched)
	}

	language, ok := GetLanguageByAlias(alias)
	if !ok {
		return nil
	}

	return []string{language}
}

// GetLanguagesByVimModeline returns a slice of possible languages for the given content.
// It complies with the signature to be a Strategy type.
func GetLanguagesByVimModeline(_ string, content []byte, _ []string) []string {
	matched := reVimModeline.FindAllSubmatch(content, -1)
	if matched == nil {
		return nil
	}

	// only take the last matched line, discard previous lines
	lastLineMatched := matched[len(matched)-1][1]
	matchedAlias := reVimLang.FindAllSubmatch(lastLineMatched, -1)
	if matchedAlias == nil {
		return nil
	}

	alias := string(matchedAlias[0][1])
	if len(matchedAlias) > 1 {
		// cases:
		// matchedAlias = [["syntax=ruby " "ruby"] ["ft=python " "python"] ["filetype=perl " "perl"]] returns OtherLanguage;
		// matchedAlias = [["syntax=python " "python"] ["ft=python " "python"] ["filetype=python " "python"]] returns "Python";
		for _, match := range matchedAlias {
			otherAlias := string(match[1])
			if otherAlias != alias {
				return nil
			}
		}
	}

	language, ok := GetLanguageByAlias(alias)
	if !ok {
		return nil
	}

	return []string{language}
}

// GetLanguagesByFilename returns a slice of possible languages for the given filename.
// It complies with the signature to be a Strategy type.
func GetLanguagesByFilename(filename string, _ []byte, _ []string) []string {
	if filename == "" {
		return nil
	}

	return data.LanguagesByFilename[filepath.Base(filename)]
}

// GetLanguagesByShebang returns a slice of possible languages for the given content.
// It complies with the signature to be a Strategy type.
func GetLanguagesByShebang(_ string, content []byte, _ []string) (languages []string) {
	interpreter := getInterpreter(content)
	return data.LanguagesByInterpreter[interpreter]
}

var (
	shebangExecHack = regexp.MustCompile(`exec (\w+).+\$0.+\$@`)
	pythonVersion   = regexp.MustCompile(`python\d\.\d+`)
)

func getInterpreter(data []byte) (interpreter string) {
	line := getFirstLine(data)
	if !hasShebang(line) {
		return ""
	}

	// skip shebang
	line = bytes.TrimSpace(line[2:])
	splitted := bytes.Fields(line)
	if len(splitted) == 0 {
		return ""
	}

	if bytes.Contains(splitted[0], []byte("env")) {
		if len(splitted) > 1 {
			interpreter = string(splitted[1])
		}
	} else {
		splittedPath := bytes.Split(splitted[0], []byte{'/'})
		interpreter = string(splittedPath[len(splittedPath)-1])
	}

	if interpreter == "sh" {
		interpreter = lookForMultilineExec(data)
	}

	if pythonVersion.MatchString(interpreter) {
		interpreter = interpreter[:strings.Index(interpreter, `.`)]
	}

	return
}

func getFirstLine(data []byte) []byte {
	buf := bufio.NewScanner(bytes.NewReader(data))
	buf.Scan()
	line := buf.Bytes()
	if err := buf.Err(); err != nil {
		return nil
	}

	return line
}

func hasShebang(line []byte) bool {
	const shebang = `#!`
	prefix := []byte(shebang)
	return bytes.HasPrefix(line, prefix)
}

func lookForMultilineExec(data []byte) string {
	const magicNumOfLines = 5
	interpreter := "sh"

	buf := bufio.NewScanner(bytes.NewReader(data))
	for i := 0; i < magicNumOfLines && buf.Scan(); i++ {
		line := buf.Bytes()
		if shebangExecHack.Match(line) {
			interpreter = shebangExecHack.FindStringSubmatch(string(line))[1]
			break
		}
	}

	if err := buf.Err(); err != nil {
		return interpreter
	}

	return interpreter
}

// GetLanguagesByExtension returns a slice of possible languages for the given filename.
// It complies with the signature to be a Strategy type.
func GetLanguagesByExtension(filename string, _ []byte, _ []string) []string {
	if !strings.Contains(filename, ".") {
		return nil
	}

	filename = strings.ToLower(filename)
	dots := getDotIndexes(filename)
	for _, dot := range dots {
		ext := filename[dot:]
		languages, ok := data.LanguagesByExtension[ext]
		if ok {
			return languages
		}
	}

	return nil
}

func getDotIndexes(filename string) []int {
	dots := make([]int, 0, 2)
	for i, letter := range filename {
		if letter == rune('.') {
			dots = append(dots, i)
		}
	}

	return dots
}

// GetLanguagesByContent returns a slice of possible languages for the given content.
// It complies with the signature to be a Strategy type.
func GetLanguagesByContent(filename string, content []byte, _ []string) []string {
	if filename == "" {
		return nil
	}

	ext := strings.ToLower(filepath.Ext(filename))
	fnMatcher, ok := data.ContentMatchers[ext]
	if !ok {
		return nil
	}

	return fnMatcher(content)
}

// GetLanguagesByClassifier uses DefaultClassifier as a Classifier and returns a sorted slice of possible languages ordered by
// decreasing language's probability. If there are not candidates it returns nil. It complies with the signature to be a Strategy type.
func GetLanguagesByClassifier(filename string, content []byte, candidates []string) (languages []string) {
	if len(candidates) == 0 {
		return nil
	}

	return GetLanguagesBySpecificClassifier(content, candidates, DefaultClassifier)
}

// GetLanguagesBySpecificClassifier returns a slice of possible languages. It takes in a Classifier to be used.
func GetLanguagesBySpecificClassifier(content []byte, candidates []string, classifier Classifier) (languages []string) {
	mapCandidates := make(map[string]float64)
	for _, candidate := range candidates {
		mapCandidates[candidate]++
	}

	return classifier.Classify(content, mapCandidates)
}

// GetLanguageExtensions returns the different extensions being used by the language.
func GetLanguageExtensions(language string) []string {
	return data.ExtensionsByLanguage[language]
}

// Type represent language's type. Either data, programming, markup, prose, or unknown.
type Type int

// Type's values.
const (
	Unknown Type = iota
	Data
	Programming
	Markup
	Prose
)

// GetLanguageType returns the type of the given language.
func GetLanguageType(language string) (langType Type) {
	intType, ok := data.LanguagesType[language]
	langType = Type(intType)
	if !ok {
		langType = Unknown
	}
	return langType
}

// GetLanguageByAlias returns either the language related to the given alias and ok set to true
// or Otherlanguage and ok set to false if the alias is not recognized.
func GetLanguageByAlias(alias string) (lang string, ok bool) {
	a := strings.Split(alias, `,`)[0]
	a = strings.ToLower(a)
	lang, ok = data.LanguagesByAlias[a]
	if !ok {
		lang = OtherLanguage
	}

	return
}
<|endoftext|>


<commit_before>package murmur3

import "hash"
import "fmt"

type Hash128 interface {
	hash.Hash
	Sum128() (uint64, uint64)
}

const (
	c1_32_128 = uint32(0x239b961b)
	c2_32_128 = uint32(0xab0e9789)
	c3_32_128 = uint32(0x38b34ae5)
	c4_32_128 = uint32(0xa1e38b93)

	c1_64_128 = uint64(0x87c37b91114253d5)
	c2_64_128 = uint64(0x4cf5ad432745937f)
)

type sum32_128 struct {
	h1     uint32
	h2     uint32
	h3     uint32
	h4     uint32
	k1     uint32
	k2     uint32
	k3     uint32
	k4     uint32
	length uint32
	offset uint8
}

type sum64_128 struct {
	h1     uint64
	h2     uint64
	k1     uint64
	k2     uint64
	length uint64
	offset uint8
}

func New32_128() Hash128 { return &sum32_128{0, 0, 0, 0, 0, 0, 0, 0, 0, 0} }
func New64_128() Hash128 { return &sum64_128{0, 0, 0, 0, 0, 0} }

func (s *sum32_128) Reset() {
	s.h1, s.h2, s.h3, s.h4 = 0, 0, 0, 0
	s.k1, s.k2, s.k3, s.k4 = 0, 0, 0, 0
	s.length, s.offset = 0, 0
}

func (s *sum64_128) Reset() {
	s.h1, s.h2, s.k1, s.k2, s.length, s.offset = 0, 0, 0, 0, 0, 0
}

func (s *sum32_128) Sum128() (uint64, uint64) {
	var h1, h2, h3, h4 = s.h1, s.h2, s.h3, s.h4
	var k1, k2, k3, k4 uint32 = s.k1, s.k2, s.k3, s.k4

	//tail
	switch {
	case s.offset > 12:
		k4 *= c4_32_128
		k4 = (k4 << 18) | (k4 >> (32 - 18))
		k4 *= c1_32_128
		h4 ^= k4
		fallthrough

	case s.offset > 8:
		k3 *= c3_32_128
		k3 = (k3 << 17) | (k3 >> (32 - 17))
		k3 *= c4_32_128
		h3 ^= k3
		fallthrough

	case s.offset > 4:
		k2 *= c2_32_128
		k2 = (k2 << 16) | (k2 >> (32 - 16))
		k2 *= c3_32_128
		h2 ^= k2
		fallthrough

	case s.offset > 0:
		k1 *= c1_32_128
		k1 = (k1 << 15) | (k1 >> (32 - 15))
		k1 *= c2_32_128
		h1 ^= k1
	}

	//finalization
	h1 ^= s.length
	h2 ^= s.length
	h3 ^= s.length
	h4 ^= s.length

	h1 += h2
	h1 += h3
	h1 += h4

	h2 += h1
	h3 += h1
	h4 += h1

	h1 ^= h1 >> 16
	h1 *= 0x85ebca6b
	h1 ^= h1 >> 13
	h1 *= 0xc2b2ae35
	h1 ^= h1 >> 16

	h2 ^= h2 >> 16
	h2 *= 0x85ebca6b
	h2 ^= h2 >> 13
	h2 *= 0xc2b2ae35
	h2 ^= h2 >> 16

	h3 ^= h3 >> 16
	h3 *= 0x85ebca6b
	h3 ^= h3 >> 13
	h3 *= 0xc2b2ae35
	h3 ^= h3 >> 16

	h4 ^= h4 >> 16
	h4 *= 0x85ebca6b
	h4 ^= h4 >> 13
	h4 *= 0xc2b2ae35
	h4 ^= h4 >> 16

	h1 += h2
	h1 += h3
	h1 += h4

	h2 += h1
	h3 += h1
	h4 += h1

	return uint64((uint64(h1) << 32) | uint64(h2)),
		uint64((uint64(h3) << 32) | uint64(h4))
}

func (s *sum64_128) Sum128() (uint64, uint64) {
	var h1, h2 uint64 = s.h1, s.h2
	var k1, k2 uint64 = s.k1, s.k2

	//tail
	switch {
	case s.offset > 8:
		k2 *= c2_64_128
		k2 = (k2 << 33) | (k2 >> (64 - 33))
		k2 *= c1_64_128
		h2 ^= k2
		fallthrough

	case s.offset > 0:
		k1 *= c1_64_128
		k1 = (k1 << 31) | (k1 >> (64 - 31))
		k1 *= c2_64_128
		h1 ^= k1
	}

	//finalization
	h1 ^= s.length
	h2 ^= s.length

	h1 += h2
	h2 += h1

	h1 ^= h1 >> 33
	h1 *= 0xff51afd7ed558ccd
	h1 ^= h1 >> 33
	h1 *= 0xc4ceb9fe1a85ec53
	h1 ^= h1 >> 33

	h2 ^= h2 >> 33
	h2 *= 0xff51afd7ed558ccd
	h2 ^= h2 >> 33
	h2 *= 0xc4ceb9fe1a85ec53
	h2 ^= h2 >> 33

	h1 += h2
	h2 += h1

	return h1, h2
}

func (s *sum32_128) Sum(in []byte) []byte {
	h1, h2 := s.Sum128()
	return append(in, byte(h1>>56), byte(h1>>48), byte(h1>>40), byte(h1>>32),
		byte(h1>>24), byte(h1>>16), byte(h1>>8), byte(h1), byte(h2>>56),
		byte(h2>>48), byte(h2>>32), byte(h2>>24), byte(h2>>16),
		byte(h2>>8), byte(h2))
}

func (s *sum64_128) Sum(in []byte) []byte {
	h1, h2 := s.Sum128()
	return append(in, byte(h1>>56), byte(h1>>48), byte(h1>>40), byte(h1>>32),
		byte(h1>>24), byte(h1>>16), byte(h1>>8), byte(h1), byte(h2>>56),
		byte(h2>>48), byte(h2>>32), byte(h2>>24), byte(h2>>16),
		byte(h2>>8), byte(h2))
}

func (s *sum32_128) Write(data []byte) (int, error) {
	length := len(data)
	if length == 0 {
		return 0, nil
	}
	s.length += uint32(length)

	for _, c := range data {

		// TODO: Might want to check this for endianness for consistency
		// across systems.
		if s.offset < 4 {
			s.k1 |= uint32(uint32(c) << uint32(s.offset*8))
		} else if s.offset >= 4 && s.offset < 8 {
			s.k2 |= uint32(uint32(c) << uint32((s.offset%4)*8))
		} else if s.offset >= 8 && s.offset < 12 {
			s.k3 |= uint32(uint32(c) << uint32((s.offset%4)*8))
		} else if s.offset >= 12 && s.offset < 16 {
			s.k4 |= uint32(uint32(c) << uint32((s.offset%4)*8))
		}
		s.offset++

		if s.offset == 16 {
			s.k1 *= c1_32_128
			s.k1 = (s.k1 << 15) | (s.k1 >> (32 - 15))
			s.k1 *= c2_32_128
			s.h1 ^= s.k1

			s.h1 = (s.h1 << 19) | (s.h1 >> (32 - 19))
			s.h1 += s.h2
			s.h1 = s.h1*5 + 0x561ccd1b

			s.k2 *= c2_32_128
			s.k2 = (s.k2 << 16) | (s.k2 >> (32 - 16))
			s.k2 *= c3_32_128
			s.h2 ^= s.k2

			s.h2 = (s.h2 << 17) | (s.h2 >> (32 - 17))
			s.h2 += s.h3
			s.h2 = s.h2*5 + 0x0bcaa747

			s.k3 *= c3_32_128
			s.k3 = (s.k3 << 17) | (s.k3 >> (32 - 17))
			s.k3 *= c4_32_128
			s.h3 ^= s.k3

			s.h3 = (s.h3 << 15) | (s.h3 >> (32 - 15))
			s.h3 += s.h4
			s.h3 = s.h3*5 + 0x96cd1c35

			s.k4 *= c4_32_128
			s.k4 = (s.k4 << 18) | (s.k4 >> (32 - 18))
			s.k4 *= c1_32_128
			s.h4 ^= s.k4

			s.h4 = (s.h4 << 13) | (s.h4 >> (32 - 13))
			s.h4 += s.h1
			s.h4 = s.h4*5 + 0x32ac3b17

			s.k1, s.k2, s.k3, s.k4 = 0, 0, 0, 0
			s.offset = 0
		}
	}
	return length, nil
}

func (s *sum64_128) Write(data []byte) (int, error) {
	length := len(data)
	if length == 0 {
		return 0, nil
	}
	s.length += uint64(length)

	for _, c := range data {

		// TODO: Might want to check this for endianness for consistency
		// across systems.
		if s.offset < 8 {
			s.k1 |= uint64(uint64(c) << uint64(s.offset*8))
		} else if s.offset >= 8 && s.offset < 16 {
			s.k2 |= uint64(uint64(c) << uint64((s.offset%8)*8))
		}
		s.offset++

		if s.offset == 16 {
			s.k1 *= c1_64_128
			s.k1 = (s.k1 << 31) | (s.k1 >> (64 - 31))
			s.k1 *= c2_64_128
			s.h1 ^= s.k1

			s.h1 = (s.h1 << 27) | (s.h1 >> (64 - 27))
			s.h1 += s.h2
			s.h1 = s.h1*5 + 0x52dce729

			s.k2 *= c2_64_128
			s.k2 = (s.k2 << 33) | (s.k2 >> (64 - 33))
			s.k2 *= c1_64_128
			s.h2 ^= s.k2

			s.h2 = (s.h2 << 31) | (s.h2 >> (64 - 31))
			s.h2 += s.h1
			s.h2 = s.h2*5 + 0x38495ab5

			s.k1 = 0
			s.k2 = 0

			s.offset = 0
		}
	}
	return length, nil
}

func (s *sum32_128) BlockSize() int { return 16 }
func (s *sum64_128) BlockSize() int { return 16 }

func (s *sum32_128) Size() int { return 16 }
func (s *sum64_128) Size() int { return 16 }
<commit_msg>Fix the import.<commit_after>package murmur3

import "hash"

type Hash128 interface {
	hash.Hash
	Sum128() (uint64, uint64)
}

const (
	c1_32_128 = uint32(0x239b961b)
	c2_32_128 = uint32(0xab0e9789)
	c3_32_128 = uint32(0x38b34ae5)
	c4_32_128 = uint32(0xa1e38b93)

	c1_64_128 = uint64(0x87c37b91114253d5)
	c2_64_128 = uint64(0x4cf5ad432745937f)
)

type sum32_128 struct {
	h1     uint32
	h2     uint32
	h3     uint32
	h4     uint32
	k1     uint32
	k2     uint32
	k3     uint32
	k4     uint32
	length uint32
	offset uint8
}

type sum64_128 struct {
	h1     uint64
	h2     uint64
	k1     uint64
	k2     uint64
	length uint64
	offset uint8
}

func New32_128() Hash128 { return &sum32_128{0, 0, 0, 0, 0, 0, 0, 0, 0, 0} }
func New64_128() Hash128 { return &sum64_128{0, 0, 0, 0, 0, 0} }

func (s *sum32_128) Reset() {
	s.h1, s.h2, s.h3, s.h4 = 0, 0, 0, 0
	s.k1, s.k2, s.k3, s.k4 = 0, 0, 0, 0
	s.length, s.offset = 0, 0
}

func (s *sum64_128) Reset() {
	s.h1, s.h2, s.k1, s.k2, s.length, s.offset = 0, 0, 0, 0, 0, 0
}

func (s *sum32_128) Sum128() (uint64, uint64) {
	var h1, h2, h3, h4 = s.h1, s.h2, s.h3, s.h4
	var k1, k2, k3, k4 uint32 = s.k1, s.k2, s.k3, s.k4

	//tail
	switch {
	case s.offset > 12:
		k4 *= c4_32_128
		k4 = (k4 << 18) | (k4 >> (32 - 18))
		k4 *= c1_32_128
		h4 ^= k4
		fallthrough

	case s.offset > 8:
		k3 *= c3_32_128
		k3 = (k3 << 17) | (k3 >> (32 - 17))
		k3 *= c4_32_128
		h3 ^= k3
		fallthrough

	case s.offset > 4:
		k2 *= c2_32_128
		k2 = (k2 << 16) | (k2 >> (32 - 16))
		k2 *= c3_32_128
		h2 ^= k2
		fallthrough

	case s.offset > 0:
		k1 *= c1_32_128
		k1 = (k1 << 15) | (k1 >> (32 - 15))
		k1 *= c2_32_128
		h1 ^= k1
	}

	//finalization
	h1 ^= s.length
	h2 ^= s.length
	h3 ^= s.length
	h4 ^= s.length

	h1 += h2
	h1 += h3
	h1 += h4

	h2 += h1
	h3 += h1
	h4 += h1

	h1 ^= h1 >> 16
	h1 *= 0x85ebca6b
	h1 ^= h1 >> 13
	h1 *= 0xc2b2ae35
	h1 ^= h1 >> 16

	h2 ^= h2 >> 16
	h2 *= 0x85ebca6b
	h2 ^= h2 >> 13
	h2 *= 0xc2b2ae35
	h2 ^= h2 >> 16

	h3 ^= h3 >> 16
	h3 *= 0x85ebca6b
	h3 ^= h3 >> 13
	h3 *= 0xc2b2ae35
	h3 ^= h3 >> 16

	h4 ^= h4 >> 16
	h4 *= 0x85ebca6b
	h4 ^= h4 >> 13
	h4 *= 0xc2b2ae35
	h4 ^= h4 >> 16

	h1 += h2
	h1 += h3
	h1 += h4

	h2 += h1
	h3 += h1
	h4 += h1

	return uint64((uint64(h1) << 32) | uint64(h2)),
		uint64((uint64(h3) << 32) | uint64(h4))
}

func (s *sum64_128) Sum128() (uint64, uint64) {
	var h1, h2 uint64 = s.h1, s.h2
	var k1, k2 uint64 = s.k1, s.k2

	//tail
	switch {
	case s.offset > 8:
		k2 *= c2_64_128
		k2 = (k2 << 33) | (k2 >> (64 - 33))
		k2 *= c1_64_128
		h2 ^= k2
		fallthrough

	case s.offset > 0:
		k1 *= c1_64_128
		k1 = (k1 << 31) | (k1 >> (64 - 31))
		k1 *= c2_64_128
		h1 ^= k1
	}

	//finalization
	h1 ^= s.length
	h2 ^= s.length

	h1 += h2
	h2 += h1

	h1 ^= h1 >> 33
	h1 *= 0xff51afd7ed558ccd
	h1 ^= h1 >> 33
	h1 *= 0xc4ceb9fe1a85ec53
	h1 ^= h1 >> 33

	h2 ^= h2 >> 33
	h2 *= 0xff51afd7ed558ccd
	h2 ^= h2 >> 33
	h2 *= 0xc4ceb9fe1a85ec53
	h2 ^= h2 >> 33

	h1 += h2
	h2 += h1

	return h1, h2
}

func (s *sum32_128) Sum(in []byte) []byte {
	h1, h2 := s.Sum128()
	return append(in, byte(h1>>56), byte(h1>>48), byte(h1>>40), byte(h1>>32),
		byte(h1>>24), byte(h1>>16), byte(h1>>8), byte(h1), byte(h2>>56),
		byte(h2>>48), byte(h2>>32), byte(h2>>24), byte(h2>>16),
		byte(h2>>8), byte(h2))
}

func (s *sum64_128) Sum(in []byte) []byte {
	h1, h2 := s.Sum128()
	return append(in, byte(h1>>56), byte(h1>>48), byte(h1>>40), byte(h1>>32),
		byte(h1>>24), byte(h1>>16), byte(h1>>8), byte(h1), byte(h2>>56),
		byte(h2>>48), byte(h2>>32), byte(h2>>24), byte(h2>>16),
		byte(h2>>8), byte(h2))
}

func (s *sum32_128) Write(data []byte) (int, error) {
	length := len(data)
	if length == 0 {
		return 0, nil
	}
	s.length += uint32(length)

	for _, c := range data {

		// TODO: Might want to check this for endianness for consistency
		// across systems.
		if s.offset < 4 {
			s.k1 |= uint32(uint32(c) << uint32(s.offset*8))
		} else if s.offset >= 4 && s.offset < 8 {
			s.k2 |= uint32(uint32(c) << uint32((s.offset%4)*8))
		} else if s.offset >= 8 && s.offset < 12 {
			s.k3 |= uint32(uint32(c) << uint32((s.offset%4)*8))
		} else if s.offset >= 12 && s.offset < 16 {
			s.k4 |= uint32(uint32(c) << uint32((s.offset%4)*8))
		}
		s.offset++

		if s.offset == 16 {
			s.k1 *= c1_32_128
			s.k1 = (s.k1 << 15) | (s.k1 >> (32 - 15))
			s.k1 *= c2_32_128
			s.h1 ^= s.k1

			s.h1 = (s.h1 << 19) | (s.h1 >> (32 - 19))
			s.h1 += s.h2
			s.h1 = s.h1*5 + 0x561ccd1b

			s.k2 *= c2_32_128
			s.k2 = (s.k2 << 16) | (s.k2 >> (32 - 16))
			s.k2 *= c3_32_128
			s.h2 ^= s.k2

			s.h2 = (s.h2 << 17) | (s.h2 >> (32 - 17))
			s.h2 += s.h3
			s.h2 = s.h2*5 + 0x0bcaa747

			s.k3 *= c3_32_128
			s.k3 = (s.k3 << 17) | (s.k3 >> (32 - 17))
			s.k3 *= c4_32_128
			s.h3 ^= s.k3

			s.h3 = (s.h3 << 15) | (s.h3 >> (32 - 15))
			s.h3 += s.h4
			s.h3 = s.h3*5 + 0x96cd1c35

			s.k4 *= c4_32_128
			s.k4 = (s.k4 << 18) | (s.k4 >> (32 - 18))
			s.k4 *= c1_32_128
			s.h4 ^= s.k4

			s.h4 = (s.h4 << 13) | (s.h4 >> (32 - 13))
			s.h4 += s.h1
			s.h4 = s.h4*5 + 0x32ac3b17

			s.k1, s.k2, s.k3, s.k4 = 0, 0, 0, 0
			s.offset = 0
		}
	}
	return length, nil
}

func (s *sum64_128) Write(data []byte) (int, error) {
	length := len(data)
	if length == 0 {
		return 0, nil
	}
	s.length += uint64(length)

	for _, c := range data {

		// TODO: Might want to check this for endianness for consistency
		// across systems.
		if s.offset < 8 {
			s.k1 |= uint64(uint64(c) << uint64(s.offset*8))
		} else if s.offset >= 8 && s.offset < 16 {
			s.k2 |= uint64(uint64(c) << uint64((s.offset%8)*8))
		}
		s.offset++

		if s.offset == 16 {
			s.k1 *= c1_64_128
			s.k1 = (s.k1 << 31) | (s.k1 >> (64 - 31))
			s.k1 *= c2_64_128
			s.h1 ^= s.k1

			s.h1 = (s.h1 << 27) | (s.h1 >> (64 - 27))
			s.h1 += s.h2
			s.h1 = s.h1*5 + 0x52dce729

			s.k2 *= c2_64_128
			s.k2 = (s.k2 << 33) | (s.k2 >> (64 - 33))
			s.k2 *= c1_64_128
			s.h2 ^= s.k2

			s.h2 = (s.h2 << 31) | (s.h2 >> (64 - 31))
			s.h2 += s.h1
			s.h2 = s.h2*5 + 0x38495ab5

			s.k1 = 0
			s.k2 = 0

			s.offset = 0
		}
	}
	return length, nil
}

func (s *sum32_128) BlockSize() int { return 16 }
func (s *sum64_128) BlockSize() int { return 16 }

func (s *sum32_128) Size() int { return 16 }
func (s *sum64_128) Size() int { return 16 }
<|endoftext|>


<commit_before>package lib

import (
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync"

	"github.com/ahmetb/go-linq"
)

// PathList is a list of paths
type PathList struct {
	SourcePath string
	TargetPath string
}

// HashList is a list of HashData struct
type HashList struct {
	List         []HashData `json:"hash_list"`
	VerifyResult bool       `json:"verify_result,omitempty"`
}

// GetDirectoryInfo returns the number of directories and files.
// If HashList.List is nil or empty, returns 0 and 0.
func (h *HashList) GetDirectoryInfo() (int, int) {
	directories := linq.From(h.List).CountWith(func(arg1 interface{}) bool {
		return arg1.(HashData).HashValue == "-"
	})
	files := len(h.List) - directories
	return directories, files
}

// HashData is hash value and file name struct
type HashData struct {
	RelativePath string `json:"relative_path"`
	HashValue    string `json:"hash_value"`
	VerifyResult bool   `json:"verify_result,omitempty"`
	Reason       string `json:"reason_of_failed,omitempty"`
}

// GetHashList returns the hash list of source.
// For source, a json file or a directory is supported.
func GetHashList(source string) (HashList, error) {
	source = filepath.Clean(source)
	list := HashList{}
	info, err := os.Stat(source)
	if err != nil {
		return list, err
	}
	if info.IsDir() {
		return generateHashList(source)
	}
	return readHashList(source)
}

// GenerateHashList generates hash information of dir path.
func generateHashList(dir string) (HashList, error) {
	type result struct {
		Data  HashData
		Error error
	}

	dir = filepath.Clean(dir)
	list := HashList{}
	c := make(chan result)
	var wg sync.WaitGroup
	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		// root path is not used.
		if strings.Compare(dir, path) == 0 {
			return nil
		}

		wg.Add(1)
		go func() {
			defer wg.Done()
			data := HashData{}
			rel, err := filepath.Rel(dir, path)
			if err != nil {
				log.Println("Get relative path error.", err)
				c <- result{Data: data, Error: err}
				return
			}
			data.RelativePath = rel

			if info.IsDir() {
				// directories do not have hash value
				data.HashValue = "-"
			} else {
				bytes, err := ioutil.ReadFile(path)
				if err != nil {
					log.Println("ReadFile error.", err)
					c <- result{Data: data, Error: err}
					return
				}
				hash := sha256.Sum256(bytes)
				data.HashValue = hex.EncodeToString(hash[:])
			}
			c <- result{Data: data, Error: nil}
		}()
		return nil
	})
	if err != nil {
		return list, err
	}

	go func() {
		wg.Wait()
		close(c)
	}()

	for r := range c {
		if r.Error != nil {
			log.Println(err)
		}
		list.List = append(list.List, r.Data)
	}
	sort.SliceStable(list.List, func(i int, j int) bool {
		return list.List[i].RelativePath < list.List[j].RelativePath
	})

	return list, nil
}

// ReadHashList reads source json file and return a hash list.
func readHashList(source string) (HashList, error) {
	list := HashList{}
	source = filepath.Clean(source)
	data, err := ioutil.ReadFile(source)
	if err != nil {
		return list, err
	}

	err = json.Unmarshal(data, &list)
	if err != nil {
		return list, err
	}
	return list, nil
}

// VerifyHashList verifies hash list of source and target.
// Then, returns HashList that has VerifyResult.
func VerifyHashList(source, target HashList, doHashCheck, doUnnecessaryCheck bool) HashList {
	result := verifyWithSource(source, target, doHashCheck)
	result = verifyWithTarget(result, target, doUnnecessaryCheck)
	sort.SliceStable(result.List, func(i int, j int) bool {
		return result.List[i].RelativePath < result.List[j].RelativePath
	})
	result.VerifyResult = linq.From(result.List).All(func(arg1 interface{}) bool {
		return arg1.(HashData).VerifyResult == true
	})
	return result
}

func verifyWithSource(source, target HashList, doHashCheck bool) HashList {
	result := HashList{VerifyResult: false}

	for _, item := range source.List {
		selected := linq.From(target.List).
			SingleWith(func(c interface{}) bool {
				return c.(HashData).RelativePath == item.RelativePath
			})

		if selected == nil {
			var message string
			if item.HashValue == "-" {
				message = "This directory does not exist"
			} else {
				message = "This file does not exist"
			}

			fail := HashData{
				RelativePath: item.RelativePath,
				HashValue:    item.HashValue,
				VerifyResult: false,
				Reason:       message,
			}
			result.List = append(result.List, fail)
			log.Printf(`Required item does not exist. "%s"`, item.RelativePath)
			continue
		}

		data := selected.(HashData)
		if data.HashValue == item.HashValue {
			data.VerifyResult = true
		} else {
			if doHashCheck {
				data.VerifyResult = false
				data.Reason = "Hash value does not match"
				log.Printf(`Hash value does not match. "%s"`, item.RelativePath)
			} else {
				data.VerifyResult = true
				data.Reason = "Ignore that the hash values do not match"
			}
		}
		result.List = append(result.List, data)
	}

	return result
}

// verifyWithTarget verifies the result of verification with the hash list
// of the target directory to check wheather the unnecessary items exist.
func verifyWithTarget(result, target HashList, doUnnecessaryCheck bool) HashList {
	for _, item := range target.List {
		more := linq.From(result.List).
			SingleWith(func(c interface{}) bool {
				return c.(HashData).RelativePath == item.RelativePath
			})

		if more == nil && doUnnecessaryCheck {
			item.VerifyResult = false
			item.Reason = "Unnecessary item exists."
			log.Printf(`Unnecessary item exists. "%s"`, item.RelativePath)
			result.List = append(result.List, item)
		} else if more == nil && !doUnnecessaryCheck {
			item.VerifyResult = true
			item.Reason = "Ignore that an unnecessary item exists."
			result.List = append(result.List, item)
		}
	}
	return result
}
<commit_msg>fix: fix the generating hash values function to reduce the memory usage (#18)<commit_after>package lib

import (
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync"

	"github.com/ahmetb/go-linq"
)

// PathList is a list of paths
type PathList struct {
	SourcePath string
	TargetPath string
}

// HashList is a list of HashData struct
type HashList struct {
	List         []HashData `json:"hash_list"`
	VerifyResult bool       `json:"verify_result,omitempty"`
}

// GetDirectoryInfo returns the number of directories and files.
// If HashList.List is nil or empty, returns 0 and 0.
func (h *HashList) GetDirectoryInfo() (int, int) {
	directories := linq.From(h.List).CountWith(func(arg1 interface{}) bool {
		return arg1.(HashData).HashValue == "-"
	})
	files := len(h.List) - directories
	return directories, files
}

// HashData is hash value and file name struct
type HashData struct {
	RelativePath string `json:"relative_path"`
	HashValue    string `json:"hash_value"`
	VerifyResult bool   `json:"verify_result,omitempty"`
	Reason       string `json:"reason_of_failed,omitempty"`
}

// GetHashList returns the hash list of source.
// For source, a json file or a directory is supported.
func GetHashList(source string) (HashList, error) {
	source = filepath.Clean(source)
	list := HashList{}
	info, err := os.Stat(source)
	if err != nil {
		return list, err
	}
	if info.IsDir() {
		return generateHashList(source)
	}
	return readHashList(source)
}

// GenerateHashList generates hash information of dir path.
func generateHashList(dir string) (HashList, error) {
	type result struct {
		Data  HashData
		Error error
	}

	dir = filepath.Clean(dir)
	list := HashList{}
	c := make(chan result)
	var wg sync.WaitGroup
	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		// root path is not used.
		if strings.Compare(dir, path) == 0 {
			return nil
		}

		wg.Add(1)
		go func() {
			defer wg.Done()
			data := HashData{}
			rel, err := filepath.Rel(dir, path)
			if err != nil {
				log.Println("Get relative path error.", err)
				c <- result{Data: data, Error: err}
				return
			}
			data.RelativePath = rel

			if info.IsDir() {
				// directories do not have hash value
				data.HashValue = "-"
			} else {
				data.HashValue, err = generateHash(path)
				if err != nil {
					log.Printf("failed to generate the hash value for '%v', %v", path, err)
					c <- result{Data: data, Error: err}
					return
				}
			}
			c <- result{Data: data, Error: nil}
		}()
		return nil
	})
	if err != nil {
		return list, err
	}

	go func() {
		wg.Wait()
		close(c)
	}()

	for r := range c {
		if r.Error != nil {
			log.Println(err)
		}
		list.List = append(list.List, r.Data)
	}
	sort.SliceStable(list.List, func(i int, j int) bool {
		return list.List[i].RelativePath < list.List[j].RelativePath
	})

	return list, nil
}

func generateHash(path string) (string, error) {
	f, err := os.Open(path)
	if err != nil {
		return "", err
	}
	defer f.Close()

	h := sha256.New()
	_, err = io.Copy(h, f)
	if err != nil {
		return "", err
	}
	return hex.EncodeToString(h.Sum(nil)), nil
}

// ReadHashList reads source json file and return a hash list.
func readHashList(source string) (HashList, error) {
	list := HashList{}
	source = filepath.Clean(source)
	data, err := ioutil.ReadFile(source)
	if err != nil {
		return list, err
	}

	err = json.Unmarshal(data, &list)
	if err != nil {
		return list, err
	}
	return list, nil
}

// VerifyHashList verifies hash list of source and target.
// Then, returns HashList that has VerifyResult.
func VerifyHashList(source, target HashList, doHashCheck, doUnnecessaryCheck bool) HashList {
	result := verifyWithSource(source, target, doHashCheck)
	result = verifyWithTarget(result, target, doUnnecessaryCheck)
	sort.SliceStable(result.List, func(i int, j int) bool {
		return result.List[i].RelativePath < result.List[j].RelativePath
	})
	result.VerifyResult = linq.From(result.List).All(func(arg1 interface{}) bool {
		return arg1.(HashData).VerifyResult == true
	})
	return result
}

func verifyWithSource(source, target HashList, doHashCheck bool) HashList {
	result := HashList{VerifyResult: false}

	for _, item := range source.List {
		selected := linq.From(target.List).
			SingleWith(func(c interface{}) bool {
				return c.(HashData).RelativePath == item.RelativePath
			})

		if selected == nil {
			var message string
			if item.HashValue == "-" {
				message = "This directory does not exist"
			} else {
				message = "This file does not exist"
			}

			fail := HashData{
				RelativePath: item.RelativePath,
				HashValue:    item.HashValue,
				VerifyResult: false,
				Reason:       message,
			}
			result.List = append(result.List, fail)
			log.Printf(`Required item does not exist. "%s"`, item.RelativePath)
			continue
		}

		data := selected.(HashData)
		if data.HashValue == item.HashValue {
			data.VerifyResult = true
		} else {
			if doHashCheck {
				data.VerifyResult = false
				data.Reason = "Hash value does not match"
				log.Printf(`Hash value does not match. "%s"`, item.RelativePath)
			} else {
				data.VerifyResult = true
				data.Reason = "Ignore that the hash values do not match"
			}
		}
		result.List = append(result.List, data)
	}

	return result
}

// verifyWithTarget verifies the result of verification with the hash list
// of the target directory to check wheather the unnecessary items exist.
func verifyWithTarget(result, target HashList, doUnnecessaryCheck bool) HashList {
	for _, item := range target.List {
		more := linq.From(result.List).
			SingleWith(func(c interface{}) bool {
				return c.(HashData).RelativePath == item.RelativePath
			})

		if more == nil && doUnnecessaryCheck {
			item.VerifyResult = false
			item.Reason = "Unnecessary item exists."
			log.Printf(`Unnecessary item exists. "%s"`, item.RelativePath)
			result.List = append(result.List, item)
		} else if more == nil && !doUnnecessaryCheck {
			item.VerifyResult = true
			item.Reason = "Ignore that an unnecessary item exists."
			result.List = append(result.List, item)
		}
	}
	return result
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/prometheus/client_golang/prometheus"
)

type (
	executor struct {
		ID          string      `json:"executor_id"`
		Name        string      `json:"executor_name"`
		FrameworkID string      `json:"framework_id"`
		Source      string      `json:"source"`
		Statistics  *statistics `json:"statistics"`
		Tasks       []task      `json:"tasks"`
	}

	statistics struct {
		Processes             float64 `json:"processes"`
		Threads               float64 `json:"threads"`

		CpusLimit             float64 `json:"cpus_limit"`
		CpusSystemTimeSecs    float64 `json:"cpus_system_time_secs"`
		CpusUserTimeSecs      float64 `json:"cpus_user_time_secs"`
		CpusThrottledTimeSecs float64 `json:"cpus_throttled_time_secs"`

		MemLimitBytes float64 `json:"mem_limit_bytes"`
		MemRssBytes   float64 `json:"mem_rss_bytes"`

		DiskLimitBytes float64 `json:"disk_limit_bytes"`
		DiskUsedBytes  float64 `json:"disk_used_bytes"`

		NetRxBytes   float64 `json:"net_rx_bytes"`
		NetRxDropped float64 `json:"net_rx_dropped"`
		NetRxErrors  float64 `json:"net_rx_errors"`
		NetRxPackets float64 `json:"net_rx_packets"`
		NetTxBytes   float64 `json:"net_tx_bytes"`
		NetTxDropped float64 `json:"net_tx_dropped"`
		NetTxErrors  float64 `json:"net_tx_errors"`
		NetTxPackets float64 `json:"net_tx_packets"`
	}

	slaveCollector struct {
		*httpClient
		metrics map[*prometheus.Desc]metric
	}

	metric struct {
		valueType prometheus.ValueType
		get       func(*statistics) float64
	}
)

func newSlaveMonitorCollector(httpClient *httpClient) prometheus.Collector {
	labels := []string{"id", "framework_id", "source"}

	return &slaveCollector{
		httpClient: httpClient,
		metrics: map[*prometheus.Desc]metric{
			// Processes
			prometheus.NewDesc(
				"processes",
				"Current number of processes",
				labels, nil,
			): metric{prometheus.GaugeValue, func(s *statistics) float64 { return s.Processes }},
			prometheus.NewDesc(
				"threads",
				"Current number of threads",
				labels, nil,
			): metric{prometheus.GaugeValue, func(s *statistics) float64 { return s.Threads }},

			// CPU
			prometheus.NewDesc(
				"cpus_limit",
				"Current limit of CPUs for task",
				labels, nil,
			): metric{prometheus.GaugeValue, func(s *statistics) float64 { return s.CpusLimit }},
			prometheus.NewDesc(
				"cpu_system_seconds_total",
				"Total system CPU seconds",
				labels, nil,
			): metric{prometheus.CounterValue, func(s *statistics) float64 { return s.CpusSystemTimeSecs }},
			prometheus.NewDesc(
				"cpu_user_seconds_total",
				"Total user CPU seconds",
				labels, nil,
			): metric{prometheus.CounterValue, func(s *statistics) float64 { return s.CpusUserTimeSecs }},
			prometheus.NewDesc(
				"cpu_throttled_seconds_total",
				"Total time CPU was throttled",
				labels, nil,
			): metric{prometheus.CounterValue, func(s *statistics) float64 { return s.CpusThrottledTimeSecs }},

			// Memory
			prometheus.NewDesc(
				"mem_limit_bytes",
				"Current memory limit in bytes",
				labels, nil,
			): metric{prometheus.GaugeValue, func(s *statistics) float64 { return s.MemLimitBytes }},
			prometheus.NewDesc(
				"mem_rss_bytes",
				"Current rss memory usage",
				labels, nil,
			): metric{prometheus.GaugeValue, func(s *statistics) float64 { return s.MemRssBytes }},

			// Disk
			prometheus.NewDesc(
				"disk_limit_bytes",
				"Current disk limit in bytes",
				labels, nil,
			): metric{prometheus.GaugeValue, func(s *statistics) float64 { return s.DiskLimitBytes }},
			prometheus.NewDesc(
				"disk_used_bytes",
				"Current disk usage",
				labels, nil,
			): metric{prometheus.GaugeValue, func(s *statistics) float64 { return s.DiskUsedBytes }},

			// Network
			// - RX
			prometheus.NewDesc(
				"network_receive_bytes_total",
				"Total bytes received",
				labels, nil,
			): metric{prometheus.CounterValue, func(s *statistics) float64 { return s.NetRxBytes }},
			prometheus.NewDesc(
				"network_receive_dropped_total",
				"Total packets dropped while receiving",
				labels, nil,
			): metric{prometheus.CounterValue, func(s *statistics) float64 { return s.NetRxDropped }},
			prometheus.NewDesc(
				"network_receive_errors_total",
				"Total errors while receiving",
				labels, nil,
			): metric{prometheus.CounterValue, func(s *statistics) float64 { return s.NetRxErrors }},
			prometheus.NewDesc(
				"network_receive_packets_total",
				"Total packets received",
				labels, nil,
			): metric{prometheus.CounterValue, func(s *statistics) float64 { return s.NetRxPackets }},
			// - TX
			prometheus.NewDesc(
				"network_transmit_bytes_total",
				"Total bytes transmitted",
				labels, nil,
			): metric{prometheus.CounterValue, func(s *statistics) float64 { return s.NetTxBytes }},
			prometheus.NewDesc(
				"network_transmit_dropped_total",
				"Total packets dropped while transmitting",
				labels, nil,
			): metric{prometheus.CounterValue, func(s *statistics) float64 { return s.NetTxDropped }},
			prometheus.NewDesc(
				"network_transmit_errors_total",
				"Total errors while transmitting",
				labels, nil,
			): metric{prometheus.CounterValue, func(s *statistics) float64 { return s.NetTxErrors }},
			prometheus.NewDesc(
				"network_transmit_packets_total",
				"Total packets transmitted",
				labels, nil,
			): metric{prometheus.CounterValue, func(s *statistics) float64 { return s.NetTxPackets }},
		},
	}
}

func (c *slaveCollector) Collect(ch chan<- prometheus.Metric) {
	stats := []executor{}
	c.fetchAndDecode("/monitor/statistics", &stats)

	for _, exec := range stats {
		for desc, m := range c.metrics {
			ch <- prometheus.MustNewConstMetric(desc, m.valueType, m.get(exec.Statistics), exec.ID, exec.FrameworkID, exec.Source)
		}
	}
}

func (c *slaveCollector) Describe(ch chan<- *prometheus.Desc) {
	for metric := range c.metrics {
		ch <- metric
	}
}
<commit_msg>Extend container cpu and memory metrics<commit_after>package main

import (
	"github.com/prometheus/client_golang/prometheus"
)

type (
	executor struct {
		ID          string      `json:"executor_id"`
		Name        string      `json:"executor_name"`
		FrameworkID string      `json:"framework_id"`
		Source      string      `json:"source"`
		Statistics  *statistics `json:"statistics"`
		Tasks       []task      `json:"tasks"`
	}

	statistics struct {
		Processes             float64 `json:"processes"`
		Threads               float64 `json:"threads"`

		CpusLimit             float64 `json:"cpus_limit"`
		CpusSystemTimeSecs    float64 `json:"cpus_system_time_secs"`
		CpusUserTimeSecs      float64 `json:"cpus_user_time_secs"`
		CpusThrottledTimeSecs float64 `json:"cpus_throttled_time_secs"`
		CpusNrPeriods         float64 `json:"cpus_nr_periods"`
		CpusNrThrottled       float64 `json:"cpus_nr_throttled"`

		MemLimitBytes float64 `json:"mem_limit_bytes"`
		MemRssBytes   float64 `json:"mem_rss_bytes"`
		MemTotalBytes float64 `json:"mem_total_bytes"`
		MemCacheBytes float64 `json:"mem_cache_bytes"`

		DiskLimitBytes float64 `json:"disk_limit_bytes"`
		DiskUsedBytes  float64 `json:"disk_used_bytes"`

		NetRxBytes   float64 `json:"net_rx_bytes"`
		NetRxDropped float64 `json:"net_rx_dropped"`
		NetRxErrors  float64 `json:"net_rx_errors"`
		NetRxPackets float64 `json:"net_rx_packets"`
		NetTxBytes   float64 `json:"net_tx_bytes"`
		NetTxDropped float64 `json:"net_tx_dropped"`
		NetTxErrors  float64 `json:"net_tx_errors"`
		NetTxPackets float64 `json:"net_tx_packets"`
	}

	slaveCollector struct {
		*httpClient
		metrics map[*prometheus.Desc]metric
	}

	metric struct {
		valueType prometheus.ValueType
		get       func(*statistics) float64
	}
)

func newSlaveMonitorCollector(httpClient *httpClient) prometheus.Collector {
	labels := []string{"id", "framework_id", "source"}

	return &slaveCollector{
		httpClient: httpClient,
		metrics: map[*prometheus.Desc]metric{
			// Processes
			prometheus.NewDesc(
				"processes",
				"Current number of processes",
				labels, nil,
			): metric{prometheus.GaugeValue, func(s *statistics) float64 { return s.Processes }},
			prometheus.NewDesc(
				"threads",
				"Current number of threads",
				labels, nil,
			): metric{prometheus.GaugeValue, func(s *statistics) float64 { return s.Threads }},

			// CPU
			prometheus.NewDesc(
				"cpus_limit",
				"Current limit of CPUs for task",
				labels, nil,
			): metric{prometheus.GaugeValue, func(s *statistics) float64 { return s.CpusLimit }},
			prometheus.NewDesc(
				"cpu_system_seconds_total",
				"Total system CPU seconds",
				labels, nil,
			): metric{prometheus.CounterValue, func(s *statistics) float64 { return s.CpusSystemTimeSecs }},
			prometheus.NewDesc(
				"cpu_user_seconds_total",
				"Total user CPU seconds",
				labels, nil,
			): metric{prometheus.CounterValue, func(s *statistics) float64 { return s.CpusUserTimeSecs }},
			prometheus.NewDesc(
				"cpu_throttled_seconds_total",
				"Total time CPU was throttled due to CFS bandwidth control",
				labels, nil,
			): metric{prometheus.CounterValue, func(s *statistics) float64 { return s.CpusThrottledTimeSecs }},
			prometheus.NewDesc(
				"cpu_nr_periods_total",
				"Total number of elapsed CFS enforcement intervals",
				labels, nil,
			): metric{prometheus.CounterValue, func(s *statistics) float64 { return s.CpusNrPeriods }},
			prometheus.NewDesc(
				"cpu_nr_throttled_total",
				"Total number of throttled CFS enforcement intervals.",
				labels, nil,
			): metric{prometheus.CounterValue, func(s *statistics) float64 { return s.CpusNrThrottled }},

			// Memory
			prometheus.NewDesc(
				"mem_limit_bytes",
				"Current memory limit in bytes",
				labels, nil,
			): metric{prometheus.GaugeValue, func(s *statistics) float64 { return s.MemLimitBytes }},
			prometheus.NewDesc(
				"mem_rss_bytes",
				"Current rss memory usage",
				labels, nil,
			): metric{prometheus.GaugeValue, func(s *statistics) float64 { return s.MemRssBytes }},
			prometheus.NewDesc(
				"mem_total_bytes",
				"Current total memory usage",
				labels, nil,
			): metric{prometheus.GaugeValue, func(s *statistics) float64 { return s.MemTotalBytes }},
			prometheus.NewDesc(
				"mem_cache_bytes",
				"Current page cache memory usage",
				labels, nil,
			): metric{prometheus.GaugeValue, func(s *statistics) float64 { return s.MemCacheBytes }},

			// Disk
			prometheus.NewDesc(
				"disk_limit_bytes",
				"Current disk limit in bytes",
				labels, nil,
			): metric{prometheus.GaugeValue, func(s *statistics) float64 { return s.DiskLimitBytes }},
			prometheus.NewDesc(
				"disk_used_bytes",
				"Current disk usage",
				labels, nil,
			): metric{prometheus.GaugeValue, func(s *statistics) float64 { return s.DiskUsedBytes }},

			// Network
			// - RX
			prometheus.NewDesc(
				"network_receive_bytes_total",
				"Total bytes received",
				labels, nil,
			): metric{prometheus.CounterValue, func(s *statistics) float64 { return s.NetRxBytes }},
			prometheus.NewDesc(
				"network_receive_dropped_total",
				"Total packets dropped while receiving",
				labels, nil,
			): metric{prometheus.CounterValue, func(s *statistics) float64 { return s.NetRxDropped }},
			prometheus.NewDesc(
				"network_receive_errors_total",
				"Total errors while receiving",
				labels, nil,
			): metric{prometheus.CounterValue, func(s *statistics) float64 { return s.NetRxErrors }},
			prometheus.NewDesc(
				"network_receive_packets_total",
				"Total packets received",
				labels, nil,
			): metric{prometheus.CounterValue, func(s *statistics) float64 { return s.NetRxPackets }},
			// - TX
			prometheus.NewDesc(
				"network_transmit_bytes_total",
				"Total bytes transmitted",
				labels, nil,
			): metric{prometheus.CounterValue, func(s *statistics) float64 { return s.NetTxBytes }},
			prometheus.NewDesc(
				"network_transmit_dropped_total",
				"Total packets dropped while transmitting",
				labels, nil,
			): metric{prometheus.CounterValue, func(s *statistics) float64 { return s.NetTxDropped }},
			prometheus.NewDesc(
				"network_transmit_errors_total",
				"Total errors while transmitting",
				labels, nil,
			): metric{prometheus.CounterValue, func(s *statistics) float64 { return s.NetTxErrors }},
			prometheus.NewDesc(
				"network_transmit_packets_total",
				"Total packets transmitted",
				labels, nil,
			): metric{prometheus.CounterValue, func(s *statistics) float64 { return s.NetTxPackets }},
		},
	}
}

func (c *slaveCollector) Collect(ch chan<- prometheus.Metric) {
	stats := []executor{}
	c.fetchAndDecode("/monitor/statistics", &stats)

	for _, exec := range stats {
		for desc, m := range c.metrics {
			ch <- prometheus.MustNewConstMetric(desc, m.valueType, m.get(exec.Statistics), exec.ID, exec.FrameworkID, exec.Source)
		}
	}
}

func (c *slaveCollector) Describe(ch chan<- *prometheus.Desc) {
	for metric := range c.metrics {
		ch <- metric
	}
}
<|endoftext|>


<commit_before>package ultimate

import (
	"context"
	"database/sql"
	"fmt"
	"sync"
	"time"

	"github.com/hellodudu/Ultimate/global"
	"github.com/hellodudu/Ultimate/logger"
)

type DBMgr struct {
	db     *sql.DB
	ctx    context.Context
	cancel context.CancelFunc
	wg     sync.WaitGroup
	chStop chan struct{}
}

func NewDBMgr() (*DBMgr, error) {
	dbMgr := &DBMgr{
		chStop: make(chan struct{}, 1),
	}

	dbMgr.ctx, dbMgr.cancel = context.WithCancel(context.Background())

	mysqlDSN := fmt.Sprintf("%s:%s@(%s:%s)/%s", global.MysqlUser, global.MysqlPwd, global.MysqlAddr, global.MysqlPort, global.MysqlDB)
	var err error
	dbMgr.db, err = sql.Open("mysql", mysqlDSN)
	if err != nil {
		logger.Fatal(err)
		return nil, err
	}

	dbMgr.initDBMgr()
	return dbMgr, nil
}

func (m *DBMgr) Run() {
	for {
		select {
		case <-m.ctx.Done():
			logger.Print("db mgr context done!")
			m.chStop <- struct{}{}
			return
		}
	}

}

func (m *DBMgr) Stop() chan struct{} {
	m.db.Close()
	m.cancel()
	return m.chStop
}

func (m *DBMgr) initDBMgr() {

	m.wg.Add(1)
	go m.loadGlobal()

	m.wg.Wait()

}

func (m *DBMgr) loadGlobal() {
	defer m.wg.Done()

	query := "select * from global"
	stmt, err := m.db.PrepareContext(m.ctx, query)
	if err != nil {
		logger.Warning("api initdb failed:", err)
		return
	}

	rows, err := stmt.QueryContext(m.ctx)
	if err != nil {
		logger.Warning("api initdb failed:", err)
		return
	}

	if !rows.Next() {
		query = fmt.Sprintf("replace into global set id=%d, time_stamp=%d, arena_season=%d, arena_week_end_time=%d, arena_season_end_time=%d", global.UltimateID, int32(time.Now().Unix()), 0, 0, 0)
		if stmp, err := m.db.PrepareContext(m.ctx, query); err == nil {
			if _, err := stmp.ExecContext(m.ctx); err == nil {
				logger.Info("sql global init query exec success:", query)
			}
		}
	}
}

func (m *DBMgr) Exec(q string) {
	go func() {
		if _, err := m.db.ExecContext(m.ctx, q); err != nil {
			logger.Error("db query failed:", err)
		}
	}()
}

func (m *DBMgr) Query(q string) (*sql.Rows, error) {
	return func() (*sql.Rows, error) {
		stmt, err := m.db.PrepareContext(m.ctx, q)
		if err != nil {
			logger.Warning(fmt.Sprintf("db query<%s> failed:", q), err)
			return nil, err
		}

		rows, err := stmt.QueryContext(m.ctx)
		if err != nil {
			logger.Warning(fmt.Sprintf("db query<%s> failed:", q), err)
			return nil, err
		}

		return rows, nil
	}()
}
<commit_msg>display query failed string<commit_after>package ultimate

import (
	"context"
	"database/sql"
	"fmt"
	"sync"
	"time"

	"github.com/hellodudu/Ultimate/global"
	"github.com/hellodudu/Ultimate/logger"
)

type DBMgr struct {
	db     *sql.DB
	ctx    context.Context
	cancel context.CancelFunc
	wg     sync.WaitGroup
	chStop chan struct{}
}

func NewDBMgr() (*DBMgr, error) {
	dbMgr := &DBMgr{
		chStop: make(chan struct{}, 1),
	}

	dbMgr.ctx, dbMgr.cancel = context.WithCancel(context.Background())

	mysqlDSN := fmt.Sprintf("%s:%s@(%s:%s)/%s", global.MysqlUser, global.MysqlPwd, global.MysqlAddr, global.MysqlPort, global.MysqlDB)
	var err error
	dbMgr.db, err = sql.Open("mysql", mysqlDSN)
	if err != nil {
		logger.Fatal(err)
		return nil, err
	}

	dbMgr.initDBMgr()
	return dbMgr, nil
}

func (m *DBMgr) Run() {
	for {
		select {
		case <-m.ctx.Done():
			logger.Print("db mgr context done!")
			m.chStop <- struct{}{}
			return
		}
	}

}

func (m *DBMgr) Stop() chan struct{} {
	m.db.Close()
	m.cancel()
	return m.chStop
}

func (m *DBMgr) initDBMgr() {

	m.wg.Add(1)
	go m.loadGlobal()

	m.wg.Wait()

}

func (m *DBMgr) loadGlobal() {
	defer m.wg.Done()

	query := "select * from global"
	stmt, err := m.db.PrepareContext(m.ctx, query)
	if err != nil {
		logger.Warning("api initdb failed:", err)
		return
	}

	rows, err := stmt.QueryContext(m.ctx)
	if err != nil {
		logger.Warning("api initdb failed:", err)
		return
	}

	if !rows.Next() {
		query = fmt.Sprintf("replace into global set id=%d, time_stamp=%d, arena_season=%d, arena_week_end_time=%d, arena_season_end_time=%d", global.UltimateID, int32(time.Now().Unix()), 0, 0, 0)
		if stmp, err := m.db.PrepareContext(m.ctx, query); err == nil {
			if _, err := stmp.ExecContext(m.ctx); err == nil {
				logger.Info("sql global init query exec success:", query)
			}
		}
	}
}

func (m *DBMgr) Exec(q string) {
	go func() {
		if _, err := m.db.ExecContext(m.ctx, q); err != nil {
			logger.Error(fmt.Sprintf("db exec<%s> failed:", q), err)
		}
	}()
}

func (m *DBMgr) Query(q string) (*sql.Rows, error) {
	return func() (*sql.Rows, error) {
		stmt, err := m.db.PrepareContext(m.ctx, q)
		if err != nil {
			logger.Warning(fmt.Sprintf("db query<%s> failed:", q), err)
			return nil, err
		}

		rows, err := stmt.QueryContext(m.ctx)
		if err != nil {
			logger.Warning(fmt.Sprintf("db query<%s> failed:", q), err)
			return nil, err
		}

		return rows, nil
	}()
}
<|endoftext|>


<commit_before>// Package einhorn allows you to communicate with the einhorn master from a Go
// worker.
package einhorn

import (
	"encoding/json"
	"errors"
	"net"
	"os"
	"strconv"
)

// CountListeners returns the number of listener fd's passed by the master.
func CountListeners() uint {
	count, err := strconv.ParseUint(os.Getenv("EINHORN_FD_COUNT"), 10, 64)
	if err != nil {
		return 0
	} else {
		return uint(count)
	}
}

// GetListener returns the passed listener with the specified index.
func GetListener(index uint) (net.Listener, error) {
	if CountListeners() < (index + 1) {
		return nil, errors.New("Too few EINHORN_FDs passed")
	}
	name := "EINHORN_FD_" + strconv.Itoa(int(index))

	listener_fileno, err := strconv.Atoi(os.Getenv(name))
	if err != nil {
		return nil, errors.New("Could not parse fd: " + err.Error())
	}

	listener_file := os.NewFile(uintptr(listener_fileno), name)
	listener, err := net.FileListener(listener_file)
	if err != nil {
		return nil, errors.New("Could not create listener from fd: " + err.Error())
	}

	return listener, nil
}

// Ack sends an ack to the einhorn master.
func Ack() error {
	return sendToMaster(workerMessage{Command: "worker:ack", Pid: os.Getpid()})
}

type workerMessage struct {
	Command string `json:"comand"`
	Pid     int
}

func sendToMaster(msg workerMessage) error {
	einhorn_sock_path := os.Getenv("EINHORN_SOCK_PATH")
	einhorn_control_conn, err := net.Dial("unix", einhorn_sock_path)
	if err != nil {
		return errors.New("Could not connect to einhorn master: " + err.Error())
	}

	serialized, err := json.Marshal(msg)
	if err != nil {
		return errors.New("Could not serialize message: " + err.Error())
	}

	_, err = einhorn_control_conn.Write(serialized)
	if err != nil {
		return errors.New("Could not write to einhorn master: " + err.Error())
	}

	_, err = einhorn_control_conn.Write([]byte("\n"))
	if err != nil {
		return errors.New("Could not write to einhorn master: " + err.Error())
	}

	err = einhorn_control_conn.Close()
	if err != nil {
		return errors.New("Could not close connection to einhorn master: " + err.Error())
	}

	return nil
}
<commit_msg>Don't use variables with underscores.<commit_after>// Package einhorn allows you to communicate with the einhorn master from a Go
// worker.
package einhorn

import (
	"encoding/json"
	"errors"
	"net"
	"os"
	"strconv"
)

// CountListeners returns the number of listener fd's passed by the master.
func CountListeners() uint {
	count, err := strconv.ParseUint(os.Getenv("EINHORN_FD_COUNT"), 10, 64)
	if err != nil {
		return 0
	} else {
		return uint(count)
	}
}

// GetListener returns the passed listener with the specified index.
func GetListener(index uint) (net.Listener, error) {
	if CountListeners() < (index + 1) {
		return nil, errors.New("Too few EINHORN_FDs passed")
	}
	name := "EINHORN_FD_" + strconv.Itoa(int(index))

	fileno, err := strconv.Atoi(os.Getenv(name))
	if err != nil {
		return nil, errors.New("Could not parse fd: " + err.Error())
	}

	file := os.NewFile(uintptr(fileno), name)
	listener, err := net.FileListener(file)
	if err != nil {
		return nil, errors.New("Could not create listener from fd: " + err.Error())
	}

	return listener, nil
}

// Ack sends an ack to the einhorn master.
func Ack() error {
	return sendToMaster(workerMessage{Command: "worker:ack", Pid: os.Getpid()})
}

type workerMessage struct {
	Command string `json:"comand"`
	Pid     int
}

func sendToMaster(msg workerMessage) error {
	sockPath := os.Getenv("EINHORN_SOCK_PATH")
	controlConn, err := net.Dial("unix", sockPath)
	if err != nil {
		return errors.New("Could not connect to einhorn master: " + err.Error())
	}

	serialized, err := json.Marshal(msg)
	if err != nil {
		return errors.New("Could not serialize message: " + err.Error())
	}

	_, err = controlConn.Write(serialized)
	if err != nil {
		return errors.New("Could not write to einhorn master: " + err.Error())
	}

	_, err = controlConn.Write([]byte("\n"))
	if err != nil {
		return errors.New("Could not write to einhorn master: " + err.Error())
	}

	err = controlConn.Close()
	if err != nil {
		return errors.New("Could not close connection to einhorn master: " + err.Error())
	}

	return nil
}
<|endoftext|>


<commit_before>package openstack

import (
	"fmt"
	"log"
	"time"

	"github.com/hashicorp/terraform/helper/hashcode"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"
	"github.com/rackspace/gophercloud"
	"github.com/rackspace/gophercloud/openstack/compute/v2/servers"
)

func resourceComputeInstance() *schema.Resource {
	return &schema.Resource{
		Create: resourceComputeInstanceCreate,
		Read:   resourceComputeInstanceRead,
		Update: resourceComputeInstanceUpdate,
		Delete: resourceComputeInstanceDelete,

		Schema: map[string]*schema.Schema{
			"name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: false,
			},

			"image_ref": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: false,
			},

			"flavor_ref": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: false,
			},

			"security_groups": &schema.Schema{
				Type:     schema.TypeSet,
				Optional: true,
				ForceNew: false,
				Elem:     &schema.Schema{Type: schema.TypeString},
				Set: func(v interface{}) int {
					return hashcode.String(v.(string))
				},
			},

			"availability_zone": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},

			"networks": &schema.Schema{
				Type:     schema.TypeList,
				Optional: true,
				ForceNew: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"uuid": &schema.Schema{
							Type:     schema.TypeString,
							Optional: true,
						},

						"port": &schema.Schema{
							Type:     schema.TypeString,
							Optional: true,
						},

						"fixed_ip": &schema.Schema{
							Type:     schema.TypeString,
							Optional: true,
						},
					},
				},
			},

			"metadata": &schema.Schema{
				Type:     schema.TypeMap,
				Optional: true,
			},

			"config_drive": &schema.Schema{
				Type:     schema.TypeBool,
				Optional: true,
				ForceNew: true,
			},

			"access_ip_v4": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
				Optional: true,
				ForceNew: false,
			},

			"access_ip_v6": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
				Optional: true,
				ForceNew: false,
			},
		},
	}
}

func resourceComputeInstanceCreate(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)
	osClient := config.computeV2Client

	createOpts := &servers.CreateOpts{
		Name:      d.Get("name").(string),
		ImageRef:  d.Get("image_ref").(string),
		FlavorRef: d.Get("flavor_ref").(string),
		//SecurityGroups []string
		AvailabilityZone: d.Get("availability_zone").(string),
		Networks:         resourceInstanceNetworks(d),
		Metadata:         resourceInstanceMetadata(d),
		ConfigDrive:      d.Get("config_drive").(bool),
	}

	log.Printf("[INFO] Requesting instance creation")
	server, err := servers.Create(osClient, createOpts).Extract()
	if err != nil {
		return fmt.Errorf("Error creating OpenStack server: %s", err)
	}
	log.Printf("[INFO] Instance ID: %s", server.ID)

	// Store the ID now
	d.SetId(server.ID)

	// Wait for the instance to become running so we can get some attributes
	// that aren't available until later.
	log.Printf(
		"[DEBUG] Waiting for instance (%s) to become running",
		server.ID)

	stateConf := &resource.StateChangeConf{
		Pending:    []string{"BUILD"},
		Target:     "ACTIVE",
		Refresh:    ServerStateRefreshFunc(osClient, server.ID),
		Timeout:    10 * time.Minute,
		Delay:      10 * time.Second,
		MinTimeout: 3 * time.Second,
	}

	serverRaw, err := stateConf.WaitForState()
	if err != nil {
		return fmt.Errorf(
			"Error waiting for instance (%s) to become ready: %s",
			server.ID, err)
	}

	server = serverRaw.(*servers.Server)

	return resourceComputeInstanceRead(d, meta)
}

func resourceComputeInstanceRead(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)
	osClient := config.computeV2Client

	server, err := servers.Get(osClient, d.Id()).Extract()
	if err != nil {
		return fmt.Errorf("Error retrieving OpenStack server: %s", err)
	}

	log.Printf("[DEBUG] Retreived Server %s: %+v", d.Id(), server)

	d.Set("name", server.Name)
	d.Set("access_ip_v4", server.AccessIPv4)
	d.Set("access_ip_v6", server.AccessIPv6)

	host := server.AccessIPv4
	if host == "" {
		if publicAddressesRaw, ok := server.Addresses["public"]; ok {
			publicAddresses := publicAddressesRaw.([]interface{})
			for _, paRaw := range publicAddresses {
				pa := paRaw.(map[string]interface{})
				if pa["version"].(float64) == 4 {
					host = pa["addr"].(string)
				}
			}
		}
	}

	log.Printf("host: %s", host)

	// Initialize the connection info
	d.SetConnInfo(map[string]string{
		"type": "ssh",
		"host": host,
	})

	d.Set("metadata", server.Metadata)

	return nil
}

func resourceComputeInstanceUpdate(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)
	osClient := config.computeV2Client

	var updateOpts servers.UpdateOpts
	// If the Metadata has changed, then update that.
	if d.HasChange("name") {
		updateOpts.Name = d.Get("name").(string)
	}
	if d.HasChange("access_ip_v4") {
		updateOpts.AccessIPv4 = d.Get("access_ip_v4").(string)
	}
	if d.HasChange("access_ip_v6") {
		updateOpts.AccessIPv4 = d.Get("access_ip_v6").(string)
	}

	// If there's nothing to update, don't waste an HTTP call.
	if updateOpts != (servers.UpdateOpts{}) {
		log.Printf("[DEBUG] Updating Server %s with options: %+v", d.Id(), updateOpts)

		_, err := servers.Update(osClient, d.Id(), updateOpts).Extract()
		if err != nil {
			return fmt.Errorf("Error updating Openstack server: %s", err)
		}
	}

	return resourceComputeInstanceRead(d, meta)
}

func resourceComputeInstanceDelete(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)
	osClient := config.computeV2Client

	err := servers.Delete(osClient, d.Id()).ExtractErr()
	if err != nil {
		return fmt.Errorf("Error deleting OpenStack server: %s", err)
	}

	// Wait for the instance to delete before moving on.
	log.Printf(
		"[DEBUG] Waiting for instance (%s) to delete",
		d.Id())

	stateConf := &resource.StateChangeConf{
		Target:     "",
		Refresh:    ServerStateRefreshFunc(osClient, d.Id()),
		Timeout:    10 * time.Minute,
		Delay:      10 * time.Second,
		MinTimeout: 3 * time.Second,
	}

	_, err = stateConf.WaitForState()
	if err != nil {
		return fmt.Errorf(
			"Error waiting for instance (%s) to delete: %s",
			d.Id(), err)
	}

	d.SetId("")
	return nil
}

// ServerStateRefreshFunc returns a resource.StateRefreshFunc that is used to watch
// an OpenStack instance.
func ServerStateRefreshFunc(client *gophercloud.ServiceClient, instanceID string) resource.StateRefreshFunc {
	return func() (interface{}, string, error) {
		s, err := servers.Get(client, instanceID).Extract()
		if err != nil {
			return nil, "", err
		}

		return s, s.Status, nil
	}
}

func resourceInstanceNetworks(d *schema.ResourceData) []servers.Network {
	rawNetworks := d.Get("networks").([]interface{})
	networks := make([]servers.Network, len(rawNetworks))
	for i, raw := range rawNetworks {
		rawMap := raw.(map[string]interface{})
		networks[i] = servers.Network{
			UUID:    rawMap["uuid"].(string),
			Port:    rawMap["port"].(string),
			FixedIP: rawMap["fixed_ip"].(string),
		}
	}
	return networks
}

func resourceInstanceMetadata(d *schema.ResourceData) map[string]string {
	m := make(map[string]string)
	for key, val := range d.Get("metadata").(map[string]interface{}) {
		m[key] = val.(string)
	}
	return m
}
<commit_msg>update openstack server metadata<commit_after>package openstack

import (
	"fmt"
	"log"
	"time"

	"github.com/hashicorp/terraform/helper/hashcode"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"
	"github.com/rackspace/gophercloud"
	"github.com/rackspace/gophercloud/openstack/compute/v2/servers"
)

func resourceComputeInstance() *schema.Resource {
	return &schema.Resource{
		Create: resourceComputeInstanceCreate,
		Read:   resourceComputeInstanceRead,
		Update: resourceComputeInstanceUpdate,
		Delete: resourceComputeInstanceDelete,

		Schema: map[string]*schema.Schema{
			"name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: false,
			},

			"image_ref": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: false,
			},

			"flavor_ref": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: false,
			},

			"security_groups": &schema.Schema{
				Type:     schema.TypeSet,
				Optional: true,
				ForceNew: false,
				Elem:     &schema.Schema{Type: schema.TypeString},
				Set: func(v interface{}) int {
					return hashcode.String(v.(string))
				},
			},

			"availability_zone": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},

			"networks": &schema.Schema{
				Type:     schema.TypeList,
				Optional: true,
				ForceNew: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"uuid": &schema.Schema{
							Type:     schema.TypeString,
							Optional: true,
						},

						"port": &schema.Schema{
							Type:     schema.TypeString,
							Optional: true,
						},

						"fixed_ip": &schema.Schema{
							Type:     schema.TypeString,
							Optional: true,
						},
					},
				},
			},

			"metadata": &schema.Schema{
				Type:     schema.TypeMap,
				Optional: true,
			},

			"config_drive": &schema.Schema{
				Type:     schema.TypeBool,
				Optional: true,
				ForceNew: true,
			},

			"access_ip_v4": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
				Optional: true,
				ForceNew: false,
			},

			"access_ip_v6": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
				Optional: true,
				ForceNew: false,
			},
		},
	}
}

func resourceComputeInstanceCreate(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)
	osClient := config.computeV2Client

	createOpts := &servers.CreateOpts{
		Name:      d.Get("name").(string),
		ImageRef:  d.Get("image_ref").(string),
		FlavorRef: d.Get("flavor_ref").(string),
		//SecurityGroups []string
		AvailabilityZone: d.Get("availability_zone").(string),
		Networks:         resourceInstanceNetworks(d),
		Metadata:         resourceInstanceMetadata(d),
		ConfigDrive:      d.Get("config_drive").(bool),
	}

	log.Printf("[INFO] Requesting instance creation")
	server, err := servers.Create(osClient, createOpts).Extract()
	if err != nil {
		return fmt.Errorf("Error creating OpenStack server: %s", err)
	}
	log.Printf("[INFO] Instance ID: %s", server.ID)

	// Store the ID now
	d.SetId(server.ID)

	// Wait for the instance to become running so we can get some attributes
	// that aren't available until later.
	log.Printf(
		"[DEBUG] Waiting for instance (%s) to become running",
		server.ID)

	stateConf := &resource.StateChangeConf{
		Pending:    []string{"BUILD"},
		Target:     "ACTIVE",
		Refresh:    ServerStateRefreshFunc(osClient, server.ID),
		Timeout:    10 * time.Minute,
		Delay:      10 * time.Second,
		MinTimeout: 3 * time.Second,
	}

	serverRaw, err := stateConf.WaitForState()
	if err != nil {
		return fmt.Errorf(
			"Error waiting for instance (%s) to become ready: %s",
			server.ID, err)
	}

	server = serverRaw.(*servers.Server)

	return resourceComputeInstanceRead(d, meta)
}

func resourceComputeInstanceRead(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)
	osClient := config.computeV2Client

	server, err := servers.Get(osClient, d.Id()).Extract()
	if err != nil {
		return fmt.Errorf("Error retrieving OpenStack server: %s", err)
	}

	log.Printf("[DEBUG] Retreived Server %s: %+v", d.Id(), server)

	d.Set("name", server.Name)
	d.Set("access_ip_v4", server.AccessIPv4)
	d.Set("access_ip_v6", server.AccessIPv6)

	host := server.AccessIPv4
	if host == "" {
		if publicAddressesRaw, ok := server.Addresses["public"]; ok {
			publicAddresses := publicAddressesRaw.([]interface{})
			for _, paRaw := range publicAddresses {
				pa := paRaw.(map[string]interface{})
				if pa["version"].(float64) == 4 {
					host = pa["addr"].(string)
				}
			}
		}
	}

	log.Printf("host: %s", host)

	// Initialize the connection info
	d.SetConnInfo(map[string]string{
		"type": "ssh",
		"host": host,
	})

	d.Set("metadata", server.Metadata)

	return nil
}

func resourceComputeInstanceUpdate(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)
	osClient := config.computeV2Client

	var updateOpts servers.UpdateOpts
	// If the Metadata has changed, then update that.
	if d.HasChange("name") {
		updateOpts.Name = d.Get("name").(string)
	}
	if d.HasChange("access_ip_v4") {
		updateOpts.AccessIPv4 = d.Get("access_ip_v4").(string)
	}
	if d.HasChange("access_ip_v6") {
		updateOpts.AccessIPv4 = d.Get("access_ip_v6").(string)
	}

	// If there's nothing to update, don't waste an HTTP call.
	if updateOpts != (servers.UpdateOpts{}) {
		log.Printf("[DEBUG] Updating Server %s with options: %+v", d.Id(), updateOpts)

		_, err := servers.Update(osClient, d.Id(), updateOpts).Extract()
		if err != nil {
			return fmt.Errorf("Error updating OpenStack server: %s", err)
		}
	}

	if d.HasChange("metadata") {
		var metadataOpts servers.MetadataOpts
		metadataOpts = make(servers.MetadataOpts)
		newMetadata := d.Get("metadata").(map[string]interface{})
		for k, v := range newMetadata {
			metadataOpts[k] = v.(string)
		}

		_, err := servers.UpdateMetadata(osClient, d.Id(), metadataOpts).Extract()
		if err != nil {
			return fmt.Errorf("Error updating OpenStack server (%s) metadata: %s", d.Id(), err)
		}
	}

	return resourceComputeInstanceRead(d, meta)
}

func resourceComputeInstanceDelete(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)
	osClient := config.computeV2Client

	err := servers.Delete(osClient, d.Id()).ExtractErr()
	if err != nil {
		return fmt.Errorf("Error deleting OpenStack server: %s", err)
	}

	// Wait for the instance to delete before moving on.
	log.Printf(
		"[DEBUG] Waiting for instance (%s) to delete",
		d.Id())

	stateConf := &resource.StateChangeConf{
		Target:     "",
		Refresh:    ServerStateRefreshFunc(osClient, d.Id()),
		Timeout:    10 * time.Minute,
		Delay:      10 * time.Second,
		MinTimeout: 3 * time.Second,
	}

	_, err = stateConf.WaitForState()
	if err != nil {
		return fmt.Errorf(
			"Error waiting for instance (%s) to delete: %s",
			d.Id(), err)
	}

	d.SetId("")
	return nil
}

// ServerStateRefreshFunc returns a resource.StateRefreshFunc that is used to watch
// an OpenStack instance.
func ServerStateRefreshFunc(client *gophercloud.ServiceClient, instanceID string) resource.StateRefreshFunc {
	return func() (interface{}, string, error) {
		s, err := servers.Get(client, instanceID).Extract()
		if err != nil {
			return nil, "", err
		}

		return s, s.Status, nil
	}
}

func resourceInstanceNetworks(d *schema.ResourceData) []servers.Network {
	rawNetworks := d.Get("networks").([]interface{})
	networks := make([]servers.Network, len(rawNetworks))
	for i, raw := range rawNetworks {
		rawMap := raw.(map[string]interface{})
		networks[i] = servers.Network{
			UUID:    rawMap["uuid"].(string),
			Port:    rawMap["port"].(string),
			FixedIP: rawMap["fixed_ip"].(string),
		}
	}
	return networks
}

func resourceInstanceMetadata(d *schema.ResourceData) map[string]string {
	m := make(map[string]string)
	for key, val := range d.Get("metadata").(map[string]interface{}) {
		m[key] = val.(string)
	}
	return m
}
<|endoftext|>


<commit_before>package core

import (
	"fmt"
	"sync"
	"testing"

	"github.com/OpenBazaar/openbazaar-go/pb"
	"github.com/OpenBazaar/openbazaar-go/repo"
	"github.com/OpenBazaar/openbazaar-go/repo/db"
	"github.com/OpenBazaar/openbazaar-go/schema"
	wi "github.com/OpenBazaar/wallet-interface"
	"github.com/op/go-logging"
)

// DISPUTE CASES
func TestPerformTaskInboundMessageScanner(t *testing.T) {
	var (
		orderMsgWithNoErr = repo.OrderMessage{
			MessageID:   "1",
			OrderID:     "1",
			MessageType: int32(pb.Message_ORDER),
			Message:     []byte("sample message"),
			MsgErr:      "",
			PeerID:      "sample",
			PeerPubkey:  []byte("sample"),
		}

		orderMsgWithErr = repo.OrderMessage{
			MessageID:   "2",
			OrderID:     "2",
			MessageType: int32(pb.Message_ORDER),
			Message:     []byte("sample message"),
			MsgErr:      ErrInsufficientFunds.Error(),
			PeerID:      "sample",
			PeerPubkey:  []byte("sample"),
		}

		existingRecords = []repo.OrderMessage{
			orderMsgWithNoErr,
			orderMsgWithErr,
		}

		appSchema = schema.MustNewCustomSchemaManager(schema.SchemaContext{
			DataPath:        schema.GenerateTempPath(),
			TestModeEnabled: true,
		})
	)

	if err := appSchema.BuildSchemaDirectories(); err != nil {
		t.Fatal(err)
	}
	defer appSchema.DestroySchemaDirectories()
	if err := appSchema.InitializeDatabase(); err != nil {
		t.Fatal(err)
	}

	database, err := appSchema.OpenDatabase()
	if err != nil {
		t.Fatal(err)
	}
	s, err := database.Prepare("insert into messages (messageID, orderID, message_type, message, peerID, err, pubkey) values (?, ?, ?, ?, ?, ?, ?)")
	if err != nil {
		t.Fatal(err)
	}

	for _, r := range existingRecords {
		_, err = s.Exec(r.MessageID, r.OrderID, r.MessageType, r.Message, r.PeerID, r.MsgErr, r.PeerPubkey)
		if err != nil {
			t.Fatal(err)
		}
	}

	datastore := db.NewSQLiteDatastore(database, new(sync.Mutex), wi.Bitcoin)
	worker := &inboundMessageScanner{
		datastore: datastore,
		logger:    logging.MustGetLogger("testInboundMsgScanner"),
	}

	//worker.PerformTask()
	fmt.Println(worker)
	msgs, err := worker.datastore.Messages().GetAllErrored()

	fmt.Println(len(msgs))
	fmt.Println(err)

}
<commit_msg>add scanner tests - first pass<commit_after>package core

import (
	"fmt"
	"sync"
	"testing"

	"github.com/OpenBazaar/openbazaar-go/pb"
	"github.com/OpenBazaar/openbazaar-go/repo"
	"github.com/OpenBazaar/openbazaar-go/repo/db"
	"github.com/OpenBazaar/openbazaar-go/schema"
	wi "github.com/OpenBazaar/wallet-interface"
	"github.com/op/go-logging"
)

func TestPerformTaskInboundMessageScanner(t *testing.T) {
	var (
		orderMsgWithNoErr = repo.OrderMessage{
			MessageID:   "1",
			OrderID:     "1",
			MessageType: int32(pb.Message_ORDER),
			Message:     []byte("sample message"),
			MsgErr:      "",
			PeerID:      "sample",
			PeerPubkey:  []byte("sample"),
		}

		orderMsgWithErr = repo.OrderMessage{
			MessageID:   "2",
			OrderID:     "2",
			MessageType: int32(pb.Message_ORDER),
			Message:     []byte("sample message"),
			MsgErr:      ErrInsufficientFunds.Error(),
			PeerID:      "sample",
			PeerPubkey:  []byte("sample"),
		}

		existingRecords = []repo.OrderMessage{
			orderMsgWithNoErr,
			orderMsgWithErr,
		}

		appSchema = schema.MustNewCustomSchemaManager(schema.SchemaContext{
			DataPath:        schema.GenerateTempPath(),
			TestModeEnabled: true,
		})
	)

	if err := appSchema.BuildSchemaDirectories(); err != nil {
		t.Fatal(err)
	}
	defer appSchema.DestroySchemaDirectories()
	if err := appSchema.InitializeDatabase(); err != nil {
		t.Fatal(err)
	}

	database, err := appSchema.OpenDatabase()
	if err != nil {
		t.Fatal(err)
	}
	s, err := database.Prepare("insert into messages (messageID, orderID, message_type, message, peerID, err, pubkey) values (?, ?, ?, ?, ?, ?, ?)")
	if err != nil {
		t.Fatal(err)
	}

	for _, r := range existingRecords {
		_, err = s.Exec(r.MessageID, r.OrderID, r.MessageType, r.Message, r.PeerID, r.MsgErr, r.PeerPubkey)
		if err != nil {
			t.Fatal(err)
		}
	}

	datastore := db.NewSQLiteDatastore(database, new(sync.Mutex), wi.Bitcoin)
	worker := &inboundMessageScanner{
		datastore: datastore,
		logger:    logging.MustGetLogger("testInboundMsgScanner"),
	}

	//worker.PerformTask()
	fmt.Println(worker)
	msgs, err := worker.datastore.Messages().GetAllErrored()

	fmt.Println(len(msgs))
	fmt.Println(err)

}
<|endoftext|>