<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"reflect"
	"regexp"
	"sort"
	"strings"

	"github.com/stts-se/rbg2p"

	"github.com/sergi/go-diff/diffmatchpatch"
)

var l = log.New(os.Stderr, "", 0)

func print(input string, orth string, transes []string) {
	fmt.Printf("%s\t%s\n", input, strings.Join(transes, "  #  "))
}

type transResult struct {
	orth    string
	transes []string
	result  bool
}

func transcribe(ruleSet rbg2p.RuleSet, orth string) transResult {
	transes, err := ruleSet.Apply(orth)
	if err != nil {
		l.Printf("Couldn't transcribe '%s' : %s", orth, err)
		return transResult{orth: orth, transes: transes, result: false}
	}
	return transResult{orth: orth, transes: transes, result: true}
}

var removeBoundariesRE = regexp.MustCompile(`[.!~] *`)
var removeStressRE = regexp.MustCompile(`[%"] *`)
var removeStress *bool

func cleanTransForDiff(t string) string {
	var res = t
	res = removeBoundariesRE.ReplaceAllString(res, "")
	if *removeStress {
		res = removeStressRE.ReplaceAllString(res, "")
	}
	//res = strings.Replace(res, "'", "", -1)
	return res
}

func cleanTransForIJDiff(t string) string {
	var res = t
	res = strings.Replace(res, " i ", " j ", -1)
	return res
}

func compareForDiff(old []string, new []string) (string, bool) {
	for i, s := range old {
		old[i] = cleanTransForDiff(s)
	}
	for i, s := range new {
		new[i] = cleanTransForDiff(s)
	}
	// var oldIJ = []string{}
	// var newIJ = []string{}
	// for _, s := range old {
	// 	oldIJ = append(oldIJ, cleanTransForIJDiff(s))
	// }
	// for _, s := range new {
	// 	newIJ = append(newIJ, cleanTransForIJDiff(s))
	// }
	if reflect.DeepEqual(old, new) {
		return "ALL EQ", true
	} else if old[0] == new[0] {
		return "#1 EQ", false
		// } else if reflect.DeepEqual(oldIJ, newIJ) {
		// 	return "ALL EQ IJ", false
		// } else if oldIJ[0] == newIJ[0] {
		// 	return "#1 EQ IJ", false
	} else {
		return "DIFF", false
	}
}

func main() {

	var f = flag.NewFlagSet(os.Args[0], flag.ContinueOnError)
	var debug = f.Bool("debug", false, "print extra debug info (default: false)")
	var force = f.Bool("force", false, "print transcriptions even if errors are found (default: false)")
	var column = f.Int("column", 0, "only convert specified column (default: first field)")
	var quiet = f.Bool("quiet", false, "inhibit warnings (default: false)")
	var test = f.Bool("test", false, "test g2p against input file; orth <tab> trans (default: false)")
	removeStress = f.Bool("test:removestress", false, "remove stress when comparing using the -test switch (default: false)")
	var ssFile = f.String("symbolset", "", "use specified symbol set file for validating the symbols in the g2p rule set (default: none; overrides the g2p rule file's symbolset, if any)")
	var help = f.Bool("help", false, "print help message")

	var usage = `go run g2p.go <FLAGS> <G2P RULE FILE> <WORDS (FILES OR LIST OF WORDS)> (optional)

FLAGS:
   -force      bool    print transcriptions even if errors are found (default: false)
   -debug      bool    print extra debug info (default: false)
   -column     string  only convert specified column (default: first field)
   -quiet      bool    inhibit warnings (default: false)
   -test       bool    test g2p against input file; orth <tab> trans (default: false)
   -test:removestress bool remove stress when comparing using the -test switch (default: false)
   -symbolset  string  use specified symbol set file for validating the symbols in the g2p rule set (default: none)
   -help       bool    print help message`

	f.Usage = func() {
		l.Printf(usage)
	}

	var args = os.Args
	if strings.HasSuffix(args[0], "g2p") {
		args = args[1:] // remove first argument if it's the program name
	}
	err := f.Parse(args)
	if err != nil {
		os.Exit(1)
	}

	args = f.Args()

	if *help {
		l.Println(usage)
		os.Exit(1)
	}

	if len(args) < 1 {
		l.Println(usage)
		os.Exit(1)
	}

	rbg2p.Debug = *debug

	g2pFile := args[0]
	ruleSet, err := rbg2p.LoadFile(g2pFile)
	if err != nil {
		l.Printf("couldn't load rule file %s : %s", g2pFile, err)
		os.Exit(1)
	}

	if *ssFile != "" {
		phonemeSet, err := rbg2p.LoadPhonemeSetFile(*ssFile, ruleSet.PhonemeDelimiter)
		if err != nil {
			l.Printf("couldn't load symbol set : %s", err)
			os.Exit(1)
		}
		ruleSet.PhonemeSet = phonemeSet
	}

	haltingError := false
	result := ruleSet.Test()
	for _, e := range result.Errors {
		l.Printf("ERROR: %v\n", e)
	}
	l.Printf("%d ERROR(S) FOR %s\n", len(result.Errors), g2pFile)
	if !*quiet {
		for _, e := range result.Warnings {
			l.Printf("WARNING: %v\n", e)
		}
	}
	l.Printf("%d WARNING(S) FOR %s\n", len(result.Warnings), g2pFile)
	if len(result.Errors) > 0 {
		haltingError = true
	}
	if len(result.FailedTests) > 0 {
		for _, e := range result.FailedTests {
			l.Printf("FAILED TEST: %v\n", e)
		}
		l.Printf("%d OF %d TESTS FAILED FOR %s\n", len(result.FailedTests), len(ruleSet.Tests), g2pFile)
		haltingError = true
	} else {
		l.Printf("ALL %d TESTS PASSED FOR %s\n", len(ruleSet.Tests), g2pFile)
	}

	if haltingError && !*force {
		os.Exit(1)
	}

	nTotal := 0
	nErrs := 0
	nTrans := 0
	nTests := 0
	testRes := make(map[string]int)
	if *test {
		fmt.Println("ORTH\tNEW TRANSES\tOLD TRANSES\tDIFFTAG\t(DIFF)?")
	}
	var processString = func(s string) {
		nTotal = nTotal + 1
		fs := strings.Split(s, "\t")
		o := fs[*column]
		res := transcribe(ruleSet, o)
		if res.result || *force {
			nTrans = nTrans + 1
			if *test {
				refTranses := fs[(*column + 1):]
				nTests++
				info, _ := compareForDiff(res.transes, refTranses)
				testRes[info]++
				outFs := []string{res.orth, strings.Join(res.transes, " # "), strings.Join(refTranses, "#"), info}
				if info == "DIFF" {
					dmp := diffmatchpatch.New()
					diffs := dmp.DiffMain(outFs[1], outFs[2], false)
					diffsOnly := []diffmatchpatch.Diff{}
					diffsOnlyText := []string{}
					for _, d := range diffs {
						if d.Type != diffmatchpatch.DiffEqual {
							diffsOnly = append(diffsOnly, d)
							diffsOnlyText = append(diffsOnlyText, d.Text)
						}
					}
					outFs = append(outFs, dmp.DiffPrettyText(diffs))
					outFs = append(outFs, fmt.Sprintf("%v", diffsOnly))
					outFs = append(outFs, strings.Join(diffsOnlyText, "|"))
				}

				fmt.Println(strings.Join(outFs, "\t"))
			} else {
				print(s, res.orth, res.transes)
			}
		}
		if !res.result {
			nErrs = nErrs + 1
		}
	}

	if len(args) > 1 {
		for i := 1; i < len(args); i++ {
			s := args[i]
			if _, err := os.Stat(s); os.IsNotExist(err) {
				processString(s)
				// nTotal = nTotal + 1
				// res := transcribe(ruleSet, s)
				// if res.result || *force {
				// 	nTrans = nTrans + 1
				// 	fmt.Printf("%s\t%s\n", s, strings.Join(res.transes, "\t"))
				// }
				// if !res.result {
				// 	nErrs = nErrs + 1
				// }
			} else {
				fh, err := os.Open(filepath.Clean(s))
				if err != nil {
					l.Println(err)
					os.Exit(1)
				}
				defer fh.Close()
				sc := bufio.NewScanner(fh)
				for sc.Scan() {
					if err := sc.Err(); err != nil {
						l.Println(err)
						os.Exit(1)
					}
					line := sc.Text()
					if strings.TrimSpace(line) == "" {
						l.Println("Skipping empty line")
						continue
					}
					if strings.HasPrefix(strings.TrimSpace(line), "#") {
						l.Println("Skipping line " + line)
						continue
					}
					processString(line)
					// 	nTotal = nTotal + 1
					// 	fs := strings.Split(line, "\t")
					// 	o, refTranses := fs[0], fs[1:]
					// 	res := transcribe(ruleSet, o)
					// 	if res.result || *force {
					// 		nTrans = nTrans + 1
					// 		if *test {
					// 			nTests++
					// 			info, _ := compareForDiff(res.transes, refTranses)
					// 			testRes[info]++
					// 			outFs := []string{res.orth, strings.Join(res.transes, " # "), strings.Join(refTranses, "#"), info}
					// 			if info == "DIFF" {
					// 				dmp := diffmatchpatch.New()
					// 				diffs := dmp.DiffMain(outFs[1], outFs[2], false)
					// 				diffsOnly := []diffmatchpatch.Diff{}
					// 				diffsOnlyText := []string{}
					// 				for _, d := range diffs {
					// 					if d.Type != diffmatchpatch.DiffEqual {
					// 						diffsOnly = append(diffsOnly, d)
					// 						diffsOnlyText = append(diffsOnlyText, d.Text)
					// 					}
					// 				}
					// 				outFs = append(outFs, dmp.DiffPrettyText(diffs))
					// 				outFs = append(outFs, fmt.Sprintf("%v", diffsOnly))
					// 				outFs = append(outFs, strings.Join(diffsOnlyText, "|"))
					// 			}

					// 			fmt.Println(strings.Join(outFs, "\t"))
					// 		} else {
					// 			print(res.orth, res.transes)
					// 		}
					// 	}
					// 	if !res.result {
					// 		nErrs = nErrs + 1
					// 	}
				}
			}
		}
	} else {
		fmt.Fprintf(os.Stderr, "Reading input from stdin...\n")
		scanner := bufio.NewScanner(os.Stdin)
		for scanner.Scan() {
			line := scanner.Text()
			if strings.TrimSpace(line) == "" {
				l.Println("Skipping empty line")
				continue
			}
			if strings.HasPrefix(strings.TrimSpace(line), "#") {
				l.Println("Skipping line " + line)
				continue
			}
			processString(line)
		}
	}
	l.Printf("%-18s: % 7d", "TOTAL INPUT", nTotal)
	l.Printf("%-18s: % 7d", "ERRORS", nErrs)
	l.Printf("%-18s: % 7d", "TRANSCRIBED", nTrans)
	if *test {
		l.Printf("%-18s: % 7d", "TESTED", nTests)
		var keys []string
		for k := range testRes {
			keys = append(keys, k)
		}
		sort.Strings(keys)
		for _, tag := range keys {
			freq := testRes[tag]
			s := " > TEST " + tag
			l.Printf("%-18s: % 7d", s, freq)
		}
	}
}
<commit_msg>g2p.go improved diff<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"reflect"
	"regexp"
	"sort"
	"strings"

	"github.com/stts-se/rbg2p"

	"github.com/sergi/go-diff/diffmatchpatch"
)

var l = log.New(os.Stderr, "", 0)

func print(input string, orth string, transes []string) {
	fmt.Printf("%s\t%s\n", input, strings.Join(transes, "  #  "))
}

type transResult struct {
	orth    string
	transes []string
	result  bool
}

func transcribe(ruleSet rbg2p.RuleSet, orth string) transResult {
	transes, err := ruleSet.Apply(orth)
	if err != nil {
		l.Printf("Couldn't transcribe '%s' : %s", orth, err)
		return transResult{orth: orth, transes: transes, result: false}
	}
	return transResult{orth: orth, transes: transes, result: true}
}

var removeBoundariesRE = regexp.MustCompile(`[.!~] *`)
var removeStressRE = regexp.MustCompile(`[%"] *`)
var removeStress *bool
var transSplitRE = regexp.MustCompile(" +# +")

func cleanTransForDiff(t string) string {
	var res = t
	res = removeBoundariesRE.ReplaceAllString(res, "")
	if *removeStress {
		res = removeStressRE.ReplaceAllString(res, "")
	}
	//res = strings.Replace(res, "'", "", -1)
	return res
}

func cleanTransForIJDiff(t string) string {
	var res = t
	res = strings.Replace(res, " i ", " j ", -1)
	return res
}

func compareForDiff(old []string, new []string) (string, bool) {
	for i, s := range old {
		old[i] = cleanTransForDiff(s)
	}
	for i, s := range new {
		new[i] = cleanTransForDiff(s)
	}
	// var oldIJ = []string{}
	// var newIJ = []string{}
	// for _, s := range old {
	// 	oldIJ = append(oldIJ, cleanTransForIJDiff(s))
	// }
	// for _, s := range new {
	// 	newIJ = append(newIJ, cleanTransForIJDiff(s))
	// }
	if reflect.DeepEqual(old, new) {
		return "ALL EQ", true
	} else if old[0] == new[0] {
		return "#1 EQ", false
		// } else if reflect.DeepEqual(oldIJ, newIJ) {
		// 	return "ALL EQ IJ", false
		// } else if oldIJ[0] == newIJ[0] {
		// 	return "#1 EQ IJ", false
	} else {
		return "DIFF", false
	}
}

func main() {

	var f = flag.NewFlagSet(os.Args[0], flag.ContinueOnError)
	var debug = f.Bool("debug", false, "print extra debug info (default: false)")
	var force = f.Bool("force", false, "print transcriptions even if errors are found (default: false)")
	var column = f.Int("column", 0, "only convert specified column (default: first field)")
	var quiet = f.Bool("quiet", false, "inhibit warnings (default: false)")
	var test = f.Bool("test", false, "test g2p against input file; orth <tab> trans (default: false)")
	removeStress = f.Bool("test:removestress", false, "remove stress when comparing using the -test switch (default: false)")
	var ssFile = f.String("symbolset", "", "use specified symbol set file for validating the symbols in the g2p rule set (default: none; overrides the g2p rule file's symbolset, if any)")
	var help = f.Bool("help", false, "print help message")

	var usage = `go run g2p.go <FLAGS> <G2P RULE FILE> <WORDS (FILES OR LIST OF WORDS)> (optional)

FLAGS:
   -force      bool    print transcriptions even if errors are found (default: false)
   -debug      bool    print extra debug info (default: false)
   -column     string  only convert specified column (default: first field)
   -quiet      bool    inhibit warnings (default: false)
   -test       bool    test g2p against input file; orth <tab> trans (default: false)
   -test:removestress bool remove stress when comparing using the -test switch (default: false)
   -symbolset  string  use specified symbol set file for validating the symbols in the g2p rule set (default: none)
   -help       bool    print help message`

	f.Usage = func() {
		l.Printf(usage)
	}

	var args = os.Args
	if strings.HasSuffix(args[0], "g2p") {
		args = args[1:] // remove first argument if it's the program name
	}
	err := f.Parse(args)
	if err != nil {
		os.Exit(1)
	}

	args = f.Args()

	if *help {
		l.Println(usage)
		os.Exit(1)
	}

	if len(args) < 1 {
		l.Println(usage)
		os.Exit(1)
	}

	rbg2p.Debug = *debug

	g2pFile := args[0]
	ruleSet, err := rbg2p.LoadFile(g2pFile)
	if err != nil {
		l.Printf("couldn't load rule file %s : %s", g2pFile, err)
		os.Exit(1)
	}

	if *ssFile != "" {
		phonemeSet, err := rbg2p.LoadPhonemeSetFile(*ssFile, ruleSet.PhonemeDelimiter)
		if err != nil {
			l.Printf("couldn't load symbol set : %s", err)
			os.Exit(1)
		}
		ruleSet.PhonemeSet = phonemeSet
	}

	haltingError := false
	result := ruleSet.Test()
	for _, e := range result.Errors {
		l.Printf("ERROR: %v\n", e)
	}
	l.Printf("%d ERROR(S) FOR %s\n", len(result.Errors), g2pFile)
	if !*quiet {
		for _, e := range result.Warnings {
			l.Printf("WARNING: %v\n", e)
		}
	}
	l.Printf("%d WARNING(S) FOR %s\n", len(result.Warnings), g2pFile)
	if len(result.Errors) > 0 {
		haltingError = true
	}
	if len(result.FailedTests) > 0 {
		for _, e := range result.FailedTests {
			l.Printf("FAILED TEST: %v\n", e)
		}
		l.Printf("%d OF %d TESTS FAILED FOR %s\n", len(result.FailedTests), len(ruleSet.Tests), g2pFile)
		haltingError = true
	} else {
		l.Printf("ALL %d TESTS PASSED FOR %s\n", len(ruleSet.Tests), g2pFile)
	}

	if haltingError && !*force {
		os.Exit(1)
	}

	nTotal := 0
	nErrs := 0
	nTrans := 0
	nTests := 0
	testRes := make(map[string]int)
	if *test {
		fmt.Println("ORTH\tG2P TRANSES\tREF TRANSES\tDIFFTAG\t(DIFF)?")
	}
	var processString = func(s string) {
		nTotal = nTotal + 1
		fs := strings.Split(s, "\t")
		o := fs[*column]
		res := transcribe(ruleSet, o)
		if res.result || *force {
			nTrans = nTrans + 1
			if *test {
				refTranses := []string{}
				for _, s := range fs[(*column + 1):] {
					for _, refT := range transSplitRE.Split(s, -1) {
						refTranses = append(refTranses, refT)
					}
				}
				nTests++
				info, _ := compareForDiff(res.transes, refTranses)
				testRes[info]++
				outFs := []string{res.orth, strings.Join(res.transes, " # "), strings.Join(refTranses, " # "), info}
				if info == "DIFF" {
					dmp := diffmatchpatch.New()
					diffs := dmp.DiffMain(outFs[1], outFs[2], false)
					diffsOnly := []diffmatchpatch.Diff{}
					diffsOnlyText := []string{}
					for _, d := range diffs {
						if d.Type != diffmatchpatch.DiffEqual {
							diffsOnly = append(diffsOnly, d)
							diffsOnlyText = append(diffsOnlyText, d.Text)
						}
					}
					outFs = append(outFs, dmp.DiffPrettyText(diffs))
					outFs = append(outFs, fmt.Sprintf("%v", diffsOnly))
					outFs = append(outFs, strings.Join(diffsOnlyText, "|"))
				}

				fmt.Println(strings.Join(outFs, "\t"))
			} else {
				print(s, res.orth, res.transes)
			}
		}
		if !res.result {
			nErrs = nErrs + 1
		}
	}

	if len(args) > 1 {
		for i := 1; i < len(args); i++ {
			s := args[i]
			if _, err := os.Stat(s); os.IsNotExist(err) {
				processString(s)
				// nTotal = nTotal + 1
				// res := transcribe(ruleSet, s)
				// if res.result || *force {
				// 	nTrans = nTrans + 1
				// 	fmt.Printf("%s\t%s\n", s, strings.Join(res.transes, "\t"))
				// }
				// if !res.result {
				// 	nErrs = nErrs + 1
				// }
			} else {
				fh, err := os.Open(filepath.Clean(s))
				if err != nil {
					l.Println(err)
					os.Exit(1)
				}
				defer fh.Close()
				sc := bufio.NewScanner(fh)
				for sc.Scan() {
					if err := sc.Err(); err != nil {
						l.Println(err)
						os.Exit(1)
					}
					line := sc.Text()
					if strings.TrimSpace(line) == "" {
						l.Println("Skipping empty line")
						continue
					}
					if strings.HasPrefix(strings.TrimSpace(line), "#") {
						l.Println("Skipping line " + line)
						continue
					}
					processString(line)
					// 	nTotal = nTotal + 1
					// 	fs := strings.Split(line, "\t")
					// 	o, refTranses := fs[0], fs[1:]
					// 	res := transcribe(ruleSet, o)
					// 	if res.result || *force {
					// 		nTrans = nTrans + 1
					// 		if *test {
					// 			nTests++
					// 			info, _ := compareForDiff(res.transes, refTranses)
					// 			testRes[info]++
					// 			outFs := []string{res.orth, strings.Join(res.transes, " # "), strings.Join(refTranses, "#"), info}
					// 			if info == "DIFF" {
					// 				dmp := diffmatchpatch.New()
					// 				diffs := dmp.DiffMain(outFs[1], outFs[2], false)
					// 				diffsOnly := []diffmatchpatch.Diff{}
					// 				diffsOnlyText := []string{}
					// 				for _, d := range diffs {
					// 					if d.Type != diffmatchpatch.DiffEqual {
					// 						diffsOnly = append(diffsOnly, d)
					// 						diffsOnlyText = append(diffsOnlyText, d.Text)
					// 					}
					// 				}
					// 				outFs = append(outFs, dmp.DiffPrettyText(diffs))
					// 				outFs = append(outFs, fmt.Sprintf("%v", diffsOnly))
					// 				outFs = append(outFs, strings.Join(diffsOnlyText, "|"))
					// 			}

					// 			fmt.Println(strings.Join(outFs, "\t"))
					// 		} else {
					// 			print(res.orth, res.transes)
					// 		}
					// 	}
					// 	if !res.result {
					// 		nErrs = nErrs + 1
					// 	}
				}
			}
		}
	} else {
		fmt.Fprintf(os.Stderr, "Reading input from stdin...\n")
		scanner := bufio.NewScanner(os.Stdin)
		for scanner.Scan() {
			line := scanner.Text()
			if strings.TrimSpace(line) == "" {
				l.Println("Skipping empty line")
				continue
			}
			if strings.HasPrefix(strings.TrimSpace(line), "#") {
				l.Println("Skipping line " + line)
				continue
			}
			processString(line)
		}
	}
	l.Printf("%-18s: % 7d", "TOTAL INPUT", nTotal)
	l.Printf("%-18s: % 7d", "ERRORS", nErrs)
	l.Printf("%-18s: % 7d", "TRANSCRIBED", nTrans)
	if *test {
		l.Printf("%-18s: % 7d", "TESTED", nTests)
		var keys []string
		for k := range testRes {
			keys = append(keys, k)
		}
		sort.Strings(keys)
		for _, tag := range keys {
			freq := testRes[tag]
			s := " > TEST " + tag
			l.Printf("%-18s: % 7d", s, freq)
		}
	}
}
<|endoftext|>


<commit_before>package cmd

import (
	"github.com/veino/processors/filter-date"
	"github.com/veino/processors/filter-drop"
	"github.com/veino/processors/filter-geoip"
	"github.com/veino/processors/filter-grok"
	"github.com/veino/processors/filter-json"
	"github.com/veino/processors/filter-mutate"
	"github.com/veino/processors/filter-split"
	"github.com/veino/processors/filter-uuid"
	"github.com/veino/processors/input-beats"
	"github.com/veino/processors/input-exec"
	"github.com/veino/processors/input-file"
	"github.com/veino/processors/input-rabbitmq"
	"github.com/veino/processors/input-stdin"
	"github.com/veino/processors/input-syslog"
	"github.com/veino/processors/input-twitter"
	"github.com/veino/processors/input-udp"
	"github.com/veino/processors/output-elasticsearch"
	"github.com/veino/processors/output-elasticsearch2"
	"github.com/veino/processors/output-file"
	"github.com/veino/processors/output-glusterfs"
	"github.com/veino/processors/output-mongodb"
	"github.com/veino/processors/output-null"
	"github.com/veino/processors/output-rabbitmq"
	"github.com/veino/processors/output-stdout"
	"github.com/veino/processors/when"
	"github.com/veino/runtime"
)

func init() {

	runtime.RegisterProcessor("input_stdin", stdin.New)
	runtime.RegisterProcessor("input_twitter", twitter.New)
	runtime.RegisterProcessor("input_file", fileinput.New)
	runtime.RegisterProcessor("input_exec", execinput.New)
	runtime.RegisterProcessor("input_beats", beatsinput.New)
	runtime.RegisterProcessor("input_rabbitmq", rabbitmqinput.New)
	runtime.RegisterProcessor("input_udp", udpinput.New)
	runtime.RegisterProcessor("input_syslog", sysloginput.New)

	runtime.RegisterProcessor("grok", grok.New)
	runtime.RegisterProcessor("mutate", mutate.New)
	runtime.RegisterProcessor("split", split.New)
	runtime.RegisterProcessor("date", date.New)
	runtime.RegisterProcessor("json", json.New)
	runtime.RegisterProcessor("uuid", uuid.New)
	runtime.RegisterProcessor("drop", drop.New)
	runtime.RegisterProcessor("geoip", geoip.New)

	runtime.RegisterProcessor("output_stdout", stdout.New)
	runtime.RegisterProcessor("output_mongodb", mongodb.New)
	runtime.RegisterProcessor("output_null", null.New)
	runtime.RegisterProcessor("output_elasticsearch", elasticsearch.New)
	runtime.RegisterProcessor("output_elasticsearch2", elasticsearch2.New)
	runtime.RegisterProcessor("output_file", fileoutput.New)
	runtime.RegisterProcessor("output_glusterfs", glusterfsoutput.New)
	runtime.RegisterProcessor("output_rabbitmq", rabbitmqoutput.New)

	runtime.RegisterProcessor("when", when.New)
	runtime.RegisterProcessor("output_when", when.New)
}
<commit_msg>adding unix input plugin<commit_after>package cmd

import (
	"github.com/veino/processors/filter-date"
	"github.com/veino/processors/filter-drop"
	"github.com/veino/processors/filter-geoip"
	"github.com/veino/processors/filter-grok"
	"github.com/veino/processors/filter-json"
	"github.com/veino/processors/filter-mutate"
	"github.com/veino/processors/filter-split"
	"github.com/veino/processors/filter-uuid"
	"github.com/veino/processors/input-beats"
	"github.com/veino/processors/input-exec"
	"github.com/veino/processors/input-file"
	"github.com/veino/processors/input-rabbitmq"
	"github.com/veino/processors/input-stdin"
	"github.com/veino/processors/input-syslog"
	"github.com/veino/processors/input-twitter"
	"github.com/veino/processors/input-udp"
	"github.com/veino/processors/input-unix"
	"github.com/veino/processors/output-elasticsearch"
	"github.com/veino/processors/output-elasticsearch2"
	"github.com/veino/processors/output-file"
	"github.com/veino/processors/output-glusterfs"
	"github.com/veino/processors/output-mongodb"
	"github.com/veino/processors/output-null"
	"github.com/veino/processors/output-rabbitmq"
	"github.com/veino/processors/output-stdout"
	"github.com/veino/processors/when"
	"github.com/veino/runtime"
)

func init() {

	runtime.RegisterProcessor("input_stdin", stdin.New)
	runtime.RegisterProcessor("input_twitter", twitter.New)
	runtime.RegisterProcessor("input_file", fileinput.New)
	runtime.RegisterProcessor("input_exec", execinput.New)
	runtime.RegisterProcessor("input_beats", beatsinput.New)
	runtime.RegisterProcessor("input_rabbitmq", rabbitmqinput.New)
	runtime.RegisterProcessor("input_udp", udpinput.New)
	runtime.RegisterProcessor("input_syslog", sysloginput.New)
	runtime.RegisterProcessor("input_unix", unixinput.New)

	runtime.RegisterProcessor("grok", grok.New)
	runtime.RegisterProcessor("mutate", mutate.New)
	runtime.RegisterProcessor("split", split.New)
	runtime.RegisterProcessor("date", date.New)
	runtime.RegisterProcessor("json", json.New)
	runtime.RegisterProcessor("uuid", uuid.New)
	runtime.RegisterProcessor("drop", drop.New)
	runtime.RegisterProcessor("geoip", geoip.New)

	runtime.RegisterProcessor("output_stdout", stdout.New)
	runtime.RegisterProcessor("output_mongodb", mongodb.New)
	runtime.RegisterProcessor("output_null", null.New)
	runtime.RegisterProcessor("output_elasticsearch", elasticsearch.New)
	runtime.RegisterProcessor("output_elasticsearch2", elasticsearch2.New)
	runtime.RegisterProcessor("output_file", fileoutput.New)
	runtime.RegisterProcessor("output_glusterfs", glusterfsoutput.New)
	runtime.RegisterProcessor("output_rabbitmq", rabbitmqoutput.New)

	runtime.RegisterProcessor("when", when.New)
	runtime.RegisterProcessor("output_when", when.New)
}
<|endoftext|>


<commit_before>package cmd

import (
	"strconv"
	"strings"
	"time"

	"github.com/mobingi/mobingi-cli/client/timeout"
	"github.com/mobingi/mobingi-cli/pkg/cli"
	"github.com/mobingi/mobingi-cli/pkg/cli/confmap"
	"github.com/mobingilabs/mobingi-sdk-go/client"
	"github.com/mobingilabs/mobingi-sdk-go/mobingi/session"
	d "github.com/mobingilabs/mobingi-sdk-go/pkg/debug"
	"github.com/mobingilabs/mobingi-sdk-go/pkg/nativestore"
	"github.com/pkg/errors"
	"github.com/spf13/viper"
)

func getApiVersionInt() int {
	v := 3
	vparam := viper.GetString(confmap.ConfigKey("apiver"))
	in, err := strconv.Atoi(strings.TrimLeft(vparam, "v"))
	if err != nil {
		return -1
	}

	v = in
	return v
}

func clisession() (*session.Session, error) {
	v := getApiVersionInt()
	if v < 0 {
		return nil, errors.New("cannot get api version")
	}

	// check if we have credentials in nativestore
	user, secret, err := nativestore.Get(cli.CliUrl)
	if err == nil {
		if user != "" && secret != "" {
			if cli.Verbose {
				d.Info("use credentials from native store")
			}

			return session.New(&session.Config{
				ClientId:        user,
				ClientSecret:    secret,
				ApiVersion:      v,
				BaseApiUrl:      viper.GetString(confmap.ConfigKey("url")),
				BaseRegistryUrl: viper.GetString(confmap.ConfigKey("rurl")),
				HttpClientConfig: &client.Config{
					Timeout: time.Second * time.Duration(timeout.Timeout),
					Verbose: cli.Verbose,
				},
			})
		}
	}

	if cli.Verbose {
		if cli.Debug {
			d.Error(err)
		}

		d.Info("cannot access native store, use config file token")
	}

	return session.New(&session.Config{
		ApiVersion:      v,
		AccessToken:     viper.GetString(confmap.ConfigKey("token")),
		BaseApiUrl:      viper.GetString(confmap.ConfigKey("url")),
		BaseRegistryUrl: viper.GetString(confmap.ConfigKey("rurl")),
		HttpClientConfig: &client.Config{
			Timeout: time.Second * time.Duration(timeout.Timeout),
			Verbose: cli.Verbose,
		},
	})
}
<commit_msg>Add verbose in session.<commit_after>package cmd

import (
	"strconv"
	"strings"
	"time"

	"github.com/mobingi/mobingi-cli/client/timeout"
	"github.com/mobingi/mobingi-cli/pkg/cli"
	"github.com/mobingi/mobingi-cli/pkg/cli/confmap"
	"github.com/mobingilabs/mobingi-sdk-go/client"
	"github.com/mobingilabs/mobingi-sdk-go/mobingi/session"
	d "github.com/mobingilabs/mobingi-sdk-go/pkg/debug"
	"github.com/mobingilabs/mobingi-sdk-go/pkg/nativestore"
	"github.com/pkg/errors"
	"github.com/spf13/viper"
)

func getApiVersionInt() int {
	v := 3
	vparam := viper.GetString(confmap.ConfigKey("apiver"))
	in, err := strconv.Atoi(strings.TrimLeft(vparam, "v"))
	if err != nil {
		return -1
	}

	v = in
	return v
}

func clisession() (*session.Session, error) {
	v := getApiVersionInt()
	if v < 0 {
		return nil, errors.New("cannot get api version")
	}

	// check if we have credentials in nativestore
	user, secret, err := nativestore.Get(cli.CliUrl)
	if err == nil {
		if user != "" && secret != "" {
			if cli.Verbose {
				d.Info("use credentials from native store")
			}

			return session.New(&session.Config{
				ClientId:        user,
				ClientSecret:    secret,
				ApiVersion:      v,
				BaseApiUrl:      viper.GetString(confmap.ConfigKey("url")),
				BaseRegistryUrl: viper.GetString(confmap.ConfigKey("rurl")),
				HttpClientConfig: &client.Config{
					Timeout: time.Second * time.Duration(timeout.Timeout),
					Verbose: cli.Verbose,
				},
			})
		}
	}

	if cli.Verbose {
		d.ErrorD(err)
		d.Info("cannot access native store, use config file token")
	}

	return session.New(&session.Config{
		ApiVersion:      v,
		AccessToken:     viper.GetString(confmap.ConfigKey("token")),
		BaseApiUrl:      viper.GetString(confmap.ConfigKey("url")),
		BaseRegistryUrl: viper.GetString(confmap.ConfigKey("rurl")),
		HttpClientConfig: &client.Config{
			Timeout: time.Second * time.Duration(timeout.Timeout),
			Verbose: cli.Verbose,
		},
	})
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"

	"github.com/spf13/cobra"
)

const Version = "0.2.2"

// versionCmd represents the version command
var versionCmd = &cobra.Command{
	Use:     "version",
	Aliases: []string{"v"},
	Short:   "Print the version of this tool",
	Run: func(cmd *cobra.Command, args []string) {
		fmt.Printf("gitlab-cli %v\n", Version)
	},
}

func init() {
	RootCmd.AddCommand(versionCmd)
}
<commit_msg>Update version to 0.3.0<commit_after>package cmd

import (
	"fmt"

	"github.com/spf13/cobra"
)

const Version = "0.3.0"

// versionCmd represents the version command
var versionCmd = &cobra.Command{
	Use:     "version",
	Aliases: []string{"v"},
	Short:   "Print the version of this tool",
	Run: func(cmd *cobra.Command, args []string) {
		fmt.Printf("gitlab-cli %v\n", Version)
	},
}

func init() {
	RootCmd.AddCommand(versionCmd)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"

	"github.com/gonuts/commander"
	"github.com/gonuts/flag"
	//gocfg "github.com/sbinet/go-config/config"
)

func hwaf_make_cmd_pkg_add() *commander.Command {
	cmd := &commander.Command{
		Run:       hwaf_run_cmd_pkg_add,
		UsageLine: "co [options] <pkg-uri> [<local-pkg-name>]",
		Short:     "add a package to the current workarea",
		Long: `
co adds a package to the current workarea.

ex:
 $ hwaf pkg co /foo/pkg
 $ hwaf pkg co Control/AthenaKernel
 $ hwaf pkg co git://github.com/mana-fwk/mana-core-athenakernel
 $ hwaf pkg co git://github.com/mana-fwk/mana-core-athenakernel Control/AthenaKernel
`,
		Flag: *flag.NewFlagSet("hwaf-pkg-co", flag.ExitOnError),
	}
	cmd.Flag.Bool("q", false, "only print error and warning messages, all other output will be suppressed")

	return cmd
}

func hwaf_run_cmd_pkg_add(cmd *commander.Command, args []string) {
	var err error
	n := "hwaf-pkg-" + cmd.Name()
	pkguri := ""
	pkgname := ""
	switch len(args) {
	case 1:
		pkguri = args[0]
		pkgname = filepath.Base(args[0])
	case 2:
		pkguri = args[0]
		pkgname = args[1]
	default:
		err = fmt.Errorf("%s: you need to give a package URL", n)
		handle_err(err)
	}

	pkguri = os.ExpandEnv(pkguri)
	//pkguri = filepath.Clean(pkguri)

	pkgname = os.ExpandEnv(pkgname)
	pkgname = filepath.Clean(pkgname)

	quiet := cmd.Flag.Lookup("q").Value.Get().(bool)

	if !quiet {
		fmt.Printf("%s: checkout package [%s]...\n", n, pkguri)
	}

	cfg := load_local_cfg()
	pkgdir := "src"
	if cfg.HasOption("hwaf-cfg", "cmtpkgs") {
		pkgdir, err = cfg.String("hwaf-cfg", "cmtpkgs")
		handle_err(err)
	}

	git := exec.Command("git", "submodule", "add",
		pkguri,
		filepath.Join(pkgdir, pkgname),
	)
	if !quiet {
		git.Stdout = os.Stdout
		git.Stderr = os.Stderr
	}
	err = git.Run()
	handle_err(err)

	git = exec.Command(
		"git", "submodule", "update",
		"--init", "--recursive",
	)
	if !quiet {
		git.Stdout = os.Stdout
		git.Stderr = os.Stderr
	}
	err = git.Run()
	handle_err(err)

	git = exec.Command(
		"git", "commit", "-m",
		fmt.Sprintf("adding package [%s]", pkgname),
	)
	if !quiet {
		git.Stdout = os.Stdout
		git.Stderr = os.Stderr
	}
	err = git.Run()
	handle_err(err)

	if !quiet {
		fmt.Printf("%s: checkout package [%s]... [ok]\n", n, pkguri)
	}
}

// EOF
<commit_msg>pkg-co: preliminary support for retrieving svn packages<commit_after>package main

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/gonuts/commander"
	"github.com/gonuts/flag"
	//gocfg "github.com/sbinet/go-config/config"
)

func hwaf_make_cmd_pkg_add() *commander.Command {
	cmd := &commander.Command{
		Run:       hwaf_run_cmd_pkg_add,
		UsageLine: "co [options] <pkg-uri> [<local-pkg-name>]",
		Short:     "add a package to the current workarea",
		Long: `
co adds a package to the current workarea.

ex:
 $ hwaf pkg co /foo/pkg
 $ hwaf pkg co Control/AthenaKernel
 $ hwaf pkg co git://github.com/mana-fwk/mana-core-athenakernel
 $ hwaf pkg co git://github.com/mana-fwk/mana-core-athenakernel Control/AthenaKernel
 $ hwaf pkg co -b=rel/mana git://github.com/mana-fwk/mana-core-athenakernel Control/AthenaKernel
 $ hwaf pkg co -b=AthenaKernel-00-00-01 svn+ssh://svn.cern.ch/reps/atlasoff/Control/AthenaKernel Control/AthenaKernel
`,
		Flag: *flag.NewFlagSet("hwaf-pkg-co", flag.ExitOnError),
	}
	cmd.Flag.Bool("q", false, "only print error and warning messages, all other output will be suppressed")
	cmd.Flag.String("b", "", "branch to checkout (default=master)")

	return cmd
}

func hwaf_run_cmd_pkg_add(cmd *commander.Command, args []string) {
	var err error
	n := "hwaf-pkg-" + cmd.Name()
	pkguri := ""
	pkgname := ""
	switch len(args) {
	case 1:
		pkguri = args[0]
		pkgname = filepath.Base(args[0])
	case 2:
		pkguri = args[0]
		pkgname = args[1]
	default:
		err = fmt.Errorf("%s: you need to give a package URL", n)
		handle_err(err)
	}

	pkguri = os.ExpandEnv(pkguri)
	//pkguri = filepath.Clean(pkguri)

	pkgname = os.ExpandEnv(pkgname)
	pkgname = filepath.Clean(pkgname)

	quiet := cmd.Flag.Lookup("q").Value.Get().(bool)
	bname := cmd.Flag.Lookup("b").Value.Get().(string)

	if !quiet {
		fmt.Printf("%s: checkout package [%s]...\n", n, pkguri)
	}

	cfg := load_local_cfg()
	pkgdir := "src"
	if cfg.HasOption("hwaf-cfg", "cmtpkgs") {
		pkgdir, err = cfg.String("hwaf-cfg", "cmtpkgs")
		handle_err(err)
	}

	if strings.HasPrefix(pkguri, "svn+ssh:/") {
		if !quiet {
			fmt.Printf("%s: svn repo. doing staging...\n", n)
		}
		staging := filepath.Join(".git", "hwaf-svn-staging")
		if !path_exists(staging) {
			err = os.MkdirAll(staging, 0700)
			handle_err(err)
		}
		_ = os.RemoveAll(filepath.Join(staging, pkgname))
		err = os.MkdirAll(filepath.Join(staging, pkgname), 0700)
		handle_err(err)
		git := exec.Command(
			"go-svn2git", "-verbose", "-revision=1", pkguri,
		)
		git.Dir = filepath.Join(staging, pkgname)
		err = git.Run()
		handle_err(err)

		pkguri, err = filepath.Abs(filepath.Join(staging, pkgname))
		handle_err(err)
		if !quiet {
			fmt.Printf("%s: svn repo. doing staging... [ok]\n", n)
		}
	}
	git := exec.Command(
		"git", "submodule", "add",
		pkguri, filepath.Join(pkgdir, pkgname),
		)
	if !quiet {
		git.Stdout = os.Stdout
		git.Stderr = os.Stderr
	}
	err = git.Run()
	handle_err(err)

	git = exec.Command(
		"git", "submodule", "update",
		"--init", "--recursive",
	)
	if !quiet {
		git.Stdout = os.Stdout
		git.Stderr = os.Stderr
	}
	err = git.Run()
	handle_err(err)

	if bname != "" {
		git = exec.Command(
			"git", "checkout", bname,
			)
		git.Dir = filepath.Join(pkgdir, pkgname)
		err = git.Run()
		handle_err(err)
	}

	git = exec.Command(
		"git", "commit", "-m",
		fmt.Sprintf("adding package [%s]", pkgname),
	)
	if !quiet {
		git.Stdout = os.Stdout
		git.Stderr = os.Stderr
	}
	err = git.Run()
	handle_err(err)

	if !quiet {
		fmt.Printf("%s: checkout package [%s]... [ok]\n", n, pkguri)
	}
}

// EOF
<|endoftext|>


<commit_before>package coco

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
)

/*
Example configuration file.
file format: json

ex.json
{
	hosts: ["host1", "host2", "host3"],
	tree: {host1:{
		    host2:{},
			host3:{}}
		  }
}

This will be compiled into the specified tree structure with HostNode Hosts and goPeer Peers.
*/

// HostConfig stores all of the relevant information of the configuration file.
type HostConfig struct {
	Hosts map[string]*HostNode // maps hostname to host
	Dir   *directory           // the directory mapping hostnames to goPeers
	Root  *HostNode            // the host root of the tree
}

// NewHostConfig creates a new host configuration that can be populated with
// hosts.
func NewHostConfig() *HostConfig {
	return &HostConfig{Hosts: make(map[string]*HostNode), Dir: newDirectory(), Root: nil}
}

// ConstructTree does a depth-first construction of the tree specified in the
// config file. It detects unknown hosts but does not detect the error of
// multiple root nodes, rather it silently choses just one. ConstructTree must
// be call AFTER populating the HostConfig with ALL the possible hosts.
func ConstructTree(tree map[string]interface{}, hc *HostConfig, parent Conn) error {
	// each k will be a sibling in the tree
	// for each sibling add its children
	for k, subtree := range tree {
		if _, ok := hc.Hosts[k]; !ok {
			fmt.Println("unknown host in tree:", k)
			return errors.New("unknown host in tree")
		}
		h := hc.Hosts[k]
		if parent == nil {
			fmt.Printf("root node is %v\n", h.name)
			hc.Root = h
		}
		h.parent = parent

		children := subtree.(map[string]interface{})
		for _ = range children {
			// ignore the error because we don't care if we have already
			// constructed this peer before.
			//gp, _ := NewGoPeer(hc.Dir, child)
			//h.AddChildren(gp)
			//fmt.Printf("added %v as child of %v\n", gp.hostname, h.name)
		}
		if err := ConstructTree(children, hc, hc.Dir.nameToPeer[h.name]); err != nil {
			return err
		}
	}
	return nil
}

// LoadConfig loads a configuration file in the format specified above. It
// populates a HostConfig with HostNode Hosts and goPeer Peers.
func LoadConfig(fname string) (*HostConfig, error) {
	hc := NewHostConfig()
	file, err := ioutil.ReadFile(fname)
	if err != nil {
		return hc, err
	}
	var m map[string]interface{}
	json.Unmarshal(file, &m)
	// read the hosts lists
	hnames := m["hosts"].([]string)
	for _, h := range hnames {
		// add to the hosts list if we havent added it before
		if _, ok := hc.Hosts[h]; !ok {
			hc.Hosts[h] = NewHostNode(h)
		}
	}
	tree := m["tree"].(map[string]interface{})
	err = ConstructTree(tree, hc, nil)
	// construct the host tree
	return hc, err
}
<commit_msg>changing around config.go to reflect new Conn<commit_after>package coco

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
)

/*
Example configuration file.
file format: json

ex.json
{
	hosts: ["host1", "host2", "host3"],
	tree: {host1:{
		    host2:{},
			host3:{}}
		  }
}

This will be compiled into the specified tree structure with HostNode Hosts and goPeer Peers.
*/

// HostConfig stores all of the relevant information of the configuration file.
type HostConfig struct {
	Hosts map[string]*HostNode // maps hostname to host
	Dir   *directory           // the directory mapping hostnames to goPeers
	Root  *HostNode            // the host root of the tree
}

// NewHostConfig creates a new host configuration that can be populated with
// hosts.
func NewHostConfig() *HostConfig {
	return &HostConfig{Hosts: make(map[string]*HostNode), Dir: newDirectory(), Root: nil}
}

// ConstructTree does a depth-first construction of the tree specified in the
// config file. It detects unknown hosts but does not detect the error of
// multiple root nodes, rather it silently choses just one. ConstructTree must
// be call AFTER populating the HostConfig with ALL the possible hosts.
func ConstructTree(tree map[string]interface{}, hc *HostConfig, parent *HostNode) error {
	// each k will be a sibling in the tree
	// for each sibling add its children
	for k, subtree := range tree {
		if _, ok := hc.Hosts[k]; !ok {
			fmt.Println("unknown host in tree:", k)
			return errors.New("unknown host in tree")
		}
		h := hc.Hosts[k]
		// add connection from host to its parent
		if parent == nil {
			fmt.Printf("root node is %v\n", h.name)
			hc.Root = h
		} else {
			gc, _ := NewGoConn(hc.Dir, h.name, parent.name)
			h.AddParent(gc)
		}

		children := subtree.(map[string]interface{})
		// add connections from parent to children
		for child := range children {
			// ignore the error because we don't care if we have already
			// constructed this peer before.
			gc, _ := NewGoConn(hc.Dir, h.name, child)
			h.AddChildren(gc)
			fmt.Printf("added %v as child of %v\n", gc.to, h.name)
		}
		if err := ConstructTree(children, hc, h); err != nil {
			return err
		}
	}
	return nil
}

// LoadConfig loads a configuration file in the format specified above. It
// populates a HostConfig with HostNode Hosts and goPeer Peers.
func LoadConfig(fname string) (*HostConfig, error) {
	hc := NewHostConfig()
	file, err := ioutil.ReadFile(fname)
	if err != nil {
		return hc, err
	}
	var m map[string]interface{}
	json.Unmarshal(file, &m)
	// read the hosts lists
	hnames := m["hosts"].([]string)
	for _, h := range hnames {
		// add to the hosts list if we havent added it before
		if _, ok := hc.Hosts[h]; !ok {
			hc.Hosts[h] = NewHostNode(h)
		}
	}
	tree := m["tree"].(map[string]interface{})
	err = ConstructTree(tree, hc, nil)
	// construct the host tree
	return hc, err
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"log"
	"math/rand"
	"net"
	"net/http"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"
)

var db = struct {
	sync.RWMutex
	m map[string]int64
}{m: make(map[string]int64)}

var header string = "\xff\xff\xff\xff"

func main() {
	go listenMaster(":20810")
	go listenAuth(":20800")
	go purge(2)
	http.HandleFunc("/getinfo", getinfoHandler)
	http.ListenAndServe(":8080", nil)
	select {}
}

func listenMaster(port string) {
	udpAddr, err := net.ResolveUDPAddr("udp", port)
	if err != nil {
		log.Fatal("Could not resolve port for master.")
	}

	conn, err := net.ListenUDP("udp", udpAddr)
	if err != nil {
		log.Fatal("Failed to open socket for master.")
	}

	defer conn.Close()

	fmt.Println("Master server is listening...")
	for {
		var buf [1024]byte
		n, addr, err := conn.ReadFromUDP(buf[0:])
		if err != nil {
			return
		}
		msg := string(buf[0:n])
		endpoint := string(addr.IP.String()) + ":" + fmt.Sprint(addr.Port)

		if strings.HasPrefix(msg[4:], "statusResponse") {
			db.Lock()
			db.m[endpoint] = time.Now().Unix()
			db.Unlock()
		} else {
			switch msg[4:] {
			case "getservers 6 full empty":
				db.RLock()
				if len(db.m) == 0 {
					// db is empty
					db.RUnlock()
					continue
				}

				res := make([]byte, 0)
				innerCount := 0
				per := 20

				for k, _ := range db.m {
					current := make([]byte, 0)
					if innerCount == 0 {
						current = append(current, []byte(fmt.Sprint(header, "getserversResponse", "\n\x00\\"))...)
					}

					octets := strings.Split(k[:strings.Index(k, ":")], ".")
					for i := 0; i < 4; i++ {
						octet, err := strconv.Atoi(octets[i])
						if err != nil {
							continue
						}
						current = append(current, byte(octet))
					}

					addrport, err := strconv.Atoi(k[strings.Index(k, ":"):])
					if err != nil {
						continue
					}
					portbuf := &bytes.Buffer{}
					binary.Write(portbuf, binary.BigEndian, uint16(addrport))

					current = append(current, portbuf.Bytes()...)
					current = append(current, []byte("\\")...)

					innerCount++
					if innerCount == per {
						current = append(current, []byte("EOT")...)
						res = append(res, current...)
						conn.WriteToUDP(res, addr)

						// reset
						innerCount = 0
						res = make([]byte, 0)
					}
				}
				db.RUnlock()
				res = append(res, []byte("EOF")...)
				conn.WriteToUDP(res, addr)

			case "heartbeat COD-4\n": //server checking in to MS
				db.Lock()
				if _, ok := db.m[endpoint]; ok {
					// just checking in
					db.m[endpoint] = time.Now().Unix()
				} else {
					// new server
					nonce := generateNonce(9)
					conn.WriteToUDP([]byte(fmt.Sprint(header, "getchallenge ", nonce, "\n")), addr)
					conn.WriteToUDP([]byte(fmt.Sprint(header, "getstatus ", nonce, "\n")), addr)
				}
				db.Unlock()
			case "heartbeat flatline":
				db.Lock()
				delete(db.m, endpoint)
				db.Unlock()
			}
		}
	}
}

// generateNonce creates a pseudorandom number.
// digits determines how long the number will be.
func generateNonce(digits int) string {
	nonce := &bytes.Buffer{}
	for i := 0; i < digits; i++ {
		nonce.WriteString(strconv.Itoa(rand.Intn(10)))
	}

	return nonce.String()
}

// listenAuth mimics the authentication server
// Packets are received but not acted upon.
func listenAuth(port string) {
	udpAddr, err := net.ResolveUDPAddr("udp", port)
	if err != nil {
		log.Fatal("Could not resolve port for auth.")
	}

	conn, err := net.ListenUDP("udp", udpAddr)
	if err != nil {
		log.Fatal("Failed to open socket for auth.")
	}

	defer conn.Close()

	fmt.Println("Authentication server is listening...")
	for {
		var buf [1024]byte
		n, _, err := conn.ReadFromUDP(buf[0:])
		if err != nil {
			return
		}

		msg := string(buf[0:n])

		if strings.HasPrefix(msg[4:], "getIpAuthorize") {
			// ignore
		}
	}
}

// purge removes inactive game servers from the database.
// A server is inactive if it has failed to send a hearbeat
// within the timeframe specified by interval in minutes.
func purge(interval int) {
	for {
		current := time.Now().Unix() - int64(interval*60)
		db.Lock()
		for k, v := range db.m {
			if v < current {
				delete(db.m, k)
			}
		}
		db.Unlock()
		time.Sleep(time.Duration(interval) * time.Minute)
	}
}

// getInfo queries the specified server.
func getInfo(addr string) (map[string]string, error) {
	udpAddr, err := net.ResolveUDPAddr("udp", addr)
	if err != nil {
		return nil, err
	}

	conn, err := net.DialUDP("udp", nil, udpAddr)
	if err != nil {
		return nil, err
	}

	defer conn.Close()

	conn.Write([]byte("\xff\xff\xff\xffgetinfo xxx"))

	var buf [512]byte
	conn.SetReadDeadline(time.Now().Add(10000 * time.Millisecond))
	n, err := conn.Read(buf[0:])
	if err != nil {
		return nil, err
	}

	response := string(buf[0:n])
	fields := strings.Split(response[strings.Index(response, "\\")+1:], "\\")
	ir := make(map[string]string)
	for i := 0; i < len(fields)-1; i = i + 2 {
		ir[fields[i]] = fields[i+1]
	}

	return ir, nil
}

// removeColorCodes returns a copy of string s without Quake color codes.
func removeColorCodes(s string) string {
	re := regexp.MustCompile("\\^[0-7]")
	return re.ReplaceAllString(s, "")
}

func getinfoHandler(w http.ResponseWriter, r *http.Request) {
	response, err := getInfo(r.FormValue("addr"))

	if err != nil {
		fmt.Fprint(w, err)
		return
	}

	fmt.Fprint(w, response)
}
<commit_msg>fixes regression<commit_after>package main

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"log"
	"math/rand"
	"net"
	"net/http"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"
)

var db = struct {
	sync.RWMutex
	m map[string]int64
}{m: make(map[string]int64)}

var header string = "\xff\xff\xff\xff"

func main() {
	go listenMaster(":20810")
	go listenAuth(":20800")
	go purge(6)
	http.HandleFunc("/getinfo", getinfoHandler)
	http.ListenAndServe(":8080", nil)
	select {}
}

func listenMaster(port string) {
	udpAddr, err := net.ResolveUDPAddr("udp", port)
	if err != nil {
		log.Fatal("Could not resolve port for master.")
	}

	conn, err := net.ListenUDP("udp", udpAddr)
	if err != nil {
		log.Fatal("Failed to open socket for master.")
	}

	defer conn.Close()

	fmt.Println("Master server is listening...")
	for {
		var buf [1024]byte
		n, addr, err := conn.ReadFromUDP(buf[0:])
		if err != nil {
			return
		}
		msg := string(buf[0:n])
		endpoint := string(addr.IP.String()) + ":" + fmt.Sprint(addr.Port)

		if strings.HasPrefix(msg[4:], "statusResponse") {
			db.Lock()
			db.m[endpoint] = time.Now().Unix()
			db.Unlock()
		} else {
			switch msg[4:] {
			case "getservers 6 full empty":
				db.RLock()
				if len(db.m) == 0 {
					// db is empty
					db.RUnlock()
					continue
				}

				res := make([]byte, 0)
				innerCount := 0
				per := 20

				for k, _ := range db.m {
					current := make([]byte, 0)
					if innerCount == 0 {
						current = append(current, []byte(fmt.Sprint(header, "getserversResponse", "\n\x00\\"))...)
					}

					octets := strings.Split(k[:strings.Index(k, ":")], ".")
					for i := 0; i < 4; i++ {
						octet, err := strconv.Atoi(octets[i])
						if err != nil {
							continue
						}
						current = append(current, byte(octet))
					}

					addrport, err := strconv.Atoi(k[strings.Index(k, ":")+1:])
					if err != nil {
						continue
					}
					portbuf := &bytes.Buffer{}
					binary.Write(portbuf, binary.BigEndian, uint16(addrport))

					current = append(current, portbuf.Bytes()...)
					current = append(current, []byte("\\")...)

					res = append(res, current...)

					innerCount++
					if innerCount == per {
						res = append(res, []byte("EOT")...)
						conn.WriteToUDP(res, addr)

						// reset
						innerCount = 0
						res = make([]byte, 0)
					}
				}
				db.RUnlock()
				res = append(res, []byte("EOF")...)
				conn.WriteToUDP(res, addr)

			case "heartbeat COD-4\n": //server checking in to MS
				db.Lock()
				if _, ok := db.m[endpoint]; ok {
					// just checking in
					db.m[endpoint] = time.Now().Unix()
				} else {
					// new server
					nonce := generateNonce(9)
					conn.WriteToUDP([]byte(fmt.Sprint(header, "getchallenge ", nonce, "\n")), addr)
					conn.WriteToUDP([]byte(fmt.Sprint(header, "getstatus ", nonce, "\n")), addr)
				}
				db.Unlock()
			case "heartbeat flatline":
				db.Lock()
				delete(db.m, endpoint)
				db.Unlock()
			}
		}
	}
}

// generateNonce creates a pseudorandom number.
// digits determines how long the number will be.
func generateNonce(digits int) string {
	nonce := &bytes.Buffer{}
	for i := 0; i < digits; i++ {
		nonce.WriteString(strconv.Itoa(rand.Intn(10)))
	}

	return nonce.String()
}

// listenAuth mimics the authentication server
// Packets are received but not acted upon.
func listenAuth(port string) {
	udpAddr, err := net.ResolveUDPAddr("udp", port)
	if err != nil {
		log.Fatal("Could not resolve port for auth.")
	}

	conn, err := net.ListenUDP("udp", udpAddr)
	if err != nil {
		log.Fatal("Failed to open socket for auth.")
	}

	defer conn.Close()

	fmt.Println("Authentication server is listening...")
	for {
		var buf [1024]byte
		n, _, err := conn.ReadFromUDP(buf[0:])
		if err != nil {
			return
		}

		msg := string(buf[0:n])

		if strings.HasPrefix(msg[4:], "getIpAuthorize") {
			// ignore
		}
	}
}

// purge removes inactive game servers from the database.
// A server is inactive if it has failed to send a hearbeat
// within the timeframe specified by interval in minutes.
func purge(interval int) {
	for {
		current := time.Now().Unix() - int64(interval*60)
		db.Lock()
		for k, v := range db.m {
			if v < current {
				delete(db.m, k)
			}
		}
		db.Unlock()
		time.Sleep(time.Duration(interval) * time.Minute)
	}
}

// getInfo queries the specified server.
func getInfo(addr string) (map[string]string, error) {
	udpAddr, err := net.ResolveUDPAddr("udp", addr)
	if err != nil {
		return nil, err
	}

	conn, err := net.DialUDP("udp", nil, udpAddr)
	if err != nil {
		return nil, err
	}

	defer conn.Close()

	conn.Write([]byte("\xff\xff\xff\xffgetinfo xxx"))

	var buf [512]byte
	conn.SetReadDeadline(time.Now().Add(10000 * time.Millisecond))
	n, err := conn.Read(buf[0:])
	if err != nil {
		return nil, err
	}

	response := string(buf[0:n])
	fields := strings.Split(response[strings.Index(response, "\\")+1:], "\\")
	ir := make(map[string]string)
	for i := 0; i < len(fields)-1; i = i + 2 {
		ir[fields[i]] = fields[i+1]
	}

	return ir, nil
}

// removeColorCodes returns a copy of string s without Quake color codes.
func removeColorCodes(s string) string {
	re := regexp.MustCompile("\\^[0-7]")
	return re.ReplaceAllString(s, "")
}

func getinfoHandler(w http.ResponseWriter, r *http.Request) {
	response, err := getInfo(r.FormValue("addr"))

	if err != nil {
		fmt.Fprint(w, err)
		return
	}

	fmt.Fprint(w, response)
}
<|endoftext|>


<commit_before>package command

import (
	"strings"

	"github.com/mitchellh/cli"
)

// NewCommand is a Command that creates a new empty YAML file to prepare a new
// issue.
type NewCommand struct {
	UI cli.Ui
}

// Run runs the code of the comand.
func (c *NewCommand) Run(args []string) int {
	return 0
}

// Help returns the description of the command.
func (*NewCommand) Help() string {
	helpText := `
Usage: msw new [options] ISSUE_NUMBER

  This command creates a new empty YAML file to prepare a new issue.

Options:

  -date=<date>			The date of the issue.

  -directory=path		The directory where to write the generated file.

`
	return strings.TrimSpace(helpText)
}

// Synopsis returns the short description of the command.
func (*NewCommand) Synopsis() string {
	return "create a new empty YAML file to prepare a new issue."
}
<commit_msg>[command/new] Handle date and directory options + generate YAML file<commit_after>package command

import (
	"bytes"
	"flag"
	"fmt"
	"html/template"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/mitchellh/cli"
)

// NewCommand is a Command that creates a new empty YAML file to prepare a new
// issue.
type NewCommand struct {
	UI cli.Ui
}

// Run runs the code of the comand.
func (c *NewCommand) Run(args []string) int {
	var dateArg string
	var directoryArg string

	cmdFlags := flag.NewFlagSet("new", flag.ContinueOnError)
	cmdFlags.Usage = func() { c.UI.Output(c.Help()) }
	cmdFlags.StringVar(&dateArg, "date", "", "date")
	cmdFlags.StringVar(&directoryArg, "directory", "", "directory")

	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	args = cmdFlags.Args()
	if len(args) != 1 {
		cmdFlags.Usage()
		return 1
	}

	number, err := strconv.Atoi(args[0])
	if err != nil {
		c.UI.Error(fmt.Sprintf("Invalid issue number, it must be an integer but got: %s", args[0]))
		return 1
	}

	var date time.Time
	if dateArg != "" {
		d, err := time.Parse("2006-01-02", dateArg)
		if err != nil {
			c.UI.Error(fmt.Sprintf("Error, %s", err))
			return 1
		}
		date = d
	} else {
		// compute next Wednesday
		now := time.Now()
		wd := int(now.Weekday())
		date = now.AddDate(0, 0, 10-wd)
	}

	var directory string
	if directoryArg != "" {
		dir, _ := filepath.Abs(directoryArg)
		directory = dir
	} else {
		dir, err := os.Getwd()
		if err != nil {
			c.UI.Error(fmt.Sprintf("Error while retrieving the current directory: %s", err))
			return 1
		}
		directory = dir
	}

	t, err := template.New("issue").Parse(issueYAML)
	if err != nil {
		c.UI.Error(fmt.Sprintf("Error parsing template: %s", err))
		return 1
	}

	data := struct {
		Number     int
		Categories []string
	}{
		Number: number,
		Categories: []string{
			"Open Science & Data",
			"Tools for Scientists",
			"Cutting-edge Science",
			"Beyond Academia",
		},
	}

	var out bytes.Buffer
	if err = t.Execute(&out, data); err != nil {
		c.UI.Error(fmt.Sprintf("Error generating YAML: %s", err))
		return 1
	}

	filename := path.Join(directory, fmt.Sprintf("%s.yml", date.Format("2006-01-02")))

	if err := ioutil.WriteFile(filename, out.Bytes(), 0644); err != nil {
		c.UI.Error(fmt.Sprintf("Error writing file: %s", err))
		return 1
	}

	c.UI.Output(fmt.Sprintf("Created new issue: %s 👌", filename))

	return 0
}

// Help returns the description of the command.
func (*NewCommand) Help() string {
	helpText := `
Usage: msw new [options] ISSUE_NUMBER

  This command creates a new empty YAML file to prepare a new issue.

Options:

  -date=<date>			The date of the issue. Defaults to next Wednesday.

  -directory=path		The directory where to write the generated file.
						Defaults to current working directory.

`
	return strings.TrimSpace(helpText)
}

// Synopsis returns the short description of the command.
func (*NewCommand) Synopsis() string {
	return "create a new empty YAML file to prepare a new issue."
}

const issueYAML = `# Generated by msw
number: {{ .Number }}
welcome_text: >
  Hi!

categories:
{{- range .Categories }}
  - title: '{{ . }}'
    links:
      - name: ''
        url: ''
        abstract: ''
{{ end }}`
<|endoftext|>


<commit_before>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package cmd

import (
	"fmt"
	"io"
	"strings"

	"github.com/juju/cmd"
	"github.com/juju/errors"

	"github.com/juju/juju/cmd/envcmd"
)

// UploadClient has the API client methods needed by UploadCommand.
type UploadClient interface {
	Upload(service, name string, resource io.Reader) error
	Close() error
}

// UploadDeps is a type that contains external functions that Upload depends on
// to function.
type UploadDeps struct {
	// NewClient returns the value that wraps the API for uploading to the server.
	NewClient func(*UploadCommand) (UploadClient, error)
	// OpenResource handles creating a reader from the resource path.
	OpenResource func(path string) (io.ReadCloser, error)
}

// UploadCommand implements the upload command.
type UploadCommand struct {
	deps UploadDeps
	envcmd.EnvCommandBase
	service       string
	resourceFiles []resourceFile
	resources     map[string]bool
}

// NewUploadCommand returns a new command that lists resources defined
// by a charm.
func NewUploadCommand(deps UploadDeps) *UploadCommand {
	return &UploadCommand{deps: deps}
}

// Info implements cmd.Command.Info
func (c *UploadCommand) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "upload",
		Args:    "service name=file [name2=file2 ...]",
		Purpose: "upload a file as a resource for a service",
		Doc: `
This command uploads a file from your local disk to the juju controller to be
used as a resource for a service.
`,
	}
}

// Init implements cmd.Command.Init. It will return an error satisfying
// errors.BadRequest if you give it an incorrect number of arguments.
func (c *UploadCommand) Init(args []string) error {
	switch len(args) {
	case 0:
		return errors.BadRequestf("missing service name")
	case 1:
		return errors.BadRequestf("no resource specified")
	}

	service := args[0]
	if service == "" { // TODO(ericsnow) names.IsValidService
		return errors.NewNotValid(nil, "missing service name")
	}
	c.service = service

	c.resources = make(map[string]bool)

	for _, arg := range args[1:] {
		if err := c.addResourceFile(arg); err != nil {
			return errors.Trace(err)
		}
	}

	return nil
}

// addResourceFile parses the given arg into a name and a resource file,
// and saves it in c.resourceFiles.
func (c *UploadCommand) addResourceFile(arg string) error {
	name, filename, err := parseResourceFileArg(arg)
	if err != nil {
		return errors.Annotatef(err, "bad resource arg %q", arg)
	}
	rf := resourceFile{
		service:  c.service,
		name:     name,
		filename: filename,
	}

	// TODO(ericsnow) Allow last one to win (a standard CLI approach)?
	if c.resources[rf.name] {
		msg := fmt.Sprintf("duplicate resource %q", rf.name)
		return errors.NewAlreadyExists(nil, msg)
	}
	c.resourceFiles = append(c.resourceFiles, rf)
	c.resources[rf.name] = true
	return nil
}

// Run implements cmd.Command.Run.
func (c *UploadCommand) Run(*cmd.Context) error {
	apiclient, err := c.deps.NewClient(c)
	if err != nil {
		return errors.Annotatef(err, "can't connect to %s", c.ConnectionName())
	}
	defer apiclient.Close()

	errs := []error{}

	for _, rf := range c.resourceFiles {
		// don't want to do a bulk upload since we're doing potentially large
		// file uploads.
		if err := c.upload(rf, apiclient); err != nil {
			name := rf.service + "/" + rf.name
			errs = append(errs, errors.Annotatef(err, "failed to upload resource %q", name))
		}
	}
	switch len(errs) {
	case 0:
		return nil
	case 1:
		return errs[0]
	default:
		msgs := make([]string, len(errs))
		for i := range errs {
			msgs[i] = errs[i].Error()
		}
		return errors.Errorf(strings.Join(msgs, "\n"))
	}
}

// upload opens the given file and calls the apiclient to upload it to the given
// service with the given name.
func (c *UploadCommand) upload(rf resourceFile, client UploadClient) error {
	f, err := c.deps.OpenResource(rf.filename)
	if err != nil {
		return errors.Trace(err)
	}
	defer f.Close()
	err = client.Upload(rf.service, rf.name, f)
	return errors.Trace(err)
}
<commit_msg>Drop a TODO.<commit_after>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package cmd

import (
	"fmt"
	"io"
	"strings"

	"github.com/juju/cmd"
	"github.com/juju/errors"

	"github.com/juju/juju/cmd/envcmd"
)

// UploadClient has the API client methods needed by UploadCommand.
type UploadClient interface {
	Upload(service, name string, resource io.Reader) error
	Close() error
}

// UploadDeps is a type that contains external functions that Upload depends on
// to function.
type UploadDeps struct {
	// NewClient returns the value that wraps the API for uploading to the server.
	NewClient func(*UploadCommand) (UploadClient, error)
	// OpenResource handles creating a reader from the resource path.
	OpenResource func(path string) (io.ReadCloser, error)
}

// UploadCommand implements the upload command.
type UploadCommand struct {
	deps UploadDeps
	envcmd.EnvCommandBase
	service       string
	resourceFiles []resourceFile
	resources     map[string]bool
}

// NewUploadCommand returns a new command that lists resources defined
// by a charm.
func NewUploadCommand(deps UploadDeps) *UploadCommand {
	return &UploadCommand{deps: deps}
}

// Info implements cmd.Command.Info
func (c *UploadCommand) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "upload",
		Args:    "service name=file [name2=file2 ...]",
		Purpose: "upload a file as a resource for a service",
		Doc: `
This command uploads a file from your local disk to the juju controller to be
used as a resource for a service.
`,
	}
}

// Init implements cmd.Command.Init. It will return an error satisfying
// errors.BadRequest if you give it an incorrect number of arguments.
func (c *UploadCommand) Init(args []string) error {
	switch len(args) {
	case 0:
		return errors.BadRequestf("missing service name")
	case 1:
		return errors.BadRequestf("no resource specified")
	}

	service := args[0]
	if service == "" { // TODO(ericsnow) names.IsValidService
		return errors.NewNotValid(nil, "missing service name")
	}
	c.service = service

	c.resources = make(map[string]bool)

	for _, arg := range args[1:] {
		if err := c.addResourceFile(arg); err != nil {
			return errors.Trace(err)
		}
	}

	return nil
}

// addResourceFile parses the given arg into a name and a resource file,
// and saves it in c.resourceFiles.
func (c *UploadCommand) addResourceFile(arg string) error {
	name, filename, err := parseResourceFileArg(arg)
	if err != nil {
		return errors.Annotatef(err, "bad resource arg %q", arg)
	}
	rf := resourceFile{
		service:  c.service,
		name:     name,
		filename: filename,
	}

	if c.resources[rf.name] {
		msg := fmt.Sprintf("duplicate resource %q", rf.name)
		return errors.NewAlreadyExists(nil, msg)
	}
	c.resourceFiles = append(c.resourceFiles, rf)
	c.resources[rf.name] = true
	return nil
}

// Run implements cmd.Command.Run.
func (c *UploadCommand) Run(*cmd.Context) error {
	apiclient, err := c.deps.NewClient(c)
	if err != nil {
		return errors.Annotatef(err, "can't connect to %s", c.ConnectionName())
	}
	defer apiclient.Close()

	errs := []error{}

	for _, rf := range c.resourceFiles {
		// don't want to do a bulk upload since we're doing potentially large
		// file uploads.
		if err := c.upload(rf, apiclient); err != nil {
			name := rf.service + "/" + rf.name
			errs = append(errs, errors.Annotatef(err, "failed to upload resource %q", name))
		}
	}
	switch len(errs) {
	case 0:
		return nil
	case 1:
		return errs[0]
	default:
		msgs := make([]string, len(errs))
		for i := range errs {
			msgs[i] = errs[i].Error()
		}
		return errors.Errorf(strings.Join(msgs, "\n"))
	}
}

// upload opens the given file and calls the apiclient to upload it to the given
// service with the given name.
func (c *UploadCommand) upload(rf resourceFile, client UploadClient) error {
	f, err := c.deps.OpenResource(rf.filename)
	if err != nil {
		return errors.Trace(err)
	}
	defer f.Close()
	err = client.Upload(rf.service, rf.name, f)
	return errors.Trace(err)
}
<|endoftext|>


<commit_before>package resource

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"time"

	"github.com/cloudfoundry-incubator/garden"
	"github.com/tedsuo/ifrit"
)

var ErrAborted = errors.New("script aborted")

const resourceProcessIDPropertyName = "concourse:resource-process"
const resourceResultPropertyName = "concourse:resource-result"

type ErrResourceScriptFailed struct {
	Path       string
	Args       []string
	ExitStatus int

	Stderr string
}

func (err ErrResourceScriptFailed) Error() string {
	msg := fmt.Sprintf(
		"resource script '%s %v' failed: exit status %d",
		err.Path,
		err.Args,
		err.ExitStatus,
	)

	if len(err.Stderr) > 0 {
		msg += "\n\nstderr:\n" + err.Stderr
	}

	return msg
}

func (resource *resource) runScript(
	path string,
	args []string,
	input interface{},
	output interface{},
	logDest io.Writer,
	inputSource ArtifactSource,
	inputDestination ArtifactDestination,
	recoverable bool,
) ifrit.Runner {
	return ifrit.RunFunc(func(signals <-chan os.Signal, ready chan<- struct{}) error {
		request, err := json.Marshal(input)
		if err != nil {
			return err
		}

		if recoverable {
			result, err := resource.container.Property(resourceResultPropertyName)
			if err == nil {
				return json.Unmarshal([]byte(result), &output)
			}
		}

		stdout := new(bytes.Buffer)
		stderr := new(bytes.Buffer)

		processIO := garden.ProcessIO{
			Stdin:  bytes.NewBuffer(request),
			Stdout: stdout,
		}

		if logDest != nil {
			processIO.Stderr = logDest
		} else {
			processIO.Stderr = stderr
		}

		var process garden.Process

		var processID string
		if recoverable {
			processID, err = resource.container.Property(resourceProcessIDPropertyName)
			if err != nil {
				processID = ""
			}
		}

		if processID != "" {
			process, err = resource.container.Attach(processID, processIO)
			if err != nil {
				return err
			}
		} else {
			if inputSource != nil {
				err := inputSource.StreamTo(inputDestination)
				if err != nil {
					return err
				}
			}

			process, err = resource.container.Run(garden.ProcessSpec{
				Path: path,
				Args: args,
			}, processIO)
			if err != nil {
				return err
			}

			if recoverable {
				err := resource.container.SetProperty(resourceProcessIDPropertyName, process.ID())
				if err != nil {
					return err
				}
			}
		}

		close(ready)

		statusCh := make(chan int, 1)
		errCh := make(chan error, 1)

		processExited := make(chan struct{})

		go func() {
			status, err := process.Wait()
			close(processExited)

			if err != nil {
				errCh <- err
			} else {
				statusCh <- status
			}
		}()

		select {
		case status := <-statusCh:
			if status != 0 {
				return ErrResourceScriptFailed{
					Path:       path,
					Args:       args,
					ExitStatus: status,

					Stderr: stderr.String(),
				}
			}

			if recoverable {
				err := resource.container.SetProperty(resourceResultPropertyName, stdout.String())
				if err != nil {
					return err
				}
			}

			return json.Unmarshal(stdout.Bytes(), output)

		case err := <-errCh:
			return err

		case <-signals:
			go process.Signal(garden.SignalTerminate)

			timer := resource.clock.NewTimer(10 * time.Second)

		OUT:
			for {
				select {
				case <-timer.C():
					process.Signal(garden.SignalKill)
				case <-processExited:
					break OUT
				}
			}

			return ErrAborted
		}
	})
}
<commit_msg>remove unnecessary channels from runScript<commit_after>package resource

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"time"

	"github.com/cloudfoundry-incubator/garden"
	"github.com/tedsuo/ifrit"
)

var ErrAborted = errors.New("script aborted")

const resourceProcessIDPropertyName = "concourse:resource-process"
const resourceResultPropertyName = "concourse:resource-result"

type ErrResourceScriptFailed struct {
	Path       string
	Args       []string
	ExitStatus int

	Stderr string
}

func (err ErrResourceScriptFailed) Error() string {
	msg := fmt.Sprintf(
		"resource script '%s %v' failed: exit status %d",
		err.Path,
		err.Args,
		err.ExitStatus,
	)

	if len(err.Stderr) > 0 {
		msg += "\n\nstderr:\n" + err.Stderr
	}

	return msg
}

func (resource *resource) runScript(
	path string,
	args []string,
	input interface{},
	output interface{},
	logDest io.Writer,
	inputSource ArtifactSource,
	inputDestination ArtifactDestination,
	recoverable bool,
) ifrit.Runner {
	return ifrit.RunFunc(func(signals <-chan os.Signal, ready chan<- struct{}) error {
		request, err := json.Marshal(input)
		if err != nil {
			return err
		}

		if recoverable {
			result, err := resource.container.Property(resourceResultPropertyName)
			if err == nil {
				return json.Unmarshal([]byte(result), &output)
			}
		}

		stdout := new(bytes.Buffer)
		stderr := new(bytes.Buffer)

		processIO := garden.ProcessIO{
			Stdin:  bytes.NewBuffer(request),
			Stdout: stdout,
		}

		if logDest != nil {
			processIO.Stderr = logDest
		} else {
			processIO.Stderr = stderr
		}

		var process garden.Process

		var processID string
		if recoverable {
			processID, err = resource.container.Property(resourceProcessIDPropertyName)
			if err != nil {
				processID = ""
			}
		}

		if processID != "" {
			process, err = resource.container.Attach(processID, processIO)
			if err != nil {
				return err
			}
		} else {
			if inputSource != nil {
				err := inputSource.StreamTo(inputDestination)
				if err != nil {
					return err
				}
			}

			process, err = resource.container.Run(garden.ProcessSpec{
				Path: path,
				Args: args,
			}, processIO)
			if err != nil {
				return err
			}

			if recoverable {
				err := resource.container.SetProperty(resourceProcessIDPropertyName, process.ID())
				if err != nil {
					return err
				}
			}
		}

		close(ready)

		processExited := make(chan struct{})

		var processStatus int
		var processErr error

		go func() {
			processStatus, processErr = process.Wait()
			close(processExited)
		}()

		select {
		case <-processExited:
			if processErr != nil {
				return processErr
			}

			if processStatus != 0 {
				return ErrResourceScriptFailed{
					Path:       path,
					Args:       args,
					ExitStatus: processStatus,

					Stderr: stderr.String(),
				}
			}

			if recoverable {
				err := resource.container.SetProperty(resourceResultPropertyName, stdout.String())
				if err != nil {
					return err
				}
			}

			return json.Unmarshal(stdout.Bytes(), output)

		case <-signals:
			go process.Signal(garden.SignalTerminate)

			timer := resource.clock.NewTimer(10 * time.Second)

		OUT:
			for {
				select {
				case <-timer.C():
					process.Signal(garden.SignalKill)
				case <-processExited:
					break OUT
				}
			}

			return ErrAborted
		}
	})
}
<|endoftext|>


<commit_before>package common

import (
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/skeswa/gophr/common/semver"
)

const (
	errorRefsFetchNoSuchRepo       = "Could not find a Github repository at %s"
	errorRefsFetchGithubError      = "Github responded with an error: %v"
	errorRefsFetchGithubParseError = "Cannot read refs from Github: %v"
	errorRefsFetchNetworkFailure   = "Could not reach Github at the moment; Please try again later"
	errorRefsParseSizeFormat       = "Could not parse refs line size: %s"
	errorRefsParseIncompleteRefs   = "Incomplete refs data received from GitHub"
)

const (
	versionRefRegexIndexLabel             = 1
	versionRefRegexIndexMajorVersion      = 2
	versionRefRegexIndexMinorVersion      = 3
	versionRefRegexIndexPatchVersion      = 4
	versionRefRegexIndexPrereleaseLabel   = 5
	versionRefRegexIndexPrereleaseVersion = 6
)

const (
	refsHead                                  = "HEAD"
	refsLineCap                               = "\n\x00"
	refsSpaceChar                             = ' '
	refsHeadPrefix                            = "refs/heads/"
	refsLineFormat                            = "%04x%s"
	refsHeadMaster                            = "refs/heads/master"
	githubRootTemplate                        = "github.com/%s/%s"
	refsMasterLineFormat                      = "%s refs/heads/master\n"
	refsSymRefAssignment                      = "symref="
	refsOldRefAssignment                      = "oldref="
	refsFetchURLTemplate                      = "https://%s.git/info/refs?service=git-upload-pack"
	refsAugmentedHeadLineFormat               = "%s HEAD\n"
	refsAugmentedSymrefHeadLineFormat         = "%s HEAD\x00symref=HEAD:%s\n"
	refsAugmentedHeadLineWithCapsFormat       = "%s HEAD\x00%s\n"
	refsAugmentedSymrefHeadLineWithCapsFormat = "%s HEAD\x00symref=HEAD:%s %s\n"
)

var (
	httpClient      = &http.Client{Timeout: 10 * time.Second}
	versionRefRegex = regexp.MustCompile(`^refs\/(?:tags|heads)\/(v?([0-9]+)(?:\.([0-9]+))?(?:\.([0-9]+))?(?:\-([a-zA-Z0-9\-_]+))?(?:\.([0-9]+))?)(?:\^\{\})?`)
)

// Refs collects information about git references for one specific repository.
type Refs struct {
	Data                 []byte
	DataStr              string
	DataLen              int
	DataStrLen           int
	Candidates           semver.SemverCandidateList
	MasterRefHash        string
	IndexHeadLineEnd     int
	IndexHeadLineStart   int
	IndexMasterLineEnd   int
	IndexMasterLineStart int
}

// NewRefs creates a new Refs instance from raw refs data fetched from Github
// (or elsewhere).
func NewRefs(data []byte) (Refs, error) {
	var (
		dataStr    = string(data)
		dataLen    = len(data)
		dataStrLen = len(dataStr)

		masterRefHash                                 string
		indexHashStart, indexHashEnd                  int
		indexNameStart, indexNameEnd                  int
		indexHeadLineStart, indexHeadLineEnd          int
		indexMasterLineStart, indexMasterLineEnd      int
		versionCandidates, sanitizedVersionCandidates []semver.SemverCandidate
	)

	for i, j := 0, 0; i < dataLen; i = j {
		// Calculate the size by reading and parsing the size string
		size, err := strconv.ParseInt(dataStr[i:i+4], 16, 32)

		// If we can't read the hex, conclude that it was invalid
		if err != nil {
			return Refs{}, fmt.Errorf(errorRefsParseSizeFormat, string(data[i:i+4]))
		}

		// If we found that the size was zero, advance it by 4 since 4 is the
		// acceptable minimum
		if size == 0 {
			size = 4
		}

		// Advance the second cursor so the next token is bounded by the two cursors
		j = i + int(size)

		// If the second cursor exceeds the string boundary, then conclude that the
		// refs data is incomplete
		if j > len(dataStr) {
			return Refs{}, errors.New(errorRefsParseIncompleteRefs)
		}

		// TODO(skeswa): figure out why this line is here
		if dataStr[0] == '#' {
			continue
		}

		// Use the cursors to get the indices of the hash
		indexHashStart = i + 4
		indexHashEnd = strings.IndexByte(
			dataStr[indexHashStart:j],
			refsSpaceChar,
		)

		// Check for invalid hash end
		if indexHashEnd < 0 || indexHashEnd != 40 {
			continue
		}

		// TODO(skeswa): figure out why this line is here
		indexHashEnd += indexHashStart

		// Use the cursors to get the indices of the name
		indexNameStart = indexHashEnd + 1
		indexNameEnd = strings.IndexAny(
			dataStr[indexNameStart:j],
			refsLineCap,
		)

		// Check for invalid name end
		if indexNameEnd < 0 {
			indexNameEnd = j
		} else {
			indexNameEnd += indexNameStart
		}

		// Get the name and hash respectively as strings
		hash := dataStr[indexHashStart:indexHashEnd]
		name := dataStr[indexNameStart:indexNameEnd]

		// Process the name and hash according to whether the name is relevant
		if name == refsHead {
			indexHeadLineStart = i
			indexHeadLineEnd = j
		} else if name == refsHeadMaster {
			indexMasterLineStart = i
			indexMasterLineEnd = j
			masterRefHash = hash
		} else if captureGroups := versionRefRegex.FindStringSubmatch(name); captureGroups != nil {
			var (
				gitRefLabel       = captureGroups[versionRefRegexIndexLabel]
				majorVersion      = captureGroups[versionRefRegexIndexMajorVersion]
				minorVersion      = captureGroups[versionRefRegexIndexMinorVersion]
				patchVersion      = captureGroups[versionRefRegexIndexPatchVersion]
				prereleaseLabel   = captureGroups[versionRefRegexIndexPrereleaseLabel]
				prereleaseVersion = captureGroups[versionRefRegexIndexPrereleaseVersion]
			)

			// Annotated tag is peeled off and overrides the same version just parsed
			if strings.HasSuffix(name, "^{}") {
				name = name[:len(name)-3]
			}

			versionCandidate, err := semver.NewSemverCandidate(
				hash,
				name,
				gitRefLabel,
				majorVersion,
				minorVersion,
				patchVersion,
				prereleaseLabel,
				prereleaseVersion)
			if err == nil {
				versionCandidates = append(versionCandidates, versionCandidate)
			}
		}
	}

	if versionCandidates != nil && len(versionCandidates) > 0 {
		// First attach the sortable type to the slice of candidates.
		versionCandidatesList := semver.SemverCandidateList(versionCandidates)
		// Sort the list of candidates.
		sort.Sort(versionCandidatesList)
		// Remove duplicates by adding them to a new slice altogether.
		var lastInsertedCandidate semver.SemverCandidate
		for i, versionCandidate := range versionCandidatesList {
			if i == 0 || versionCandidate.CompareTo(lastInsertedCandidate) != 0 {
				sanitizedVersionCandidates = append(sanitizedVersionCandidates, versionCandidate)
				lastInsertedCandidate = versionCandidate
			}
		}
	}

	return Refs{
		Data:                 data,
		DataStr:              dataStr,
		DataLen:              dataLen,
		DataStrLen:           dataStrLen,
		Candidates:           sanitizedVersionCandidates,
		MasterRefHash:        masterRefHash,
		IndexHeadLineEnd:     indexHeadLineEnd,
		IndexMasterLineEnd:   indexMasterLineEnd,
		IndexHeadLineStart:   indexHeadLineStart,
		IndexMasterLineStart: indexMasterLineStart,
	}, nil
}

// FetchRefs downloads and processes refs data from Github and ultimately
// contructs a Refs instance with it.
func FetchRefs(author, repo string) (Refs, error) {
	githubRoot := fmt.Sprintf(
		githubRootTemplate,
		author,
		repo,
	)

	res, err := httpClient.Get(fmt.Sprintf(refsFetchURLTemplate, githubRoot))
	if err != nil {
		return Refs{}, errors.New(errorRefsFetchNetworkFailure)
	}

	defer res.Body.Close()

	if res.StatusCode >= 400 && res.StatusCode < 500 {
		return Refs{}, fmt.Errorf(errorRefsFetchNoSuchRepo, githubRoot)
	} else if res.StatusCode >= 500 {
		// FYI no reliable way to get test coverage here; this never happens
		return Refs{}, fmt.Errorf(errorRefsFetchGithubError, res.Status)
	}

	data, err := ioutil.ReadAll(res.Body)
	if err != nil {
		// FYI no reliable way to get test coverage here; this never happens
		return Refs{}, fmt.Errorf(errorRefsFetchGithubParseError, err)
	}

	return NewRefs(data)
}

// Reserialize changes the refs data to incorporate the selected version as
// the HEAD instead of the default HEAD.
//
// This code was written by Gustavo Niemeyer, Nathan Youngman and
// Geert-Johan Riemer.
func (refsData Refs) Reserialize(versionRefName, versionRefHash string) []byte {
	var (
		buf bytes.Buffer

		data                 = refsData.Data
		dataLen              = refsData.DataLen
		dataStr              = refsData.DataStr
		indexHeadLineEnd     = refsData.IndexHeadLineEnd
		indexHeadLineStart   = refsData.IndexHeadLineStart
		indexMasterLineEnd   = refsData.IndexMasterLineEnd
		indexMasterLineStart = refsData.IndexMasterLineStart
	)

	// Size the buffer to be a little bigger than
	buf.Grow(dataLen + 256)

	// Copy the header as-is.
	buf.Write(data[:indexHeadLineStart])

	// Extract the original capabilities.
	caps := ""
	indexNullByte := strings.Index(
		dataStr[indexHeadLineStart:indexHeadLineEnd],
		"\x00",
	)

	// IF we found a zero byte, replace the symref with an oldref
	if indexNullByte > 0 {
		caps = strings.Replace(
			dataStr[indexHeadLineStart+indexNullByte+1:indexHeadLineEnd-1],
			refsSymRefAssignment,
			refsOldRefAssignment,
			-1,
		)
	}

	// Insert the HEAD reference line with the right hash and a proper symref
	// capability.
	var line string
	if strings.HasPrefix(versionRefName, refsHeadPrefix) {
		if caps == "" {
			line = fmt.Sprintf(
				refsAugmentedSymrefHeadLineFormat,
				versionRefHash,
				versionRefName,
			)
		} else {
			line = fmt.Sprintf(
				refsAugmentedSymrefHeadLineWithCapsFormat,
				versionRefHash,
				versionRefName,
				caps,
			)
		}
	} else {
		if caps == "" {
			line = fmt.Sprintf(refsAugmentedHeadLineFormat, versionRefHash)
		} else {
			line = fmt.Sprintf(refsAugmentedHeadLineWithCapsFormat, versionRefHash, caps)
		}
	}
	fmt.Fprintf(&buf, "%04x%s", 4+len(line), line)

	// Insert the master reference line.
	line = fmt.Sprintf(refsMasterLineFormat, versionRefHash)
	fmt.Fprintf(&buf, refsLineFormat, 4+len(line), line)

	// Append the rest, dropping the original master line if necessary.
	if indexMasterLineStart > 0 {
		buf.Write(data[indexHeadLineEnd:indexMasterLineStart])
		buf.Write(data[indexMasterLineEnd:])
	} else {
		buf.Write(data[indexHeadLineEnd:])
	}

	return buf.Bytes()
}

// CheckIfRefExists downloads and processes refs data from Github and checks
// whether a given ref exists in the remote refs list.
func CheckIfRefExists(author, repo string, ref string) (bool, error) {
	githubRoot := fmt.Sprintf(
		githubRootTemplate,
		author,
		repo,
	)

	res, err := httpClient.Get(fmt.Sprintf(refsFetchURLTemplate, githubRoot))
	if err != nil {
		return false, errors.New(errorRefsFetchNetworkFailure)
	}

	defer res.Body.Close()

	if res.StatusCode >= 400 && res.StatusCode < 500 {
		return false, fmt.Errorf(errorRefsFetchNoSuchRepo, githubRoot)
	} else if res.StatusCode >= 500 {
		// FYI no reliable way to get test coverage here; this never happens
		return false, fmt.Errorf(errorRefsFetchGithubError, res.Status)
	}

	data, err := ioutil.ReadAll(res.Body)
	if err != nil {
		// FYI no reliable way to get test coverage here; this never happens
		return false, fmt.Errorf(errorRefsFetchGithubParseError, err)
	}

	refsString := string(data)
	refExists := strings.Contains(refsString, ref)

	return refExists, nil
}
<commit_msg>refs params are now generated properly<commit_after>package common

import (
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/skeswa/gophr/common/semver"
)

const (
	errorRefsFetchNoSuchRepo       = "Could not find a Github repository at %s"
	errorRefsFetchGithubError      = "Github responded with an error: %v"
	errorRefsFetchGithubParseError = "Cannot read refs from Github: %v"
	errorRefsFetchNetworkFailure   = "Could not reach Github at the moment; Please try again later"
	errorRefsParseSizeFormat       = "Could not parse refs line size: %s"
	errorRefsParseIncompleteRefs   = "Incomplete refs data received from GitHub"
)

const (
	versionRefRegexIndexLabel             = 1
	versionRefRegexIndexMajorVersion      = 2
	versionRefRegexIndexMinorVersion      = 3
	versionRefRegexIndexPatchVersion      = 4
	versionRefRegexIndexPrereleaseLabel   = 5
	versionRefRegexIndexPrereleaseVersion = 6
)

const (
	refsHead                                  = "HEAD"
	refsLineCap                               = "\n\x00"
	refsSpaceChar                             = ' '
	refsHeadPrefix                            = "refs/heads/"
	refsLineFormat                            = "%04x%s"
	refsHeadMaster                            = "refs/heads/master"
	githubRootTemplate                        = "github.com/%s/%s"
	refsMasterLineFormat                      = "%s refs/heads/master\n"
	refsSymRefAssignment                      = "symref="
	refsOldRefAssignment                      = "oldref="
	refsFetchURLTemplate                      = "https://%s.git/info/refs?service=git-upload-pack"
	refsAugmentedHeadLineFormat               = "%s HEAD\n"
	refsAugmentedSymrefHeadLineFormat         = "%s HEAD\x00symref=HEAD:%s\n"
	refsAugmentedHeadLineWithCapsFormat       = "%s HEAD\x00%s\n"
	refsAugmentedSymrefHeadLineWithCapsFormat = "%s HEAD\x00symref=HEAD:%s %s\n"
)

var (
	httpClient      = &http.Client{Timeout: 10 * time.Second}
	versionRefRegex = regexp.MustCompile(`^refs\/(?:tags|heads)\/(v?([0-9]+)(?:\.([0-9]+))?(?:\.([0-9]+))?(?:\-([a-zA-Z0-9\-_]+))?(?:\.([0-9]+))?)(?:\^\{\})?`)
)

// Refs collects information about git references for one specific repository.
type Refs struct {
	Data                 []byte
	DataStr              string
	DataLen              int
	DataStrLen           int
	Candidates           semver.SemverCandidateList
	MasterRefHash        string
	IndexHeadLineEnd     int
	IndexHeadLineStart   int
	IndexMasterLineEnd   int
	IndexMasterLineStart int
}

// NewRefs creates a new Refs instance from raw refs data fetched from Github
// (or elsewhere).
func NewRefs(data []byte) (Refs, error) {
	var (
		dataStr    = string(data)
		dataLen    = len(data)
		dataStrLen = len(dataStr)

		masterRefHash                                 string
		indexHashStart, indexHashEnd                  int
		indexNameStart, indexNameEnd                  int
		indexHeadLineStart, indexHeadLineEnd          int
		indexMasterLineStart, indexMasterLineEnd      int
		versionCandidates, sanitizedVersionCandidates []semver.SemverCandidate
	)

	for i, j := 0, 0; i < dataLen; i = j {
		// Calculate the size by reading and parsing the size string
		size, err := strconv.ParseInt(dataStr[i:i+4], 16, 32)

		// If we can't read the hex, conclude that it was invalid
		if err != nil {
			return Refs{}, fmt.Errorf(errorRefsParseSizeFormat, string(data[i:i+4]))
		}

		// If we found that the size was zero, advance it by 4 since 4 is the
		// acceptable minimum
		if size == 0 {
			size = 4
		}

		// Advance the second cursor so the next token is bounded by the two cursors
		j = i + int(size)

		// If the second cursor exceeds the string boundary, then conclude that the
		// refs data is incomplete
		if j > len(dataStr) {
			return Refs{}, errors.New(errorRefsParseIncompleteRefs)
		}

		// TODO(skeswa): figure out why this line is here
		if dataStr[0] == '#' {
			continue
		}

		// Use the cursors to get the indices of the hash
		indexHashStart = i + 4
		indexHashEnd = strings.IndexByte(
			dataStr[indexHashStart:j],
			refsSpaceChar,
		)

		// Check for invalid hash end
		if indexHashEnd < 0 || indexHashEnd != 40 {
			continue
		}

		// TODO(skeswa): figure out why this line is here
		indexHashEnd += indexHashStart

		// Use the cursors to get the indices of the name
		indexNameStart = indexHashEnd + 1
		indexNameEnd = strings.IndexAny(
			dataStr[indexNameStart:j],
			refsLineCap,
		)

		// Check for invalid name end
		if indexNameEnd < 0 {
			indexNameEnd = j
		} else {
			indexNameEnd += indexNameStart
		}

		// Get the name and hash respectively as strings
		hash := dataStr[indexHashStart:indexHashEnd]
		name := dataStr[indexNameStart:indexNameEnd]

		// Process the name and hash according to whether the name is relevant
		if name == refsHead {
			indexHeadLineStart = i
			indexHeadLineEnd = j
		} else if name == refsHeadMaster {
			indexMasterLineStart = i
			indexMasterLineEnd = j
			masterRefHash = hash
		} else if captureGroups := versionRefRegex.FindStringSubmatch(name); captureGroups != nil {
			var (
				gitRefLabel       = captureGroups[versionRefRegexIndexLabel]
				majorVersion      = captureGroups[versionRefRegexIndexMajorVersion]
				minorVersion      = captureGroups[versionRefRegexIndexMinorVersion]
				patchVersion      = captureGroups[versionRefRegexIndexPatchVersion]
				prereleaseLabel   = captureGroups[versionRefRegexIndexPrereleaseLabel]
				prereleaseVersion = captureGroups[versionRefRegexIndexPrereleaseVersion]
			)

			// Annotated tag is peeled off and overrides the same version just parsed
			if strings.HasSuffix(name, "^{}") {
				name = name[:len(name)-3]
			}

			versionCandidate, err := semver.NewSemverCandidate(
				hash,
				name,
				gitRefLabel,
				majorVersion,
				minorVersion,
				patchVersion,
				prereleaseLabel,
				prereleaseVersion)
			if err == nil {
				versionCandidates = append(versionCandidates, versionCandidate)
			}
		}
	}

	if versionCandidates != nil && len(versionCandidates) > 0 {
		// First attach the sortable type to the slice of candidates.
		versionCandidatesList := semver.SemverCandidateList(versionCandidates)
		// Sort the list of candidates.
		sort.Sort(versionCandidatesList)
		// Remove duplicates by adding them to a new slice altogether.
		var lastInsertedCandidate semver.SemverCandidate
		for i, versionCandidate := range versionCandidatesList {
			if i == 0 || versionCandidate.CompareTo(lastInsertedCandidate) != 0 {
				sanitizedVersionCandidates = append(sanitizedVersionCandidates, versionCandidate)
				lastInsertedCandidate = versionCandidate
			}
		}
	}

	return Refs{
		Data:                 data,
		DataStr:              dataStr,
		DataLen:              dataLen,
		DataStrLen:           dataStrLen,
		Candidates:           sanitizedVersionCandidates,
		MasterRefHash:        masterRefHash,
		IndexHeadLineEnd:     indexHeadLineEnd,
		IndexMasterLineEnd:   indexMasterLineEnd,
		IndexHeadLineStart:   indexHeadLineStart,
		IndexMasterLineStart: indexMasterLineStart,
	}, nil
}

// FetchRefs downloads and processes refs data from Github and ultimately
// contructs a Refs instance with it.
func FetchRefs(author, repo string) (Refs, error) {
	githubRoot := fmt.Sprintf(
		githubRootTemplate,
		author,
		repo,
	)

	res, err := httpClient.Get(fmt.Sprintf(refsFetchURLTemplate, githubRoot))
	if err != nil {
		return Refs{}, errors.New(errorRefsFetchNetworkFailure)
	}

	defer res.Body.Close()

	if res.StatusCode >= 400 && res.StatusCode < 500 {
		return Refs{}, fmt.Errorf(errorRefsFetchNoSuchRepo, githubRoot)
	} else if res.StatusCode >= 500 {
		// FYI no reliable way to get test coverage here; this never happens
		return Refs{}, fmt.Errorf(errorRefsFetchGithubError, res.Status)
	}

	data, err := ioutil.ReadAll(res.Body)
	if err != nil {
		// FYI no reliable way to get test coverage here; this never happens
		return Refs{}, fmt.Errorf(errorRefsFetchGithubParseError, err)
	}

	return NewRefs(data)
}

// Reserialize changes the refs data to incorporate the selected version as
// the HEAD instead of the default HEAD.
//
// This code was written by Gustavo Niemeyer, Nathan Youngman and
// Geert-Johan Riemer.
func (refsData Refs) Reserialize(versionRefName, versionRefHash string) []byte {
	var (
		buf bytes.Buffer

		data                 = refsData.Data
		dataLen              = refsData.DataLen
		dataStr              = refsData.DataStr
		indexHeadLineEnd     = refsData.IndexHeadLineEnd
		indexHeadLineStart   = refsData.IndexHeadLineStart
		indexMasterLineEnd   = refsData.IndexMasterLineEnd
		indexMasterLineStart = refsData.IndexMasterLineStart
	)

	// Size the buffer to be a little bigger than
	buf.Grow(dataLen + 256)

	// Copy the header as-is.
	buf.Write(data[:indexHeadLineStart])

	// Extract the original capabilities.
	caps := ""
	indexNullByte := strings.Index(
		dataStr[indexHeadLineStart:indexHeadLineEnd],
		"\x00",
	)

	// IF we found a zero byte, replace the symref with an oldref
	if indexNullByte > 0 {
		caps = strings.Replace(
			dataStr[indexHeadLineStart+indexNullByte+1:indexHeadLineEnd-1],
			refsSymRefAssignment,
			refsOldRefAssignment,
			-1,
		)
	}

	// Insert the HEAD reference line with the right hash and a proper symref
	// capability.
	var line string
	if strings.HasPrefix(versionRefName, refsHeadPrefix) {
		if caps == "" {
			line = fmt.Sprintf(
				refsAugmentedSymrefHeadLineFormat,
				versionRefHash,
				versionRefName,
			)
		} else {
			line = fmt.Sprintf(
				refsAugmentedSymrefHeadLineWithCapsFormat,
				versionRefHash,
				versionRefName,
				caps,
			)
		}
	} else {
		if caps == "" {
			line = fmt.Sprintf(refsAugmentedHeadLineFormat, versionRefHash)
		} else {
			line = fmt.Sprintf(refsAugmentedHeadLineWithCapsFormat, versionRefHash, caps)
		}
	}
	fmt.Fprintf(&buf, "%04x%s", 4+len(line), line)

	// Insert the master reference line.
	line = fmt.Sprintf(refsMasterLineFormat, versionRefHash)
	fmt.Fprintf(&buf, refsLineFormat, 4+len(line), line)

	// Append the rest, dropping the original master line if necessary.
	if indexMasterLineStart > 0 {
		buf.Write(data[indexHeadLineEnd:indexMasterLineStart])
		buf.Write(data[indexMasterLineEnd:])
	} else {
		buf.Write(data[indexHeadLineEnd:])
	}

	return buf.Bytes()
}

// CheckIfRefExists downloads and processes refs data from Github and checks
// whether a given ref exists in the remote refs list.
func CheckIfRefExists(author, repo string, ref string) (bool, error) {
	ref = BuildGitHubBranch(ref)
	repo = BuildNewGitHubRepoName(author, repo)
	author = GitHubGophrPackageOrgName
	githubRoot := fmt.Sprintf(
		githubRootTemplate,
		author,
		repo,
	)

	res, err := httpClient.Get(fmt.Sprintf(refsFetchURLTemplate, githubRoot))
	if err != nil {
		return false, errors.New(errorRefsFetchNetworkFailure)
	}

	defer res.Body.Close()

	if res.StatusCode >= 400 && res.StatusCode < 500 {
		return false, fmt.Errorf(errorRefsFetchNoSuchRepo, githubRoot)
	} else if res.StatusCode >= 500 {
		// FYI no reliable way to get test coverage here; this never happens
		return false, fmt.Errorf(errorRefsFetchGithubError, res.Status)
	}

	data, err := ioutil.ReadAll(res.Body)
	if err != nil {
		// FYI no reliable way to get test coverage here; this never happens
		return false, fmt.Errorf(errorRefsFetchGithubParseError, err)
	}

	refsString := string(data)
	refExists := strings.Contains(refsString, ref)

	return refExists, nil
}
<|endoftext|>