<commit_before>package main

import (
	"fmt"
	"os"

	"math"
	"sort"

	"github.com/codegangsta/cli"
	"github.com/mattn/go-tty"
	"github.com/pkg/browser"
)

type ActionType int

const (
	LABEL_AS_POSITIVE ActionType = iota
	LABEL_AS_NEGATIVE
	SAVE
	HELP
	SKIP
	EXIT
)

func input2ActionType() (ActionType, error) {
	t, err := tty.Open()
	defer t.Close()
	if err != nil {
		return EXIT, err
	}
	var r rune
	for r == 0 {
		r, err = t.ReadRune()
		if err != nil {
			return SKIP, err
		}
	}
	switch r {
	case 'p':
		return LABEL_AS_POSITIVE, nil
	case 'n':
		return LABEL_AS_NEGATIVE, nil
	case 's':
		return SAVE, nil
	case 'h':
		return HELP, nil
	case 'e':
		return EXIT, nil
	default:
		return SKIP, nil
	}
}

var ActionHelpDoc = `
p: Label this example as positive.
n: Label this example as negative.
s: Save additionally annotated examples in 'output-filename'.
h: Show this help.
e: Exit.
`

func doAnnotate(c *cli.Context) error {
	inputFilename := c.String("input-filename")
	outputFilename := c.String("output-filename")
	openUrl := c.Bool("open-url")
	filterStatusCodeOk := c.Bool("filter-status-code-ok")
	showActiveFeatures := c.Bool("show-active-features")

	if inputFilename == "" {
		_ = cli.ShowCommandHelp(c, "annotate")
		return cli.NewExitError("`input-filename` is a required field.", 1)
	}

	if outputFilename == "" {
		_ = cli.ShowCommandHelp(c, "annotate")
		return cli.NewExitError("`output-filename` is a required field.", 1)
	}

	cacheFilename := CacheFilename

	cache, err := LoadCache(cacheFilename)
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
	}

	examples, err := ReadExamples(inputFilename)
	if err != nil {
		return err
	}

	AttachMetaData(cache, examples)
	if filterStatusCodeOk {
		examples = FilterStatusCodeOkExamples(examples)
	}
	model := TrainedModel(examples)

annotationLoop:
	for {
		unlabeledExamples := model.SortByScore(examples)
		if len(unlabeledExamples) == 0 {
			break
		}
		e := unlabeledExamples[0]
		if e == nil {
			break
		}
		fmt.Println("Label this example (Score: " + fmt.Sprintf("%+0.03f", e.Score) + "): " + e.Url + " (" + e.Title + ")")
		cache.Add(*e)

		if openUrl {
			browser.OpenURL(e.Url)
		}
		if showActiveFeatures {
			ShowActiveFeatures(model, *e, 5)
		}

		act, err := input2ActionType()
		if err != nil {
			return err
		}
		switch act {
		case LABEL_AS_POSITIVE:
			fmt.Println("Labeled as positive")
			e.Annotate(POSITIVE)
		case LABEL_AS_NEGATIVE:
			fmt.Println("Labeled as negative")
			e.Annotate(NEGATIVE)
		case SKIP:
			fmt.Println("Skiped this example")
			continue
		case SAVE:
			fmt.Println("Saved labeld examples")
			WriteExamples(examples, outputFilename)
		case HELP:
			fmt.Println(ActionHelpDoc)
		case EXIT:
			fmt.Println("EXIT")
			break annotationLoop
		default:
			break annotationLoop
		}
		model = TrainedModel(examples)
	}

	WriteExamples(examples, outputFilename)
	cache.Save(cacheFilename)

	return nil
}

var commandAnnotate = cli.Command{
	Name:  "annotate",
	Usage: "Annotate URLs",
	Description: `
Annotate URLs using active learning.
`,
	Action: doAnnotate,
	Flags: []cli.Flag{
		cli.StringFlag{Name: "input-filename"},
		cli.StringFlag{Name: "output-filename"},
		cli.BoolFlag{Name: "open-url", Usage: "Open url in background"},
		cli.BoolFlag{Name: "filter-status-code-ok", Usage: "Use only examples with status code = 200"},
		cli.BoolFlag{Name: "show-active-features"},
	},
}

type FeatureWeightPair struct {
	Feature string
	Weight  float64
}

type FeatureWeightPairs []FeatureWeightPair

func ShowActiveFeatures(model *Model, example Example, n int) {
	result := FeatureWeightPairs{}
	for _, f := range example.Fv {
		result = append(result, FeatureWeightPair{f, model.GetAveragedWeight(f)})
	}
	sort.Sort(sort.Reverse(result))

	cnt := 0
	for _, pair := range result {
		if cnt >= n {
			break
		}
		fmt.Println(fmt.Sprintf("%+0.1f %s", pair.Weight, pair.Feature))
		cnt++
	}
}

func (slice FeatureWeightPairs) Len() int {
	return len(slice)
}

func (slice FeatureWeightPairs) Less(i, j int) bool {
	return math.Abs(slice[i].Weight) < math.Abs(slice[j].Weight)
}

func (slice FeatureWeightPairs) Swap(i, j int) {
	slice[i], slice[j] = slice[j], slice[i]
}
<commit_msg>スコアと同一側でactiveな重みのみを出すように変更<commit_after>package main

import (
	"fmt"
	"os"

	"math"
	"sort"

	"github.com/codegangsta/cli"
	"github.com/mattn/go-tty"
	"github.com/pkg/browser"
)

type ActionType int

const (
	LABEL_AS_POSITIVE ActionType = iota
	LABEL_AS_NEGATIVE
	SAVE
	HELP
	SKIP
	EXIT
)

func input2ActionType() (ActionType, error) {
	t, err := tty.Open()
	defer t.Close()
	if err != nil {
		return EXIT, err
	}
	var r rune
	for r == 0 {
		r, err = t.ReadRune()
		if err != nil {
			return SKIP, err
		}
	}
	switch r {
	case 'p':
		return LABEL_AS_POSITIVE, nil
	case 'n':
		return LABEL_AS_NEGATIVE, nil
	case 's':
		return SAVE, nil
	case 'h':
		return HELP, nil
	case 'e':
		return EXIT, nil
	default:
		return SKIP, nil
	}
}

var ActionHelpDoc = `
p: Label this example as positive.
n: Label this example as negative.
s: Save additionally annotated examples in 'output-filename'.
h: Show this help.
e: Exit.
`

func doAnnotate(c *cli.Context) error {
	inputFilename := c.String("input-filename")
	outputFilename := c.String("output-filename")
	openUrl := c.Bool("open-url")
	filterStatusCodeOk := c.Bool("filter-status-code-ok")
	showActiveFeatures := c.Bool("show-active-features")

	if inputFilename == "" {
		_ = cli.ShowCommandHelp(c, "annotate")
		return cli.NewExitError("`input-filename` is a required field.", 1)
	}

	if outputFilename == "" {
		_ = cli.ShowCommandHelp(c, "annotate")
		return cli.NewExitError("`output-filename` is a required field.", 1)
	}

	cacheFilename := CacheFilename

	cache, err := LoadCache(cacheFilename)
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
	}

	examples, err := ReadExamples(inputFilename)
	if err != nil {
		return err
	}

	AttachMetaData(cache, examples)
	if filterStatusCodeOk {
		examples = FilterStatusCodeOkExamples(examples)
	}
	model := TrainedModel(examples)

annotationLoop:
	for {
		unlabeledExamples := model.SortByScore(examples)
		if len(unlabeledExamples) == 0 {
			break
		}
		e := unlabeledExamples[0]
		if e == nil {
			break
		}
		fmt.Println("Label this example (Score: " + fmt.Sprintf("%+0.03f", e.Score) + "): " + e.Url + " (" + e.Title + ")")
		cache.Add(*e)

		if openUrl {
			browser.OpenURL(e.Url)
		}
		if showActiveFeatures {
			ShowActiveFeatures(model, *e, 5)
		}

		act, err := input2ActionType()
		if err != nil {
			return err
		}
		switch act {
		case LABEL_AS_POSITIVE:
			fmt.Println("Labeled as positive")
			e.Annotate(POSITIVE)
		case LABEL_AS_NEGATIVE:
			fmt.Println("Labeled as negative")
			e.Annotate(NEGATIVE)
		case SKIP:
			fmt.Println("Skiped this example")
			continue
		case SAVE:
			fmt.Println("Saved labeld examples")
			WriteExamples(examples, outputFilename)
		case HELP:
			fmt.Println(ActionHelpDoc)
		case EXIT:
			fmt.Println("EXIT")
			break annotationLoop
		default:
			break annotationLoop
		}
		model = TrainedModel(examples)
	}

	WriteExamples(examples, outputFilename)
	cache.Save(cacheFilename)

	return nil
}

var commandAnnotate = cli.Command{
	Name:  "annotate",
	Usage: "Annotate URLs",
	Description: `
Annotate URLs using active learning.
`,
	Action: doAnnotate,
	Flags: []cli.Flag{
		cli.StringFlag{Name: "input-filename"},
		cli.StringFlag{Name: "output-filename"},
		cli.BoolFlag{Name: "open-url", Usage: "Open url in background"},
		cli.BoolFlag{Name: "filter-status-code-ok", Usage: "Use only examples with status code = 200"},
		cli.BoolFlag{Name: "show-active-features"},
	},
}

type FeatureWeightPair struct {
	Feature string
	Weight  float64
}

type FeatureWeightPairs []FeatureWeightPair

func ShowActiveFeatures(model *Model, example Example, n int) {
	result := FeatureWeightPairs{}
	for _, f := range example.Fv {
		result = append(result, FeatureWeightPair{f, model.GetAveragedWeight(f)})
	}
	sort.Sort(sort.Reverse(result))

	cnt := 0
	for _, pair := range result {
		if cnt >= n {
			break
		}
		if (example.Score > 0.0 && pair.Weight > 0.0) || (example.Score < 0.0 && pair.Weight < 0.0) {
			fmt.Println(fmt.Sprintf("%+0.1f %s", pair.Weight, pair.Feature))
			cnt++
		}
	}
}

func (slice FeatureWeightPairs) Len() int {
	return len(slice)
}

func (slice FeatureWeightPairs) Less(i, j int) bool {
	return math.Abs(slice[i].Weight) < math.Abs(slice[j].Weight)
}

func (slice FeatureWeightPairs) Swap(i, j int) {
	slice[i], slice[j] = slice[j], slice[i]
}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"io"
	"log"
	"fmt"
	"flag"
	"strings"
	"net/http"
	"os/exec"
	"text/template"
	"html"
)

type TemplateData struct {
	Name string
	Link string
	Content string
}

const (
	TemplateName = ".tmpl"
	InterpreterName = ".interpreters"
)

var siteRoot *string = flag.String("r", ".", "Path to files")

var rootName *string = flag.String("n", "debug",
"Name given to template when / is requested")

var nameFormat *string = flag.String("f", "%s - debug",
"String used by fmt to get name to give to template, one string is " +
"given for parsing, the name of the file less it's suffix.")

var serverPort *string = flag.String("p", "80", "Port to listen on")

var maxBytes *int = flag.Int("m", 2 * 1024 * 1024,
"Max file size that will be given to templates. Also the chunk size " +
"that is read in before writing to the stream")

/*
 * Split s on last occurence of pattern, so returns (most, suffix).
 * If no matches of pattern were found then returns (s, "").
 */
func splitSuffix(s string, pattern string) (string, string) {
	l := strings.LastIndex(s, pattern)
	if l > 0 {
		return s[:l], s[l+1:]
	} else {
		return s, ""
	}
}

func findFile(path string, name string) string {
	for {
		path, _ = splitSuffix(path, "/")
		if path == "" {
			return os.DevNull
		}
		p := path + "/" + name
		_, err := os.Stat(p)
		if err == nil {
			return p
		}
	}
}

func dirIndex(file *os.File) string {
	names, err := file.Readdirnames(0)
	if err != nil {
		return ""
	}
	file.Seek(0, 0)

	dir := file.Name()
	if !strings.HasSuffix(dir, "/") {
		dir += "/"
	}

	for _, name := range names {
		if strings.HasPrefix(name, "index") {
			return name
		}
	}

	return ""
}

func readLine(file *os.File, bytes []byte) (string, error) {
	n, err := file.Read(bytes)
	if err != nil {
		return "", err
	}

	s := string(bytes[:n])
	l := strings.IndexByte(s, '\n') + 1

	if l > 0 {
		file.Seek(int64(l - n), 1)
		return s[:l-1], nil
	} else {
		return "", nil
	}
}

func findInterpreter(path string) (bool, []string) {
	intPath := findFile(path, InterpreterName)
	if intPath == "" {
		return false, []string{}
	}

	file, err := os.Open(intPath)
	if err != nil {
		log.Print(err)
		return false, []string{}
	}

	bytes := make([]byte, 256)

	for {
		line, err := readLine(file, bytes)
		if err != nil {
			return false, []string{}
		} else if len(line) == 0 || line[0] == '#' {
			continue
		}

		suffix := strings.SplitN(line, " ", 2)
		if len(suffix) > 0 && strings.HasSuffix(path, suffix[0]) {
			parts := strings.Split(line, " ")
			if len(parts) < 2 {
				log.Print("Error in interpreter file. ", path)
				continue
			} else {
				return strings.HasPrefix(parts[1], "y"),
					parts[2:]
			}
		}
	}
}

func runInterpreter(interpreter []string,
		values map[string][]string, file *os.File) ([]byte, error) {
	dir, base := splitSuffix(file.Name(), "/")
	cmd := exec.Command(interpreter[0])
	cmd.Args = append(interpreter, base)
	cmd.Dir = dir

	l := len(cmd.Env) + len(values) + 1
	env := make([]string, l)
	copy(env, cmd.Env)

	i := len(cmd.Env) + 1
	for name, value := range values {
		env[i] = name + "=" + value[0]
		i++
	}

	cmd.Env = env
	return cmd.Output()
}

func processFile(w http.ResponseWriter, req *http.Request,
		data *TemplateData, file *os.File, fi os.FileInfo) {
	var err error
	var bytes []byte = make([]byte, *maxBytes)
	var n int

	useTemplate, interpreter := findInterpreter(file.Name())

	if len(interpreter) > 0 {
		bytes, err = runInterpreter(interpreter,
				req.URL.Query(), file)

	} else {
		n, err = file.Read(bytes)
	}

	if err != nil {
		log.Print(err)
		io.WriteString(w, "ERROR")
		return
	}

	if useTemplate {
		tmplPath := findFile(file.Name(), TemplateName)
		tmpl, err := template.ParseFiles(tmplPath)
		if err == nil {
			data.Content = string(bytes)
			tmpl.Execute(w, data)
			return
		}
	}

	/* No template/error opening template */
	if len(interpreter) > 0 {
		req.ContentLength = int64(len(bytes))
		w.Write(bytes)
	} else {
		req.ContentLength = fi.Size()
		log.Print("giving file, len: ", req.ContentLength)
		for {
			log.Print("giving bytes: ", n)
			w.Write(bytes[:n])
			n, err = file.Read(bytes)
			if err != nil {
				break
			}
		}
	}

}

func handler(w http.ResponseWriter, req *http.Request) {
	var file *os.File
	var err error
	var name string

	log.Print(req.RemoteAddr, " request: ", req.URL.String())

	path := "." + html.EscapeString(req.URL.Path)

	file, err = os.Open(path)
	if err != nil {
		log.Print("404 ", err)
		w.WriteHeader(http.StatusNotFound)
		io.WriteString(w, "404: " + html.EscapeString(req.URL.Path))
		return
	}
	defer file.Close()

	fi, err := file.Stat()
	if err != nil {
		log.Print(err)
		return
	}

	data := new(TemplateData)
	data.Link = req.URL.Path

	if strings.HasPrefix(fi.Name(), "index") {
		path, _ = splitSuffix(path, "/")
		_, name = splitSuffix(path, "/")
		path += "/"
	} else {
		name, _ = splitSuffix(fi.Name(), ".")
	}

	if path == "./" {
		data.Name = *rootName
	} else {
		data.Name = fmt.Sprintf(*nameFormat, name)
	}

	if fi.IsDir() {
		index := dirIndex(file)

		if index == "" {
			w.WriteHeader(http.StatusNotFound)
			io.WriteString(w, "404: " +
				html.EscapeString(req.URL.Path))
			return
		} else if !strings.HasSuffix(path, "/") {
			url := req.URL.Scheme + req.URL.Path +
				"/" + req.URL.RawQuery
			http.Redirect(w, req, url,
				http.StatusMovedPermanently)
		} else {
			path += index
			file, err = os.Open(path)
			if err != nil {
				log.Print(err)
				return
			}
			defer file.Close()
			/* Fall through to process file */
		}
	}

	processFile(w, req, data, file, fi)
}

func main() {
	flag.Parse()

	os.Chdir(*siteRoot)

	http.HandleFunc("/", handler)
	err := http.ListenAndServe(":" + *serverPort, nil)
	if err != nil {
		log.Fatal("ListenAndServe: ", err)
	}
}
<commit_msg>now stops sending if the client disconnects<commit_after>package main

import (
	"os"
	"io"
	"log"
	"fmt"
	"flag"
	"strings"
	"net/http"
	"os/exec"
	"text/template"
	"html"
)

type TemplateData struct {
	Name string
	Link string
	Content string
}

const (
	TemplateName = ".tmpl"
	InterpreterName = ".interpreters"
)

var siteRoot *string = flag.String("r", ".", "Path to files")

var rootName *string = flag.String("n", "debug",
"Name given to template when / is requested")

var nameFormat *string = flag.String("f", "%s - debug",
"String used by fmt to get name to give to template, one string is " +
"given for parsing, the name of the file less it's suffix.")

var serverPort *string = flag.String("p", "80", "Port to listen on")

var maxBytes *int = flag.Int("m", 2 * 1024 * 1024,
"Max file size that will be given to templates. Also the chunk size " +
"that is read in before writing to the stream")

/*
 * Split s on last occurence of pattern, so returns (most, suffix).
 * If no matches of pattern were found then returns (s, "").
 */
func splitSuffix(s string, pattern string) (string, string) {
	l := strings.LastIndex(s, pattern)
	if l > 0 {
		return s[:l], s[l+1:]
	} else {
		return s, ""
	}
}

func findFile(path string, name string) string {
	for {
		path, _ = splitSuffix(path, "/")
		if path == "" {
			return os.DevNull
		}
		p := path + "/" + name
		_, err := os.Stat(p)
		if err == nil {
			return p
		}
	}
}

func dirIndex(file *os.File) string {
	names, err := file.Readdirnames(0)
	if err != nil {
		return ""
	}
	file.Seek(0, 0)

	dir := file.Name()
	if !strings.HasSuffix(dir, "/") {
		dir += "/"
	}

	for _, name := range names {
		if strings.HasPrefix(name, "index") {
			return name
		}
	}

	return ""
}

func readLine(file *os.File, bytes []byte) (string, error) {
	n, err := file.Read(bytes)
	if err != nil {
		return "", err
	}

	s := string(bytes[:n])
	l := strings.IndexByte(s, '\n') + 1

	if l > 0 {
		file.Seek(int64(l - n), 1)
		return s[:l-1], nil
	} else {
		return "", nil
	}
}

func findInterpreter(path string) (bool, []string) {
	intPath := findFile(path, InterpreterName)
	if intPath == "" {
		return false, []string{}
	}

	file, err := os.Open(intPath)
	if err != nil {
		log.Print(err)
		return false, []string{}
	}

	bytes := make([]byte, 256)

	for {
		line, err := readLine(file, bytes)
		if err != nil {
			return false, []string{}
		} else if len(line) == 0 || line[0] == '#' {
			continue
		}

		suffix := strings.SplitN(line, " ", 2)
		if len(suffix) > 0 && strings.HasSuffix(path, suffix[0]) {
			parts := strings.Split(line, " ")
			if len(parts) < 2 {
				log.Print("Error in interpreter file. ", path)
				continue
			} else {
				return strings.HasPrefix(parts[1], "y"),
					parts[2:]
			}
		}
	}
}

func runInterpreter(interpreter []string,
		values map[string][]string, file *os.File) ([]byte, error) {
	dir, base := splitSuffix(file.Name(), "/")
	cmd := exec.Command(interpreter[0])
	cmd.Args = append(interpreter, base)
	cmd.Dir = dir

	l := len(cmd.Env) + len(values) + 1
	env := make([]string, l)
	copy(env, cmd.Env)

	i := len(cmd.Env) + 1
	for name, value := range values {
		env[i] = name + "=" + value[0]
		i++
	}

	cmd.Env = env
	return cmd.Output()
}

func processFile(w http.ResponseWriter, req *http.Request,
		data *TemplateData, file *os.File, fi os.FileInfo) {
	var err error
	var bytes []byte = make([]byte, *maxBytes)
	var n int

	useTemplate, interpreter := findInterpreter(file.Name())

	if len(interpreter) > 0 {
		bytes, err = runInterpreter(interpreter,
				req.URL.Query(), file)

	} else {
		n, err = file.Read(bytes)
	}

	if err != nil {
		log.Print(err)
		io.WriteString(w, "ERROR")
		return
	}

	if useTemplate {
		tmplPath := findFile(file.Name(), TemplateName)
		tmpl, err := template.ParseFiles(tmplPath)
		if err == nil {
			data.Content = string(bytes)
			tmpl.Execute(w, data)
			return
		}
	}

	/* No template/error opening template */
	if len(interpreter) > 0 {
		req.ContentLength = int64(len(bytes))
		w.Write(bytes)
	} else {
		req.ContentLength = fi.Size()
		for {
			_, err = w.Write(bytes[:n])
			if err != nil {
				break
			}
			n, err = file.Read(bytes)
			if err != nil {
				break
			}
		}
	}

}

func handler(w http.ResponseWriter, req *http.Request) {
	var file *os.File
	var err error
	var name string

	log.Print(req.RemoteAddr, " request: ", req.URL.String())

	path := "." + html.EscapeString(req.URL.Path)

	file, err = os.Open(path)
	if err != nil {
		log.Print("404 ", err)
		w.WriteHeader(http.StatusNotFound)
		io.WriteString(w, "404: " + html.EscapeString(req.URL.Path))
		return
	}
	defer file.Close()

	fi, err := file.Stat()
	if err != nil {
		log.Print(err)
		return
	}

	data := new(TemplateData)
	data.Link = req.URL.Path

	if strings.HasPrefix(fi.Name(), "index") {
		path, _ = splitSuffix(path, "/")
		_, name = splitSuffix(path, "/")
		path += "/"
	} else {
		name, _ = splitSuffix(fi.Name(), ".")
	}

	if path == "./" {
		data.Name = *rootName
	} else {
		data.Name = fmt.Sprintf(*nameFormat, name)
	}

	if fi.IsDir() {
		index := dirIndex(file)

		if index == "" {
			w.WriteHeader(http.StatusNotFound)
			io.WriteString(w, "404: " +
				html.EscapeString(req.URL.Path))
			return
		} else if !strings.HasSuffix(path, "/") {
			url := req.URL.Scheme + req.URL.Path +
				"/" + req.URL.RawQuery
			http.Redirect(w, req, url,
				http.StatusMovedPermanently)
		} else {
			path += index
			file, err = os.Open(path)
			if err != nil {
				log.Print(err)
				return
			}
			defer file.Close()
			/* Fall through to process file */
		}
	}

	processFile(w, req, data, file, fi)
}

func main() {
	flag.Parse()

	os.Chdir(*siteRoot)

	http.HandleFunc("/", handler)
	err := http.ListenAndServe(":" + *serverPort, nil)
	if err != nil {
		log.Fatal("ListenAndServe: ", err)
	}
}
<|endoftext|>


<commit_before>package starfish

import (
	"log"
	"testing"
	"time"
)

const (
	TESTVALUE1 = 1
	TESTVALUE2 = 2
	TESTVALUE3 = 3
	TESTVALUE4 = 4
	SCRIPT     = `r>l5(?v~~~/:!|Ou+1Ox:@=?~~~~~~~!
~~l5(?v" "/
 ~;!?l<` // Script used in "BenchmarkScript"
)

var (
	INITIALSTACK = []float64{float64('h'), float64('e'), float64('l'), float64('l'), float64('o'),
		float64(' '), float64('w'), float64('o'), float64('r'), float64('l'), float64('d')} // Stack used in "BenchmarkScript"
)

func runscript(script string, initialstack []float64, compMode bool) *CodeBox {
	cB := NewCodeBox(script, initialstack, compMode)
	now := time.Now()
	for !cB.Swim() {
		if time.Since(now) >= time.Second {
			log.Fatalln("script taking too long...")
		}
	}
	return cB
}

func BenchmarkScript(b *testing.B) {
	for i := 0; i < b.N; i++ {
		b.StopTimer()
		stack := make([]float64, len(INITIALSTACK))
		copy(stack, INITIALSTACK)
		cB := NewCodeBox(SCRIPT, stack, false)
		b.StartTimer()
		for !cB.Swim() {
		}
	}
	log.Println(b.N)
}

func TestStackRegister(t *testing.T) {
	cB := runscript("&;", []float64{TESTVALUE1, TESTVALUE2, TESTVALUE3}, false)
	s := cB.stacks[0]
	if len(s.S) != 2 || s.register != TESTVALUE3 || s.S[0] != TESTVALUE1 || !s.filledRegister {
		t.FailNow()
	}
	s.Register()
	if len(s.S) != 3 || s.S[0] != TESTVALUE1 || s.S[2] != TESTVALUE3 || s.filledRegister {
		t.FailNow()
	}
}

func TestStackExtend(t *testing.T) {
	cB := runscript(":;", []float64{TESTVALUE1, TESTVALUE2}, false)
	s := cB.stacks[0]
	if len(s.S) != 3 || s.S[2] != TESTVALUE2 {
		t.FailNow()
	}
}

func TestStackReverse(t *testing.T) {
	cB := runscript("r;", []float64{TESTVALUE1, TESTVALUE2, TESTVALUE3}, false)
	s := cB.stacks[0]
	if s.S[0] != TESTVALUE3 || s.S[1] != TESTVALUE2 || s.S[2] != TESTVALUE1 {
		t.FailNow()
	}
}

func TestStackSwapTwo(t *testing.T) {
	cB := runscript("$;", []float64{TESTVALUE1, TESTVALUE2, TESTVALUE3}, false)
	s := cB.stacks[0]
	if s.S[0] != TESTVALUE1 || s.S[1] != TESTVALUE3 || s.S[2] != TESTVALUE2 {
		t.FailNow()
	}
}

func TestStackSwapThree(t *testing.T) {
	cB := runscript("@;", []float64{TESTVALUE1, TESTVALUE2, TESTVALUE3, TESTVALUE4}, false)
	s := cB.stacks[0]
	if s.S[0] != TESTVALUE1 || s.S[1] != TESTVALUE4 || s.S[2] != TESTVALUE2 || s.S[3] != TESTVALUE3 {
		t.FailNow()
	}
}

func TestStackShiftLeft(t *testing.T) {
	cB := runscript("{;", []float64{TESTVALUE1, TESTVALUE2, TESTVALUE3, TESTVALUE4}, false)
	s := cB.stacks[0]
	if s.S[0] != TESTVALUE2 || s.S[1] != TESTVALUE3 || s.S[2] != TESTVALUE4 || s.S[3] != TESTVALUE1 {
		t.FailNow()
	}
}

func TestStackShiftRight(t *testing.T) {
	cB := runscript("};", []float64{TESTVALUE1, TESTVALUE2, TESTVALUE3, TESTVALUE4}, false)
	s := cB.stacks[0]
	if s.S[0] != TESTVALUE4 || s.S[1] != TESTVALUE1 || s.S[2] != TESTVALUE2 || s.S[3] != TESTVALUE3 {
		t.FailNow()
	}
}

func TestNewStackCloseStack(t *testing.T) {
	cB := NewCodeBox("[]", []float64{TESTVALUE1, TESTVALUE2, TESTVALUE3, TESTVALUE4, 2}, false)
	cB.Swim()
	s := cB.stacks[0]
	s2 := cB.stacks[1]
	if s.S[0] != TESTVALUE1 || s.S[1] != TESTVALUE2 || s2.S[0] != TESTVALUE3 || s2.S[1] != TESTVALUE4 || len(s.S) != 2 || len(s2.S) != 2 {
		t.FailNow()
	}

	cB.Swim()
	s = cB.stacks[0]
	if s.S[0] != TESTVALUE1 || s.S[1] != TESTVALUE2 || s.S[2] != TESTVALUE3 || s.S[3] != TESTVALUE4 || len(s.S) != 4 {
		t.FailNow()
	}
}

func TestNewStackCloseStackCompatibility(t *testing.T) {
	cB := NewCodeBox("[]", []float64{TESTVALUE1, TESTVALUE2, TESTVALUE3, TESTVALUE4, 2}, true)
	cB.Swim()
	s := cB.stacks[0]
	s2 := cB.stacks[1]
	if s.S[0] != TESTVALUE1 || s.S[1] != TESTVALUE2 || s2.S[1] != TESTVALUE3 || s2.S[0] != TESTVALUE4 || len(s.S) != 2 || len(s2.S) != 2 {
		t.FailNow()
	}

	cB.Swim()
	s = cB.stacks[0]
	if s.S[0] != TESTVALUE1 || s.S[1] != TESTVALUE2 || s.S[2] != TESTVALUE3 || s.S[3] != TESTVALUE4 || len(s.S) != 4 {
		t.FailNow()
	}
}

func TestPrintBox(t *testing.T) {
	cB := NewCodeBox(`"Hello test!";`, []float64{}, false)
	cB.PrintBox()
}

func TestStackLength(t *testing.T) {
	cB := NewCodeBox(";", []float64{TESTVALUE1, TESTVALUE2, TESTVALUE3}, false)
	if cB.StackLength() != 3 {
		t.Fail()
	}
}

func TestStackReturn(t *testing.T) {
	cB := NewCodeBox(";", []float64{TESTVALUE1, TESTVALUE3}, false)
	s := cB.Stack()
	if s[0] != TESTVALUE1 || s[1] != TESTVALUE3 {
		t.Fail()
	}
}

func TestMovement(t *testing.T) {
	cB := NewCodeBox(">;", []float64{}, false)
	cB.Swim()
	if !cB.Swim() {
		t.Fail()
	}

	cB = NewCodeBox("<;", []float64{}, false)
	cB.Swim()
	if !cB.Swim() {
		t.Fail()
	}

	cB = NewCodeBox("^\n;", []float64{}, false)
	cB.Swim()
	if !cB.Swim() {
		t.Fail()
	}

	cB = NewCodeBox("v\n;", []float64{}, false)
	cB.Swim()
	if !cB.Swim() {
		t.Fail()
	}
}
<commit_msg>Expanded tests slightly<commit_after>package starfish

import (
	"log"
	"testing"
	"time"
)

const (
	TESTVALUE1 = 1
	TESTVALUE2 = 2
	TESTVALUE3 = 3
	TESTVALUE4 = 4
	SCRIPT     = `r>l5(?v~~~/:!|Ou+1Ox:@=?~~~~~~~!
~~l5(?v" "/
 ~;!?l<` // Script used in "BenchmarkScript"
)

var (
	INITIALSTACK = []float64{float64('h'), float64('e'), float64('l'), float64('l'), float64('o'),
		float64(' '), float64('w'), float64('o'), float64('r'), float64('l'), float64('d')} // Stack used in "BenchmarkScript"
)

func runscript(script string, initialstack []float64, compMode bool) *CodeBox {
	cB := NewCodeBox(script, initialstack, compMode)
	now := time.Now()
	for !cB.Swim() {
		if time.Since(now) >= time.Second {
			log.Fatalln("script taking too long...")
		}
	}
	return cB
}

func BenchmarkScript(b *testing.B) {
	for i := 0; i < b.N; i++ {
		b.StopTimer()
		stack := make([]float64, len(INITIALSTACK))
		copy(stack, INITIALSTACK)
		cB := NewCodeBox(SCRIPT, stack, false)
		b.StartTimer()
		for !cB.Swim() {
		}
	}
	log.Println(b.N)
}

func TestStackRegister(t *testing.T) {
	cB := runscript("&;", []float64{TESTVALUE1, TESTVALUE2, TESTVALUE3}, false)
	s := cB.stacks[0]
	if len(s.S) != 2 || s.register != TESTVALUE3 || s.S[0] != TESTVALUE1 || !s.filledRegister {
		t.FailNow()
	}
	s.Register()
	if len(s.S) != 3 || s.S[0] != TESTVALUE1 || s.S[2] != TESTVALUE3 || s.filledRegister {
		t.FailNow()
	}
}

func TestStackExtend(t *testing.T) {
	cB := runscript(":;", []float64{TESTVALUE1, TESTVALUE2}, false)
	s := cB.stacks[0]
	if len(s.S) != 3 || s.S[2] != TESTVALUE2 {
		t.FailNow()
	}
}

func TestStackReverse(t *testing.T) {
	cB := runscript("r;", []float64{TESTVALUE1, TESTVALUE2, TESTVALUE3}, false)
	s := cB.stacks[0]
	if s.S[0] != TESTVALUE3 || s.S[1] != TESTVALUE2 || s.S[2] != TESTVALUE1 {
		t.FailNow()
	}
}

func TestStackSwapTwo(t *testing.T) {
	cB := runscript("$;", []float64{TESTVALUE1, TESTVALUE2, TESTVALUE3}, false)
	s := cB.stacks[0]
	if s.S[0] != TESTVALUE1 || s.S[1] != TESTVALUE3 || s.S[2] != TESTVALUE2 {
		t.FailNow()
	}
}

func TestStackSwapThree(t *testing.T) {
	cB := runscript("@;", []float64{TESTVALUE1, TESTVALUE2, TESTVALUE3, TESTVALUE4}, false)
	s := cB.stacks[0]
	if s.S[0] != TESTVALUE1 || s.S[1] != TESTVALUE4 || s.S[2] != TESTVALUE2 || s.S[3] != TESTVALUE3 {
		t.FailNow()
	}
}

func TestStackShiftLeft(t *testing.T) {
	cB := runscript("{;", []float64{TESTVALUE1, TESTVALUE2, TESTVALUE3, TESTVALUE4}, false)
	s := cB.stacks[0]
	if s.S[0] != TESTVALUE2 || s.S[1] != TESTVALUE3 || s.S[2] != TESTVALUE4 || s.S[3] != TESTVALUE1 {
		t.FailNow()
	}
}

func TestStackShiftRight(t *testing.T) {
	cB := runscript("};", []float64{TESTVALUE1, TESTVALUE2, TESTVALUE3, TESTVALUE4}, false)
	s := cB.stacks[0]
	if s.S[0] != TESTVALUE4 || s.S[1] != TESTVALUE1 || s.S[2] != TESTVALUE2 || s.S[3] != TESTVALUE3 {
		t.FailNow()
	}
}

func TestNewStackCloseStack(t *testing.T) {
	cB := NewCodeBox("[]", []float64{TESTVALUE1, TESTVALUE2, TESTVALUE3, TESTVALUE4, 2}, false)
	cB.Swim()
	s := cB.stacks[0]
	s2 := cB.stacks[1]
	if s.S[0] != TESTVALUE1 || s.S[1] != TESTVALUE2 || s2.S[0] != TESTVALUE3 || s2.S[1] != TESTVALUE4 || len(s.S) != 2 || len(s2.S) != 2 {
		t.FailNow()
	}

	cB.Swim()
	s = cB.stacks[0]
	if s.S[0] != TESTVALUE1 || s.S[1] != TESTVALUE2 || s.S[2] != TESTVALUE3 || s.S[3] != TESTVALUE4 || len(s.S) != 4 {
		t.FailNow()
	}
}

func TestNewStackCloseStackCompatibility(t *testing.T) {
	cB := NewCodeBox("[]", []float64{TESTVALUE1, TESTVALUE2, TESTVALUE3, TESTVALUE4, 2}, true)
	cB.Swim()
	s := cB.stacks[0]
	s2 := cB.stacks[1]
	if s.S[0] != TESTVALUE1 || s.S[1] != TESTVALUE2 || s2.S[1] != TESTVALUE3 || s2.S[0] != TESTVALUE4 || len(s.S) != 2 || len(s2.S) != 2 {
		t.FailNow()
	}

	cB.Swim()
	s = cB.stacks[0]
	if s.S[0] != TESTVALUE1 || s.S[1] != TESTVALUE2 || s.S[2] != TESTVALUE3 || s.S[3] != TESTVALUE4 || len(s.S) != 4 {
		t.FailNow()
	}
}

func TestPrintBox(t *testing.T) {
	cB := NewCodeBox(`"Hello test!";`, []float64{}, false)
	cB.PrintBox()
}

func TestStackLength(t *testing.T) {
	cB := NewCodeBox("l;", []float64{TESTVALUE1, TESTVALUE2, TESTVALUE3}, false)
	cB.Swim()
	if cB.Stack()[3] != 3 {
		t.Fail()
	}
}

func TestStackReturn(t *testing.T) {
	cB := NewCodeBox(";", []float64{TESTVALUE1, TESTVALUE3}, false)
	s := cB.Stack()
	if s[0] != TESTVALUE1 || s[1] != TESTVALUE3 {
		t.Fail()
	}
}

func TestMovement(t *testing.T) {
	cB := NewCodeBox(">;", []float64{}, false)
	cB.Swim()
	if !cB.Swim() {
		t.Fail()
	}

	cB = NewCodeBox("<;", []float64{}, false)
	cB.Swim()
	if !cB.Swim() {
		t.Fail()
	}

	cB = NewCodeBox("^\n;", []float64{}, false)
	cB.Swim()
	if !cB.Swim() {
		t.Fail()
	}

	cB = NewCodeBox("v\n;", []float64{}, false)
	cB.Swim()
	if !cB.Swim() {
		t.Fail()
	}

	cB = NewCodeBox("`;\n`", []float64{}, false)
	for i := 0;i < 5;i++ {
		if cB.Swim() {
			t.Fail()
		}
	}
	if !cB.Swim() {
		t.Fail()
	}
}
<|endoftext|>


<commit_before>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package backups

import (
	"bytes"
	"crypto/sha1"
	"encoding/base64"
	"encoding/json"
	"io"
	"os"
	"syscall"
	"time"

	"github.com/juju/errors"
	"github.com/juju/utils/filestorage"

	"github.com/juju/juju/version"
)

// checksumFormat identifies how to interpret the checksum for a backup
// generated with this version of juju.
const checksumFormat = "SHA-1, base64 encoded"

// Origin identifies where a backup archive came from.  While it is
// more about where and Metadata about what and when, that distinction
// does not merit special consideration.  Instead, Origin exists
// separately from Metadata due to its use as an argument when
// requesting the creation of a new backup.
type Origin struct {
	Environment string
	Machine     string
	Hostname    string
	Version     version.Number
}

// UnknownString is a marker value for string fields with unknown values.
const UnknownString = "<unknown>"

// UnknownVersion is a marker value for version fields with unknown values.
var UnknownVersion = version.MustParse("9999.9999.9999")

// UnknownOrigin returns a new backups origin with unknown values.
func UnknownOrigin() Origin {
	return Origin{
		Environment: UnknownString,
		Machine:     UnknownString,
		Hostname:    UnknownString,
		Version:     UnknownVersion,
	}
}

// Metadata contains the metadata for a single state backup archive.
type Metadata struct {
	*filestorage.FileMetadata

	// Started records when the backup was started.
	Started time.Time
	// Finished records when the backup was complete.
	Finished *time.Time
	// Origin identifies where the backup was created.
	Origin Origin
	// Notes is an optional user-supplied annotation.
	Notes string
}

// NewMetadata returns a new Metadata for a state backup archive.  Only
// the start time and the version are set.
func NewMetadata() *Metadata {
	return &Metadata{
		FileMetadata: filestorage.NewMetadata(),
		Started:      time.Now().UTC(),
		Origin: Origin{
			Version: version.Current.Number,
		},
	}
}

// NewMetadataState composes a new backup metadata with its origin
// values set.  The environment UUID comes from state.  The hostname is
// retrieved from the OS.
func NewMetadataState(db DB, machine string) (*Metadata, error) {
	// hostname could be derived from the environment...
	hostname, err := os.Hostname()
	if err != nil {
		// If os.Hostname() is not working, something is woefully wrong.
		// Run for the hills.
		return nil, errors.Annotate(err, "could not get hostname (system unstable?)")
	}

	meta := NewMetadata()
	meta.Origin.Environment = db.EnvironTag().Id()
	meta.Origin.Machine = machine
	meta.Origin.Hostname = hostname
	return meta, nil
}

// MarkComplete populates the remaining metadata values.  The default
// checksum format is used.
func (m *Metadata) MarkComplete(size int64, checksum string) error {
	if size == 0 {
		return errors.New("missing size")
	}
	if checksum == "" {
		return errors.New("missing checksum")
	}
	format := checksumFormat
	finished := time.Now().UTC()

	if err := m.SetFileInfo(size, checksum, format); err != nil {
		return errors.Annotate(err, "unexpected failure")
	}
	m.Finished = &finished

	return nil
}

type flatMetadata struct {
	ID string

	// file storage

	Checksum       string
	ChecksumFormat string
	Size           int64
	Stored         time.Time

	// backup

	Started     time.Time
	Finished    time.Time
	Notes       string
	Environment string
	Machine     string
	Hostname    string
	Version     version.Number
}

// TODO(ericsnow) Move AsJSONBuffer to filestorage.Metadata.

// AsJSONBuffer returns a bytes.Buffer containing the JSON-ified metadata.
func (m *Metadata) AsJSONBuffer() (io.Reader, error) {
	flat := flatMetadata{
		ID: m.ID(),

		Checksum:       m.Checksum(),
		ChecksumFormat: m.ChecksumFormat(),
		Size:           m.Size(),

		Started:     m.Started,
		Notes:       m.Notes,
		Environment: m.Origin.Environment,
		Machine:     m.Origin.Machine,
		Hostname:    m.Origin.Hostname,
		Version:     m.Origin.Version,
	}

	stored := m.Stored()
	if stored != nil {
		flat.Stored = *stored
	}

	if m.Finished != nil {
		flat.Finished = *m.Finished
	}

	var outfile bytes.Buffer
	if err := json.NewEncoder(&outfile).Encode(flat); err != nil {
		return nil, errors.Trace(err)
	}
	return &outfile, nil
}

// NewMetadataJSONReader extracts a new metadata from the JSON file.
func NewMetadataJSONReader(in io.Reader) (*Metadata, error) {
	var flat flatMetadata
	if err := json.NewDecoder(in).Decode(&flat); err != nil {
		return nil, errors.Trace(err)
	}

	meta := NewMetadata()
	meta.SetID(flat.ID)

	err := meta.SetFileInfo(flat.Size, flat.Checksum, flat.ChecksumFormat)
	if err != nil {
		return nil, errors.Trace(err)
	}

	if !flat.Stored.IsZero() {
		meta.SetStored(&flat.Stored)
	}

	meta.Started = flat.Started
	if !flat.Finished.IsZero() {
		meta.Finished = &flat.Finished
	}
	meta.Notes = flat.Notes
	meta.Origin = Origin{
		Environment: flat.Environment,
		Machine:     flat.Machine,
		Hostname:    flat.Hostname,
		Version:     flat.Version,
	}

	return meta, nil
}

// BuildMetadata generates the metadata for a backup archive file.
func BuildMetadata(file *os.File) (*Metadata, error) {

	// Extract the file size.
	fi, err := file.Stat()
	if err != nil {
		return nil, errors.Trace(err)
	}
	size := fi.Size()

	// Extract the timestamp.
	var timestamp *time.Time
	rawstat := fi.Sys()
	if rawstat != nil {
		stat, ok := rawstat.(*syscall.Stat_t)
		if ok {
			ts := time.Unix(int64(stat.Ctim.Sec), 0)
			timestamp = &ts
		}
	}
	if timestamp == nil {
		// Fall back to modification time.
		ts := fi.ModTime()
		timestamp = &ts
	}

	// Get the checksum.
	hasher := sha1.New()
	_, err = io.Copy(hasher, file)
	if err != nil {
		return nil, errors.Trace(err)
	}
	rawsum := hasher.Sum(nil)
	checksum := base64.StdEncoding.EncodeToString(rawsum)

	// Build the metadata.
	meta := NewMetadata()
	err = meta.MarkComplete(size, checksum)
	if err != nil {
		return nil, errors.Trace(err)
	}
	meta.Finished = timestamp
	return meta, nil
}
<commit_msg>*time.Time -> time.Time.<commit_after>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package backups

import (
	"bytes"
	"crypto/sha1"
	"encoding/base64"
	"encoding/json"
	"io"
	"os"
	"syscall"
	"time"

	"github.com/juju/errors"
	"github.com/juju/utils/filestorage"

	"github.com/juju/juju/version"
)

// checksumFormat identifies how to interpret the checksum for a backup
// generated with this version of juju.
const checksumFormat = "SHA-1, base64 encoded"

// Origin identifies where a backup archive came from.  While it is
// more about where and Metadata about what and when, that distinction
// does not merit special consideration.  Instead, Origin exists
// separately from Metadata due to its use as an argument when
// requesting the creation of a new backup.
type Origin struct {
	Environment string
	Machine     string
	Hostname    string
	Version     version.Number
}

// UnknownString is a marker value for string fields with unknown values.
const UnknownString = "<unknown>"

// UnknownVersion is a marker value for version fields with unknown values.
var UnknownVersion = version.MustParse("9999.9999.9999")

// UnknownOrigin returns a new backups origin with unknown values.
func UnknownOrigin() Origin {
	return Origin{
		Environment: UnknownString,
		Machine:     UnknownString,
		Hostname:    UnknownString,
		Version:     UnknownVersion,
	}
}

// Metadata contains the metadata for a single state backup archive.
type Metadata struct {
	*filestorage.FileMetadata

	// Started records when the backup was started.
	Started time.Time
	// Finished records when the backup was complete.
	Finished *time.Time
	// Origin identifies where the backup was created.
	Origin Origin
	// Notes is an optional user-supplied annotation.
	Notes string
}

// NewMetadata returns a new Metadata for a state backup archive.  Only
// the start time and the version are set.
func NewMetadata() *Metadata {
	return &Metadata{
		FileMetadata: filestorage.NewMetadata(),
		Started:      time.Now().UTC(),
		Origin: Origin{
			Version: version.Current.Number,
		},
	}
}

// NewMetadataState composes a new backup metadata with its origin
// values set.  The environment UUID comes from state.  The hostname is
// retrieved from the OS.
func NewMetadataState(db DB, machine string) (*Metadata, error) {
	// hostname could be derived from the environment...
	hostname, err := os.Hostname()
	if err != nil {
		// If os.Hostname() is not working, something is woefully wrong.
		// Run for the hills.
		return nil, errors.Annotate(err, "could not get hostname (system unstable?)")
	}

	meta := NewMetadata()
	meta.Origin.Environment = db.EnvironTag().Id()
	meta.Origin.Machine = machine
	meta.Origin.Hostname = hostname
	return meta, nil
}

// MarkComplete populates the remaining metadata values.  The default
// checksum format is used.
func (m *Metadata) MarkComplete(size int64, checksum string) error {
	if size == 0 {
		return errors.New("missing size")
	}
	if checksum == "" {
		return errors.New("missing checksum")
	}
	format := checksumFormat
	finished := time.Now().UTC()

	if err := m.SetFileInfo(size, checksum, format); err != nil {
		return errors.Annotate(err, "unexpected failure")
	}
	m.Finished = &finished

	return nil
}

type flatMetadata struct {
	ID string

	// file storage

	Checksum       string
	ChecksumFormat string
	Size           int64
	Stored         time.Time

	// backup

	Started     time.Time
	Finished    time.Time
	Notes       string
	Environment string
	Machine     string
	Hostname    string
	Version     version.Number
}

// TODO(ericsnow) Move AsJSONBuffer to filestorage.Metadata.

// AsJSONBuffer returns a bytes.Buffer containing the JSON-ified metadata.
func (m *Metadata) AsJSONBuffer() (io.Reader, error) {
	flat := flatMetadata{
		ID: m.ID(),

		Checksum:       m.Checksum(),
		ChecksumFormat: m.ChecksumFormat(),
		Size:           m.Size(),

		Started:     m.Started,
		Notes:       m.Notes,
		Environment: m.Origin.Environment,
		Machine:     m.Origin.Machine,
		Hostname:    m.Origin.Hostname,
		Version:     m.Origin.Version,
	}

	stored := m.Stored()
	if stored != nil {
		flat.Stored = *stored
	}

	if m.Finished != nil {
		flat.Finished = *m.Finished
	}

	var outfile bytes.Buffer
	if err := json.NewEncoder(&outfile).Encode(flat); err != nil {
		return nil, errors.Trace(err)
	}
	return &outfile, nil
}

// NewMetadataJSONReader extracts a new metadata from the JSON file.
func NewMetadataJSONReader(in io.Reader) (*Metadata, error) {
	var flat flatMetadata
	if err := json.NewDecoder(in).Decode(&flat); err != nil {
		return nil, errors.Trace(err)
	}

	meta := NewMetadata()
	meta.SetID(flat.ID)

	err := meta.SetFileInfo(flat.Size, flat.Checksum, flat.ChecksumFormat)
	if err != nil {
		return nil, errors.Trace(err)
	}

	if !flat.Stored.IsZero() {
		meta.SetStored(&flat.Stored)
	}

	meta.Started = flat.Started
	if !flat.Finished.IsZero() {
		meta.Finished = &flat.Finished
	}
	meta.Notes = flat.Notes
	meta.Origin = Origin{
		Environment: flat.Environment,
		Machine:     flat.Machine,
		Hostname:    flat.Hostname,
		Version:     flat.Version,
	}

	return meta, nil
}

// BuildMetadata generates the metadata for a backup archive file.
func BuildMetadata(file *os.File) (*Metadata, error) {

	// Extract the file size.
	fi, err := file.Stat()
	if err != nil {
		return nil, errors.Trace(err)
	}
	size := fi.Size()

	// Extract the timestamp.
	var timestamp time.Time
	rawstat := fi.Sys()
	if rawstat != nil {
		stat, ok := rawstat.(*syscall.Stat_t)
		if ok {
			timestamp = time.Unix(int64(stat.Ctim.Sec), 0)
		}
	}
	if timestamp.IsZero() {
		// Fall back to modification time.
		timestamp = fi.ModTime()
	}

	// Get the checksum.
	hasher := sha1.New()
	_, err = io.Copy(hasher, file)
	if err != nil {
		return nil, errors.Trace(err)
	}
	rawsum := hasher.Sum(nil)
	checksum := base64.StdEncoding.EncodeToString(rawsum)

	// Build the metadata.
	meta := NewMetadata()
	err = meta.MarkComplete(size, checksum)
	if err != nil {
		return nil, errors.Trace(err)
	}
	meta.Finished = &timestamp
	return meta, nil
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"testing"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/sns"
	"github.com/hashicorp/terraform/helper/acctest"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
)

func TestAccAWSSNSTopicSubscription_basic(t *testing.T) {
	ri := acctest.RandInt()

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAWSSNSTopicSubscriptionDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccAWSSNSTopicSubscriptionConfig(ri),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSSNSTopicExists("aws_sns_topic.test_topic"),
					testAccCheckAWSSNSTopicSubscriptionExists("aws_sns_topic_subscription.test_subscription"),
				),
			},
		},
	})
}

func testAccCheckAWSSNSTopicSubscriptionDestroy(s *terraform.State) error {
	conn := testAccProvider.Meta().(*AWSClient).snsconn

	for _, rs := range s.RootModule().Resources {
		if rs.Type != "aws_sns_topic" {
			continue
		}

		// Try to find key pair
		req := &sns.GetSubscriptionAttributesInput{
			SubscriptionArn: aws.String(rs.Primary.ID),
		}

		_, err := conn.GetSubscriptionAttributes(req)

		if err == nil {
			return fmt.Errorf("Subscription still exists, can't continue.")
		}

		// Verify the error is an API error, not something else
		_, ok := err.(awserr.Error)
		if !ok {
			return err
		}
	}

	return nil
}

func testAccCheckAWSSNSTopicSubscriptionExists(n string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[n]
		if !ok {
			return fmt.Errorf("Not found: %s", n)
		}

		if rs.Primary.ID == "" {
			return fmt.Errorf("No SNS subscription with that ARN exists")
		}

		conn := testAccProvider.Meta().(*AWSClient).snsconn

		params := &sns.GetSubscriptionAttributesInput{
			SubscriptionArn: aws.String(rs.Primary.ID),
		}
		_, err := conn.GetSubscriptionAttributes(params)

		if err != nil {
			return err
		}

		return nil
	}
}

func testAccAWSSNSTopicSubscriptionConfig(i int) string {
	return fmt.Sprintf(`
resource "aws_sns_topic" "test_topic" {
    name = "terraform-test-topic"
}

resource "aws_sqs_queue" "test_queue" {
	name = "terraform-subscription-test-queue-%d"
}

resource "aws_sns_topic_subscription" "test_subscription" {
    topic_arn = "${aws_sns_topic.test_topic.arn}"
    protocol = "sqs"
    endpoint = "${aws_sqs_queue.test_queue.arn}"
}
`, i)
}
<commit_msg>Added unit test for the obfuscateEndpointPassword function<commit_after>package aws

import (
	"fmt"
	"testing"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/sns"
	"github.com/hashicorp/terraform/helper/acctest"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
)

func TestAccAWSSNSTopicSubscription_basic(t *testing.T) {
	ri := acctest.RandInt()

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAWSSNSTopicSubscriptionDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccAWSSNSTopicSubscriptionConfig(ri),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSSNSTopicExists("aws_sns_topic.test_topic"),
					testAccCheckAWSSNSTopicSubscriptionExists("aws_sns_topic_subscription.test_subscription"),
				),
			},
		},
	})
}

func testAccCheckAWSSNSTopicSubscriptionDestroy(s *terraform.State) error {
	conn := testAccProvider.Meta().(*AWSClient).snsconn

	for _, rs := range s.RootModule().Resources {
		if rs.Type != "aws_sns_topic" {
			continue
		}

		// Try to find key pair
		req := &sns.GetSubscriptionAttributesInput{
			SubscriptionArn: aws.String(rs.Primary.ID),
		}

		_, err := conn.GetSubscriptionAttributes(req)

		if err == nil {
			return fmt.Errorf("Subscription still exists, can't continue.")
		}

		// Verify the error is an API error, not something else
		_, ok := err.(awserr.Error)
		if !ok {
			return err
		}
	}

	return nil
}

func testAccCheckAWSSNSTopicSubscriptionExists(n string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[n]
		if !ok {
			return fmt.Errorf("Not found: %s", n)
		}

		if rs.Primary.ID == "" {
			return fmt.Errorf("No SNS subscription with that ARN exists")
		}

		conn := testAccProvider.Meta().(*AWSClient).snsconn

		params := &sns.GetSubscriptionAttributesInput{
			SubscriptionArn: aws.String(rs.Primary.ID),
		}
		_, err := conn.GetSubscriptionAttributes(params)

		if err != nil {
			return err
		}

		return nil
	}
}

func TestObfuscateEndpointPassword(t *testing.T) {
	checks := map[string]string{
		"https://example.com/myroute":                   "https://example.com/myroute",
		"https://username@example.com/myroute":          "https://username@example.com/myroute",
		"https://username:password@example.com/myroute": "https://username:****@example.com/myroute",
	}

	for endpoint, expected := range checks {
		out := obfuscateEndpointPassword(endpoint)

		if expected != out {
			t.Fatalf("Expected %v, got %v", expected, out)
		}
	}
}

func testAccAWSSNSTopicSubscriptionConfig(i int) string {
	return fmt.Sprintf(`
resource "aws_sns_topic" "test_topic" {
    name = "terraform-test-topic"
}

resource "aws_sqs_queue" "test_queue" {
	name = "terraform-subscription-test-queue-%d"
}

resource "aws_sns_topic_subscription" "test_subscription" {
    topic_arn = "${aws_sns_topic.test_topic.arn}"
    protocol = "sqs"
    endpoint = "${aws_sqs_queue.test_queue.arn}"
}
`, i)
}
<|endoftext|>


<commit_before>package process

import (
    "bytes"
    "io"
    "os/exec"
    "github.com/johnny-morrice/pipeline"
)

type Infow io.Writer
type Infor io.Reader
type Pngw io.Writer
type Pngr io.Reader

func PngBuff() (Pngr, Pngw) {
    buff := &bytes.Buffer{}
    return Pngr(buff), Pngw(buff)
}

func InfoBuff() (Infor, Infow) {
    buff := &bytes.Buffer{}
    return Infor(buff), Infow(buff)
}

// Config creates a new Info, given the args, and sends it to stdout.
func Config(stdout Infow, stderr io.Writer, args []string) error {
    config := configbrot(args)
    return runPipeCmd(config, &bytes.Buffer{}, stdout, stderr)
}

// Render sends a new fractal image to the passed stdout pipe, corresponding to the Info
// serialized in stdin.
func Render(stdin Infor, stdout Pngw, stderr io.Writer) error {
    render := renderbrot()
    return runPipeCmd(render, stdin, stdout, stderr)
}

// ConfigRender sends a new fractal image to the passed stdout pipe, corresponding to configbrot's
// processing of the args slice.
func ConfigRender(stdout Pngw, stderr io.Writer, args []string) error {
    config := configbrot(args)
    render := renderbrot()

    pl := pipeline.New(&bytes.Buffer{}, stdout, stderr)
    pl.Chain(config, render)
    return pl.Exec()
}

// Zoom magnifies a section of the Info read from stdin, sending it to stdout.
func Zoom(stdin Infor, stdout Infow, stderr io.Writer, args[]string) error {
    zoom := zoombrot(args)
    return runPipeCmd(zoom, stdin, stdout, stderr)
}

// Zoom reads Info from stdin, and sends a fractal to stdout, returning the magnified Info,
// serialized as an Infor.
func ZoomRender(stdin Infor, stdout Pngw, stderr io.Writer, args []string) (Infor, error) {
    zoomBuff := &bytes.Buffer{}
    zoomerr := Zoom(stdin, zoomBuff, stderr, args)
    if zoomerr != nil {
        return nil, zoomerr
    }

    outbuff := &bytes.Buffer{}
    rendin := io.TeeReader(zoomBuff, outbuff)

    err := Render(rendin, stdout, stderr)

    return outbuff, err
}

func zoombrot(args []string) *exec.Cmd {
    return exec.Command("zoombrot", args...)
}

func configbrot(args []string) *exec.Cmd {
    return exec.Command("configbrot", args...)
}

func renderbrot() *exec.Cmd {
    return exec.Command("renderbrot")
}

func runPipeCmd(cmd *exec.Cmd, stdin io.Reader, stdout, stderr io.Writer) error {
    cmd.Stdin = stdin
    cmd.Stdout = stdout
    cmd.Stderr = stderr
    return cmd.Run()
}<commit_msg>Delete typed streams; add ZoomArgs helper<commit_after>package process

import (
    "bytes"
    "io"
    "fmt"
    "os/exec"
    "github.com/johnny-morrice/pipeline"
    lib "github.com/johnny-morrice/godelbrot/libgodelbrot"
)

// Config creates a new Info, given the args, and sends it to stdout.
func Config(stdout io.Writer, stderr io.Writer, args []string) error {
    config := configbrot(args)
    return runPipeCmd(config, &bytes.Buffer{}, stdout, stderr)
}

// Render sends a new fractal image to the passed stdout pipe, corresponding to the Info
// serialized in stdin.
func Render(stdin io.Reader, stdout io.Writer, stderr io.Writer) error {
    render := renderbrot()
    return runPipeCmd(render, stdin, stdout, stderr)
}

// ConfigRender sends a new fractal image to the passed stdout pipe, corresponding to configbrot's
// processing of the args slice.
func ConfigRender(stdout io.Writer, stderr io.Writer, args []string) error {
    config := configbrot(args)
    render := renderbrot()

    pl := pipeline.New(&bytes.Buffer{}, stdout, stderr)
    pl.Chain(config, render)
    return pl.Exec()
}

// Zoom magnifies a section of the Info read from stdin, sending it to stdout.
func Zoom(stdin io.Reader, stdout io.Writer, stderr io.Writer, args[]string) error {
    zoom := zoombrot(args)
    return runPipeCmd(zoom, stdin, stdout, stderr)
}

// Zoom reads Info from stdin, and sends a fractal to stdout, returning the magnified Info,
// serialized as an io.Reader.
func ZoomRender(stdin io.Reader, stdout io.Writer, stderr io.Writer, args []string) (io.Reader, error) {
    zoomBuff := &bytes.Buffer{}
    zoomerr := Zoom(stdin, zoomBuff, stderr, args)
    if zoomerr != nil {
        return nil, zoomerr
    }

    outbuff := &bytes.Buffer{}
    rendin := io.TeeReader(zoomBuff, outbuff)

    err := Render(rendin, stdout, stderr)

    return outbuff, err
}

func ZoomArgs(target lib.ZoomTarget) []string {
    formal := []string{
        "frames",
        "incprec",
        "reconf",
        "xmax",
        "xmin",
        "ymax",
        "ymin",
    }
    actual := []string{
        fmt.Sprint(target.Frames),
        fmt.Sprint(target.UpPrec),
        fmt.Sprint(target.UpPrec),
        fmt.Sprint(target.Xmin),
        fmt.Sprint(target.Xmax),
        fmt.Sprint(target.Ymin),
        fmt.Sprint(target.Ymax),
    }

    opts := make([]string, len(formal))
    for i, fm := range formal {
        opts[i] = fmt.Sprintf("-%v=%v", fm, actual[i])
    }

    return opts
}

func zoombrot(args []string) *exec.Cmd {
    return exec.Command("zoombrot", args...)
}

func configbrot(args []string) *exec.Cmd {
    return exec.Command("configbrot", args...)
}

func renderbrot() *exec.Cmd {
    return exec.Command("renderbrot")
}

func runPipeCmd(cmd *exec.Cmd, stdin io.Reader, stdout, stderr io.Writer) error {
    cmd.Stdin = stdin
    cmd.Stdout = stdout
    cmd.Stderr = stderr
    return cmd.Run()
}<|endoftext|>


<commit_before>package program

// Version can be requested through the command line with:
//
//     c2go -v
//
// See https://github.com/elliotchance/c2go/wiki/Release-Process
const Version = "v0.24.8 Caesium 2018-06-25"
<commit_msg>Bump version: v0.25.0 Dubnium 2018-06-26<commit_after>package program

// Version can be requested through the command line with:
//
//     c2go -v
//
// See https://github.com/elliotchance/c2go/wiki/Release-Process
const Version = "v0.25.0 Dubnium 2018-06-26"
<|endoftext|>


<commit_before>package project

func (p *Project) GetSystemdUnitOrder() (s string) {
	if p.Systemd_unit_after != "" {
		s = "After=" + p.Systemd_unit_after
	} else {
		s = "After=network.service"
	}
	return
}
<commit_msg>systemd_unit_after ahora también afecta systemd unit requires<commit_after>package project

func (p *Project) GetSystemdUnitOrder() (s string) {
	if p.Systemd_unit_after != "" {
		s = "After=" + p.Systemd_unit_after
		s += "\nRequires=" + p.Systemd_unit_after
	} else {
		s = "After=network.target"
	}
	return
}
<|endoftext|>


<commit_before>package adm

import (
	"bytes"
	"encoding/json"
	"fmt"
	"os"

	"gopkg.in/resty.v0"
)

// Exported functions

// DELETE
func DeleteReq(p string) (*resty.Response, error) {
	return handleRequestOptions(client.R().Delete(p))
}

func DeleteReqBody(body interface{}, p string) (*resty.Response, error) {
	return handleRequestOptions(
		client.R().SetBody(body).SetContentLength(true).Delete(p))
}

// GET
func GetReq(p string) (*resty.Response, error) {
	return handleRequestOptions(client.R().Get(p))
}

// PATCH
func PatchReqBody(body interface{}, p string) (*resty.Response, error) {
	return handleRequestOptions(
		client.R().SetBody(body).SetContentLength(true).Patch(p))
}

// POST
func PostReqBody(body interface{}, p string) (*resty.Response, error) {
	return handleRequestOptions(
		client.R().SetBody(body).SetContentLength(true).Post(p))
}

// PUT
func PutReq(p string) (*resty.Response, error) {
	return handleRequestOptions(client.R().Put(p))
}

// Private functions

func handleRequestOptions(resp *resty.Response, err error) (*resty.Response, error) {
	if err != nil {
		return nil, err
	}

	if resp.StatusCode() >= 300 {
		return resp, fmt.Errorf("Request error: %s", resp.Status())
	}

	if !(async || jobSave) {
		return resp, nil
	}

	var result *proto.Result
	if result, err = decodeResponse(resp); err != nil {
		return nil, err
	}

	if jobSave {
		if result.StatusCode == 202 && result.JobId != "" {
			cache.SaveJob(result.JobId, result.JobType)
		}
	}

	if async {
		asyncWait(result)
	}
	return resp, nil
}

func asyncWait(result *proto.Result) {
	if !async {
		return
	}

	if result.StatusCode == 202 && result.JobId != "" {
		fmt.Fprintf(os.Stderr, "Waiting for job: %s\n", result.JobId)
		_, err := PutReq(fmt.Sprintf("/jobs/%s", result.JobId))
		if err != nil {
			fmt.Fprintf(os.Stderr, "Wait error: %s\n", err.Error())
		}
	}
}

func decodeResponse(resp *resty.Response) (*proto.Result, error) {
	result := proto.Result{}
	decoder := json.NewDecoder(bytes.NewReader(resp.Body()))
	err := decoder.Decode(&result)
	if err != nil {
		return nil, err
	}

	return &result, nil
}

// vim: ts=4 sw=4 sts=4 noet fenc=utf-8 ffs=unix
<commit_msg>Add libadm.PutReqBody()<commit_after>package adm

import (
	"bytes"
	"encoding/json"
	"fmt"
	"os"

	"gopkg.in/resty.v0"
)

// Exported functions

// DELETE
func DeleteReq(p string) (*resty.Response, error) {
	return handleRequestOptions(client.R().Delete(p))
}

func DeleteReqBody(body interface{}, p string) (*resty.Response, error) {
	return handleRequestOptions(
		client.R().SetBody(body).SetContentLength(true).Delete(p))
}

// GET
func GetReq(p string) (*resty.Response, error) {
	return handleRequestOptions(client.R().Get(p))
}

// PATCH
func PatchReqBody(body interface{}, p string) (*resty.Response, error) {
	return handleRequestOptions(
		client.R().SetBody(body).SetContentLength(true).Patch(p))
}

// POST
func PostReqBody(body interface{}, p string) (*resty.Response, error) {
	return handleRequestOptions(
		client.R().SetBody(body).SetContentLength(true).Post(p))
}

// PUT
func PutReq(p string) (*resty.Response, error) {
	return handleRequestOptions(client.R().Put(p))
}

func PutReqBody(body interface{}, p string) (*resty.Response, error) {
	return handleRequestOptions(
		client.R().SetBody(body).SetContentLength(true).Put(p))
}

// Private functions

func handleRequestOptions(resp *resty.Response, err error) (*resty.Response, error) {
	if err != nil {
		return nil, err
	}

	if resp.StatusCode() >= 300 {
		return resp, fmt.Errorf("Request error: %s", resp.Status())
	}

	if !(async || jobSave) {
		return resp, nil
	}

	var result *proto.Result
	if result, err = decodeResponse(resp); err != nil {
		return nil, err
	}

	if jobSave {
		if result.StatusCode == 202 && result.JobId != "" {
			cache.SaveJob(result.JobId, result.JobType)
		}
	}

	if async {
		asyncWait(result)
	}
	return resp, nil
}

func asyncWait(result *proto.Result) {
	if !async {
		return
	}

	if result.StatusCode == 202 && result.JobId != "" {
		fmt.Fprintf(os.Stderr, "Waiting for job: %s\n", result.JobId)
		_, err := PutReq(fmt.Sprintf("/jobs/%s", result.JobId))
		if err != nil {
			fmt.Fprintf(os.Stderr, "Wait error: %s\n", err.Error())
		}
	}
}

func decodeResponse(resp *resty.Response) (*proto.Result, error) {
	result := proto.Result{}
	decoder := json.NewDecoder(bytes.NewReader(resp.Body()))
	err := decoder.Decode(&result)
	if err != nil {
		return nil, err
	}

	return &result, nil
}

// vim: ts=4 sw=4 sts=4 noet fenc=utf-8 ffs=unix
<|endoftext|>


<commit_before>// The MIT License (MIT)
//
// Copyright (c) 2013-2015 Oryx(ossrs)
//
// Permission is hereby granted, free of charge, to any person obtaining a copy of
// this software and associated documentation files (the "Software"), to deal in
// the Software without restriction, including without limitation the rights to
// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
// the Software, and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
// FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
// IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package protocol

import "time"

const (
	// timeout for rtmp.
	HandshakeTimeout   = 2100 * time.Millisecond
	ConnectAppTimeout  = 5000 * time.Millisecond
	IdentifyTimeout    = ConnectAppTimeout
	FmlePublishTimeout = IdentifyTimeout
	PublishRecvTimeout = 10 * time.Second

	// the input cache, to read from network and put in it.
	RtmpInCache = 16
)
<commit_msg>set timeout to 30 for publish.<commit_after>// The MIT License (MIT)
//
// Copyright (c) 2013-2015 Oryx(ossrs)
//
// Permission is hereby granted, free of charge, to any person obtaining a copy of
// this software and associated documentation files (the "Software"), to deal in
// the Software without restriction, including without limitation the rights to
// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
// the Software, and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
// FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
// IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package protocol

import "time"

const (
	// timeout for rtmp.
	HandshakeTimeout   = 2100 * time.Millisecond
	ConnectAppTimeout  = 5000 * time.Millisecond
	IdentifyTimeout    = ConnectAppTimeout
	FmlePublishTimeout = IdentifyTimeout
	PublishRecvTimeout = 30 * time.Second

	// the input cache, to read from network and put in it.
	RtmpInCache = 16
)
<|endoftext|>


<commit_before>// This file is copied from https://github.com/yinghuocho/gotun2socks/blob/master/udp.go

package proxy

import (
	"log"
	"sync"
	"time"

	"github.com/miekg/dns"
)

const COMMON_DNS_PORT = 53

type dnsCacheEntry struct {
	msg *dns.Msg
	exp time.Time
}

type DNSCache struct {
	servers []string
	mutex   sync.Mutex
	storage map[string]*dnsCacheEntry
}

func NewDNSCache() *DNSCache {
	cache := &DNSCache{storage: make(map[string]*dnsCacheEntry)}
	go cache.cleanUp()
	return cache
}

func packUint16(i uint16) []byte { return []byte{byte(i >> 8), byte(i)} }

func cacheKey(q dns.Question) string {
	return string(append([]byte(q.Name), packUint16(q.Qtype)...))
}

func (c *DNSCache) cleanUp() {
	ticker := time.NewTicker(5 * time.Minute)
	defer ticker.Stop()
	for {
		select {
		case <-ticker.C:
			c.mutex.Lock()
			log.Printf("cleaning up dns cache, %v entries", len(c.storage))
			newStorage := make(map[string]*dnsCacheEntry)
			for key, entry := range c.storage {
				if time.Now().Before(entry.exp) {
					newStorage[key] = entry
				}
			}
			c.storage = newStorage
			log.Printf("cleanup done, remaining %v entries", len(c.storage))
			c.mutex.Unlock()
		}
	}
}

func (c *DNSCache) Query(payload []byte) *dns.Msg {
	request := new(dns.Msg)
	e := request.Unpack(payload)
	if e != nil {
		return nil
	}
	if len(request.Question) == 0 {
		return nil
	}

	c.mutex.Lock()
	defer c.mutex.Unlock()
	key := cacheKey(request.Question[0])
	entry := c.storage[key]
	if entry == nil {
		return nil
	}
	if time.Now().After(entry.exp) {
		delete(c.storage, key)
		return nil
	}
	entry.msg.Id = request.Id
	log.Printf("got dns answer with key: %v", key)
	return entry.msg
}

func (c *DNSCache) Store(payload []byte) {
	resp := new(dns.Msg)
	e := resp.Unpack(payload)
	if e != nil {
		return
	}
	if resp.Rcode != dns.RcodeSuccess {
		return
	}
	if len(resp.Question) == 0 || len(resp.Answer) == 0 {
		return
	}

	c.mutex.Lock()
	defer c.mutex.Unlock()
	key := cacheKey(resp.Question[0])
	c.storage[key] = &dnsCacheEntry{
		msg: resp,
		exp: time.Now().Add(time.Duration(resp.Answer[0].Header().Ttl) * time.Second),
	}
	log.Printf("stored dns answer with key: %v, ttl: %v sec", key, resp.Answer[0].Header().Ttl)
}
<commit_msg>remove a log<commit_after>// This file is copied from https://github.com/yinghuocho/gotun2socks/blob/master/udp.go

package proxy

import (
	"log"
	"sync"
	"time"

	"github.com/miekg/dns"
)

const COMMON_DNS_PORT = 53

type dnsCacheEntry struct {
	msg *dns.Msg
	exp time.Time
}

type DNSCache struct {
	servers []string
	mutex   sync.Mutex
	storage map[string]*dnsCacheEntry
}

func NewDNSCache() *DNSCache {
	cache := &DNSCache{storage: make(map[string]*dnsCacheEntry)}
	go cache.cleanUp()
	return cache
}

func packUint16(i uint16) []byte { return []byte{byte(i >> 8), byte(i)} }

func cacheKey(q dns.Question) string {
	return string(append([]byte(q.Name), packUint16(q.Qtype)...))
}

func (c *DNSCache) cleanUp() {
	ticker := time.NewTicker(5 * time.Minute)
	defer ticker.Stop()
	for {
		select {
		case <-ticker.C:
			c.mutex.Lock()
			log.Printf("cleaning up dns cache, %v entries", len(c.storage))
			newStorage := make(map[string]*dnsCacheEntry)
			for key, entry := range c.storage {
				if time.Now().Before(entry.exp) {
					newStorage[key] = entry
				}
			}
			c.storage = newStorage
			log.Printf("cleanup done, remaining %v entries", len(c.storage))
			c.mutex.Unlock()
		}
	}
}

func (c *DNSCache) Query(payload []byte) *dns.Msg {
	request := new(dns.Msg)
	e := request.Unpack(payload)
	if e != nil {
		return nil
	}
	if len(request.Question) == 0 {
		return nil
	}

	c.mutex.Lock()
	defer c.mutex.Unlock()
	key := cacheKey(request.Question[0])
	entry := c.storage[key]
	if entry == nil {
		return nil
	}
	if time.Now().After(entry.exp) {
		delete(c.storage, key)
		return nil
	}
	entry.msg.Id = request.Id
	// log.Printf("got dns answer with key: %v", key)
	return entry.msg
}

func (c *DNSCache) Store(payload []byte) {
	resp := new(dns.Msg)
	e := resp.Unpack(payload)
	if e != nil {
		return
	}
	if resp.Rcode != dns.RcodeSuccess {
		return
	}
	if len(resp.Question) == 0 || len(resp.Answer) == 0 {
		return
	}

	c.mutex.Lock()
	defer c.mutex.Unlock()
	key := cacheKey(resp.Question[0])
	c.storage[key] = &dnsCacheEntry{
		msg: resp,
		exp: time.Now().Add(time.Duration(resp.Answer[0].Header().Ttl) * time.Second),
	}
	log.Printf("stored dns answer with key: %v, ttl: %v sec", key, resp.Answer[0].Header().Ttl)
}
<|endoftext|>


<commit_before>package loader

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	urlModule "net/url"
	"strconv"
	"sync"
	"time"

	"github.com/AxelUser/gowork/errors"
	"github.com/AxelUser/gowork/events"
	"github.com/AxelUser/gowork/models"
)

func createBaseURL(config models.ParserConfig) (string, error) {
	url, err := urlModule.Parse(config.URL)
	if err != nil {
		return "", errors.NewLoadDataError(config.URL, "Could not parse Base URL", err)
	}

	q := url.Query()
	for k, v := range config.Defaults {
		q.Add(k, fmt.Sprintf("%v", v))
	}

	url.RawQuery = q.Encode()
	urlString := url.String()
	return urlString, nil
}

func createURLs(baseURL string, queries []models.ParserQuery) (map[string]string, error) {
	skillURLMap := make(map[string]string)

	for _, query := range queries {
		req, err := http.NewRequest("GET", baseURL, nil)
		if err != nil {
			return nil, errors.NewLoadSkillError(query.Alias, "Could not create request for skill", err)
		}

		q := req.URL.Query()
		q.Add("text", query.Text)

		req.URL.RawQuery = q.Encode()

		skillURL := req.URL.String()

		skillURLMap[query.Alias] = skillURL
	}

	return skillURLMap, nil
}

func getNextPageURL(alias string, url string, nextPage int) (string, error) {
	urlBuild, err := urlModule.Parse(url)
	if err != nil {
		return "", errors.NewLoadSkillError(alias, "Could not create URL for next page", err)
	}

	q := urlBuild.Query()
	q.Set("page", strconv.Itoa(nextPage))
	urlBuild.RawQuery = q.Encode()
	nextPageURL := urlBuild.String()

	return nextPageURL, nil
}

func loadPage(alias string, url string) (*models.VacancySearchPage, error) {
	res, httpErr := http.Get(url)
	if httpErr != nil {
		return nil, errors.NewLoadSkillError(alias, "Could not send GET request", httpErr)
	}
	defer res.Body.Close()

	body, bodyErr := ioutil.ReadAll(res.Body)
	if bodyErr != nil {
		return nil, errors.NewLoadSkillError(alias, "Could not read data from Body", bodyErr)
	}

	var page models.VacancySearchPage
	jsonErr := json.Unmarshal(body, &page)
	if jsonErr != nil {
		return nil, errors.NewLoadSkillError(alias, "Could not unmarshal JSON", jsonErr)
	}

	return &page, nil
}

func loadAllPages(alias string, firstPageURL string, firstPage models.VacancySearchPage) ([]models.VacancySearchPage, error) {
	pages := []models.VacancySearchPage{firstPage}
	pageURL := firstPageURL
	currentPage := &firstPage

	for !isLastPage(*currentPage) {
		pageURL, err := getNextPageURL(alias, pageURL, currentPage.Page+1)
		if err != nil {
			return pages, err
		}

		currentPage, err = loadPage(alias, pageURL)
		if err != nil {
			return pages, err
		}

		pages = append(pages, *currentPage)
	}

	return pages, nil
}

func parseVacancyStats(page models.VacancySearchPage) []models.VacancyStats {
	data := make([]models.VacancyStats, len(page.Items))
	// Handle empty items and log!
	for i, v := range page.Items {
		data[i] = models.NewVacancyStats(v.ID, v.URL, v.Salary.From, v.Salary.To, v.Salary.Currency)
	}

	return data
}

func isLastPage(page models.VacancySearchPage) bool {
	return page.Page >= page.Pages-1
}

func loadDataPerSkillAsync(jobsCh <-chan models.LoaderJob, eventCh chan<- events.DataLoadedEvent, wg *sync.WaitGroup) {
	defer wg.Done()
	for job := range jobsCh {
		pageURL := job.URL
		pageModel, err := loadPage(job.Alias, pageURL)
		if err != nil {
			eventCh <- events.NewDataLoadedEventWithError(job.Alias, job.URL, nil, err)
		} else {
			pages, err := loadAllPages(job.Alias, pageURL, *pageModel)
			var allStats []models.VacancyStats
			for _, page := range pages {
				allStats = append(allStats, parseVacancyStats(page)...)
			}

			eventCh <- events.NewDataLoadedEventWithError(job.Alias, pageURL, allStats, err)
		}
	}
}

func loadAll(urls map[string]string, count int) (map[string][]models.VacancyStats, int, error) {
	all := make(map[string][]models.VacancyStats)
	dataCh := make(chan events.DataLoadedEvent, len(urls))
	jobsCh := make(chan models.LoaderJob, len(urls))
	var wg sync.WaitGroup

	wg.Add(count)
	//create workers for loading vacancies
	for i := 0; i < count; i++ {
		go loadDataPerSkillAsync(jobsCh, dataCh, &wg)
	}

	log.Printf("Workers in pool: %d\n", count)

	for alias, url := range urls {
		jobsCh <- models.NewLoaderJob(alias, url)
	}
	close(jobsCh)

	totalCount := 0
	for i := 0; i < len(urls); i++ {
		event := <-dataCh
		log.Println(event)
		if event.HasData() {
			totalCount += len(event.Data)
			all[event.Skill] = event.Data
		}
	}
	wg.Wait()
	return all, totalCount, nil
}

// Load data from HeadHunter API
func Load(config models.ParserConfig) (map[string][]models.VacancyStats, error) {
	timeStart := time.Now()

	allStats := make(map[string][]models.VacancyStats)

	baseURL, err := createBaseURL(config)
	if err != nil {
		return allStats, err
	}
	urls, err := createURLs(baseURL, config.Queries)
	if err != nil {
		return allStats, err
	}
	log.Printf("Loading vacancies from %s", config.URL)

	allStats, totalCount, err := loadAll(urls, config.WorkersCount)
	if err != nil {
		return nil, err
	}

	elapsed := time.Since(timeStart)
	log.Printf("\nLoaded %d item(s) in %s\n", totalCount, elapsed)

	return allStats, nil
}
<commit_msg>Minor improvements in logging<commit_after>package loader

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	urlModule "net/url"
	"strconv"
	"sync"
	"time"

	"github.com/AxelUser/gowork/errors"
	"github.com/AxelUser/gowork/events"
	"github.com/AxelUser/gowork/models"
)

func createBaseURL(config models.ParserConfig) (string, error) {
	url, err := urlModule.Parse(config.URL)
	if err != nil {
		return "", errors.NewLoadDataError(config.URL, "Could not parse Base URL", err)
	}

	q := url.Query()
	for k, v := range config.Defaults {
		q.Add(k, fmt.Sprintf("%v", v))
	}

	url.RawQuery = q.Encode()
	urlString := url.String()
	return urlString, nil
}

func createURLs(baseURL string, queries []models.ParserQuery) (map[string]string, error) {
	skillURLMap := make(map[string]string)

	for _, query := range queries {
		req, err := http.NewRequest("GET", baseURL, nil)
		if err != nil {
			return nil, errors.NewLoadSkillError(query.Alias, "Could not create request for skill", err)
		}

		q := req.URL.Query()
		q.Add("text", query.Text)

		req.URL.RawQuery = q.Encode()

		skillURL := req.URL.String()

		skillURLMap[query.Alias] = skillURL
	}

	return skillURLMap, nil
}

func getNextPageURL(alias string, url string, nextPage int) (string, error) {
	urlBuild, err := urlModule.Parse(url)
	if err != nil {
		return "", errors.NewLoadSkillError(alias, "Could not create URL for next page", err)
	}

	q := urlBuild.Query()
	q.Set("page", strconv.Itoa(nextPage))
	urlBuild.RawQuery = q.Encode()
	nextPageURL := urlBuild.String()

	return nextPageURL, nil
}

func loadPage(alias string, url string) (*models.VacancySearchPage, error) {
	res, httpErr := http.Get(url)
	if httpErr != nil {
		return nil, errors.NewLoadSkillError(alias, "Could not send GET request", httpErr)
	}
	defer res.Body.Close()

	body, bodyErr := ioutil.ReadAll(res.Body)
	if bodyErr != nil {
		return nil, errors.NewLoadSkillError(alias, "Could not read data from Body", bodyErr)
	}

	var page models.VacancySearchPage
	jsonErr := json.Unmarshal(body, &page)
	if jsonErr != nil {
		return nil, errors.NewLoadSkillError(alias, "Could not unmarshal JSON", jsonErr)
	}

	return &page, nil
}

func loadAllPages(alias string, firstPageURL string, firstPage models.VacancySearchPage) ([]models.VacancySearchPage, error) {
	pages := []models.VacancySearchPage{firstPage}
	pageURL := firstPageURL
	currentPage := &firstPage

	for !isLastPage(*currentPage) {
		pageURL, err := getNextPageURL(alias, pageURL, currentPage.Page+1)
		if err != nil {
			return pages, err
		}

		currentPage, err = loadPage(alias, pageURL)
		if err != nil {
			return pages, err
		}

		pages = append(pages, *currentPage)
	}

	return pages, nil
}

func parseVacancyStats(page models.VacancySearchPage) []models.VacancyStats {
	data := make([]models.VacancyStats, len(page.Items))
	// Handle empty items and log!
	for i, v := range page.Items {
		data[i] = models.NewVacancyStats(v.ID, v.URL, v.Salary.From, v.Salary.To, v.Salary.Currency)
	}

	return data
}

func isLastPage(page models.VacancySearchPage) bool {
	return page.Page >= page.Pages-1
}

func loadDataPerSkillAsync(jobsCh <-chan models.LoaderJob, eventCh chan<- events.DataLoadedEvent, wg *sync.WaitGroup) {
	defer wg.Done()
	for job := range jobsCh {
		pageURL := job.URL
		pageModel, err := loadPage(job.Alias, pageURL)
		if err != nil {
			eventCh <- events.NewDataLoadedEventWithError(job.Alias, job.URL, nil, err)
		} else {
			pages, err := loadAllPages(job.Alias, pageURL, *pageModel)
			var allStats []models.VacancyStats
			for _, page := range pages {
				allStats = append(allStats, parseVacancyStats(page)...)
			}

			eventCh <- events.NewDataLoadedEventWithError(job.Alias, pageURL, allStats, err)
		}
	}
}

func loadAll(urls map[string]string, count int) (map[string][]models.VacancyStats, int, error) {
	all := make(map[string][]models.VacancyStats)
	dataCh := make(chan events.DataLoadedEvent, len(urls))
	jobsCh := make(chan models.LoaderJob, len(urls))
	var wg sync.WaitGroup

	wg.Add(count)
	//create workers for loading vacancies
	for i := 0; i < count; i++ {
		go loadDataPerSkillAsync(jobsCh, dataCh, &wg)
	}

	log.Printf("Workers in pool: %d\n", count)

	for alias, url := range urls {
		jobsCh <- models.NewLoaderJob(alias, url)
	}
	close(jobsCh)

	totalCount := 0
	for i := 0; i < len(urls); i++ {
		event := <-dataCh
		log.Println(event)
		if event.HasData() {
			totalCount += len(event.Data)
			all[event.Skill] = event.Data
		}
	}

	log.Println("Waiting for workers to complete their jobs")
	wg.Wait()

	return all, totalCount, nil
}

// Load data from HeadHunter API
func Load(config models.ParserConfig) (map[string][]models.VacancyStats, error) {
	timeStart := time.Now()

	allStats := make(map[string][]models.VacancyStats)

	baseURL, err := createBaseURL(config)
	if err != nil {
		return allStats, err
	}
	urls, err := createURLs(baseURL, config.Queries)
	if err != nil {
		return allStats, err
	}
	log.Printf("Loading vacancies from %s", config.URL)

	allStats, totalCount, err := loadAll(urls, config.WorkersCount)
	if err != nil {
		return nil, err
	}

	elapsed := time.Since(timeStart)
	log.Printf("Loaded %d item(s) in %s\n", totalCount, elapsed)

	return allStats, nil
}
<|endoftext|>


<commit_before>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ga4gh

import (
	"fmt"
	"testing"

	_ "github.com/golang/glog"
	"github.com/google/go-cmp/cmp"
	"github.com/dgrijalva/jwt-go"
	"github.com/GoogleCloudPlatform/healthcare-federated-access-services/lib/testkeys"
)

const (
	fixedKeyID = "k"
)

func TestNewAccessFromData(t *testing.T) {
	d, j := fakeAccessDataAndJWT(t)

	p, err := NewAccessFromData(d, RS256, testkeys.Default.Private, testkeys.Default.ID)
	if err != nil {
		t.Fatalf("NewAccessFromData(_) failed: %v", err)
	}
	got := p.JWT()

	want := j
	if diff := cmp.Diff(want, got); diff != "" {
		t.Fatalf("NewAccessFromData(%v) returned diff (-want +got):\n%s", d, diff)
	}
}

func TestNewAccessFromJWT(t *testing.T) {
	d, j := fakeAccessDataAndJWT(t)

	p, err := NewAccessFromJWT(j)
	if err != nil {
		t.Fatalf("NewVisaFromJWT(%v) failed: %v", j, err)
	}
	got := p.Data()

	want := d
	if diff := cmp.Diff(want, got, cmp.AllowUnexported(Visa{})); diff != "" {
		t.Fatalf("NewVisaFromJWT(%v) returned diff (-want +got):\n%s", j, diff)
	}
}

func TestAccessJSONFormat(t *testing.T) {
	_, j := fakeAccessDataAndJWT(t)
	got, err := payloadFromJWT(string(j))
	if err != nil {
		t.Fatalf("payloadFromJWT(%v) failed: %v", j, err)
	}
	want := fakeAccessDataJSON()
	if diff := cmp.Diff(jsontxt(want), jsontxt(got), cmp.Transformer("", jsontxtCanonical)); diff != "" {
		t.Fatalf("JSON(%v) returned diff (-want +got):\n%s", j, diff)
	}
}

func TestAccessVerify(t *testing.T) {
	d, _ := fakeAccessDataAndJWT(t)

	p, err := NewAccessFromData(d, RS256, testkeys.Default.Private, testkeys.Default.ID)
	if err != nil {
		t.Fatalf("NewAccessFromData(%v) failed: %v", d, err)
	}

	if err := p.Verify(testkeys.Default.Public); err != nil {
		t.Fatalf("Verify(_) failed: %v", err)
	}
}

func fakeAccessDataAndJWT(t *testing.T) (*AccessData, AccessJWT) {
	t.Helper()

	d := fakeAccessData()
	m := toAccessDataWithVisaJWT(d)
	token := jwt.NewWithClaims(RS256, m)
	token.Header[jwtHeaderKeyID] = testkeys.Default.ID
	signed, err := token.SignedString(testkeys.Default.Private)
	if err != nil {
		t.Fatalf("token.SignedString(_) failed: %v", err)
	}
	j := AccessJWT(signed)

	t.Logf("Data: %#v", d)
	t.Logf("JWT: %v", j)
	t.Logf("You can verify the Data and JWT match on https://jwt.io/")

	return d, j
}

func fakeAccessData() *AccessData {
	return &AccessData{
		StdClaims: StdClaims{
			ID:        "fake-passport-id",
			Subject:   "fake-passport-subject",
			Issuer:    "fake-passport-issuer",
			IssuedAt:  fakeStart(),
			ExpiresAt: fakeEnd(),
		},
		Scope: "openid fake-passport-scope",
	}
}

func fakeAccessDataJSON() string {
	return `{
    "exp": ` + fmt.Sprintf("%v", fakeEnd()) + `,
    "jti": "fake-passport-id",
    "iat": ` + fmt.Sprintf("%v", fakeStart()) + `,
    "iss": "fake-passport-issuer",
    "sub": "fake-passport-subject",
    "scope": "openid fake-passport-scope"
  }`
}
<commit_msg>replace t.Logf with glog.Infof<commit_after>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ga4gh

import (
	"fmt"
	"testing"

	glog "github.com/golang/glog"
	"github.com/google/go-cmp/cmp"
	"github.com/dgrijalva/jwt-go"
	"github.com/GoogleCloudPlatform/healthcare-federated-access-services/lib/testkeys"
)

const (
	fixedKeyID = "k"
)

func TestNewAccessFromData(t *testing.T) {
	d, j := fakeAccessDataAndJWT(t)

	p, err := NewAccessFromData(d, RS256, testkeys.Default.Private, testkeys.Default.ID)
	if err != nil {
		t.Fatalf("NewAccessFromData(_) failed: %v", err)
	}
	got := p.JWT()

	want := j
	if diff := cmp.Diff(want, got); diff != "" {
		t.Fatalf("NewAccessFromData(%v) returned diff (-want +got):\n%s", d, diff)
	}
}

func TestNewAccessFromJWT(t *testing.T) {
	d, j := fakeAccessDataAndJWT(t)

	p, err := NewAccessFromJWT(j)
	if err != nil {
		t.Fatalf("NewVisaFromJWT(%v) failed: %v", j, err)
	}
	got := p.Data()

	want := d
	if diff := cmp.Diff(want, got, cmp.AllowUnexported(Visa{})); diff != "" {
		t.Fatalf("NewVisaFromJWT(%v) returned diff (-want +got):\n%s", j, diff)
	}
}

func TestAccessJSONFormat(t *testing.T) {
	_, j := fakeAccessDataAndJWT(t)
	got, err := payloadFromJWT(string(j))
	if err != nil {
		t.Fatalf("payloadFromJWT(%v) failed: %v", j, err)
	}
	want := fakeAccessDataJSON()
	if diff := cmp.Diff(jsontxt(want), jsontxt(got), cmp.Transformer("", jsontxtCanonical)); diff != "" {
		t.Fatalf("JSON(%v) returned diff (-want +got):\n%s", j, diff)
	}
}

func TestAccessVerify(t *testing.T) {
	d, _ := fakeAccessDataAndJWT(t)

	p, err := NewAccessFromData(d, RS256, testkeys.Default.Private, testkeys.Default.ID)
	if err != nil {
		t.Fatalf("NewAccessFromData(%v) failed: %v", d, err)
	}

	if err := p.Verify(testkeys.Default.Public); err != nil {
		t.Fatalf("Verify(_) failed: %v", err)
	}
}

func fakeAccessDataAndJWT(t *testing.T) (*AccessData, AccessJWT) {
	t.Helper()

	d := fakeAccessData()
	m := toAccessDataWithVisaJWT(d)
	token := jwt.NewWithClaims(RS256, m)
	token.Header[jwtHeaderKeyID] = testkeys.Default.ID
	signed, err := token.SignedString(testkeys.Default.Private)
	if err != nil {
		t.Fatalf("token.SignedString(_) failed: %v", err)
	}
	j := AccessJWT(signed)

	glog.Infof("Data: %#v", d)
	glog.Infof("JWT: %v", j)
	glog.Infof("You can verify the Data and JWT match on https://jwt.io/")

	return d, j
}

func fakeAccessData() *AccessData {
	return &AccessData{
		StdClaims: StdClaims{
			ID:        "fake-passport-id",
			Subject:   "fake-passport-subject",
			Issuer:    "fake-passport-issuer",
			IssuedAt:  fakeStart(),
			ExpiresAt: fakeEnd(),
		},
		Scope: "openid fake-passport-scope",
	}
}

func fakeAccessDataJSON() string {
	return `{
    "exp": ` + fmt.Sprintf("%v", fakeEnd()) + `,
    "jti": "fake-passport-id",
    "iat": ` + fmt.Sprintf("%v", fakeStart()) + `,
    "iss": "fake-passport-issuer",
    "sub": "fake-passport-subject",
    "scope": "openid fake-passport-scope"
  }`
}
<|endoftext|>


<commit_before>package whois

import (
	"net/url"
	"strings"
)

type nrAdapter struct {
	DefaultAdapter
}

func (a *nrAdapter) Prepare(req *Request) error {
	labels := strings.SplitN(req.Query, ".", 2)
	values := url.Values{}
	values.Set("subdomain", labels[0])
	values.Set("tld", labels[1])
	req.URL = "http://cenpac.net.nr/dns/whois.html?" + values.Encode()
	req.Body = nil // Always override existing request body
	return nil
}

func init() {
	BindAdapter(
		&nrAdapter{},
		"cenpac.net.nr",
	)
}
<commit_msg>correct .nr whois server<commit_after>package whois

import (
	"net/url"
	"strings"
)

type nrAdapter struct {
	DefaultAdapter
}

func (a *nrAdapter) Prepare(req *Request) error {
	labels := strings.SplitN(req.Query, ".", 2)
	values := url.Values{}
	values.Set("subdomain", labels[0])
	values.Set("tld", labels[1])
	req.URL = "http://www.cenpac.net.nr/dns/whois.html?" + values.Encode()
	req.Body = nil // Always override existing request body
	return nil
}

func init() {
	BindAdapter(
		&nrAdapter{},
		"cenpac.net.nr",
		"www.cenpac.net.nr",
	)
}
<|endoftext|>


<commit_before>package netlink

import (
	"fmt"
	"log"
	"net"
	"strings"
	"syscall"

	"github.com/vishvananda/netlink/nl"
	"github.com/vishvananda/netns"
)

// IFA_FLAGS is a u32 attribute.
const IFA_FLAGS = 0x8

// AddrAdd will add an IP address to a link device.
// Equivalent to: `ip addr add $addr dev $link`
func AddrAdd(link Link, addr *Addr) error {
	return pkgHandle.AddrAdd(link, addr)
}

// AddrAdd will add an IP address to a link device.
// Equivalent to: `ip addr add $addr dev $link`
func (h *Handle) AddrAdd(link Link, addr *Addr) error {
	req := h.newNetlinkRequest(syscall.RTM_NEWADDR, syscall.NLM_F_CREATE|syscall.NLM_F_EXCL|syscall.NLM_F_ACK)
	return h.addrHandle(link, addr, req)
}

// AddrReplace will replace (or, if not present, add) an IP address on a link device.
// Equivalent to: `ip addr replace $addr dev $link`
func AddrReplace(link Link, addr *Addr) error {
	return pkgHandle.AddrReplace(link, addr)
}

// AddrReplace will replace (or, if not present, add) an IP address on a link device.
// Equivalent to: `ip addr replace $addr dev $link`
func (h *Handle) AddrReplace(link Link, addr *Addr) error {
	req := h.newNetlinkRequest(syscall.RTM_NEWADDR, syscall.NLM_F_CREATE|syscall.NLM_F_REPLACE|syscall.NLM_F_ACK)
	return h.addrHandle(link, addr, req)
}

// AddrDel will delete an IP address from a link device.
// Equivalent to: `ip addr del $addr dev $link`
func AddrDel(link Link, addr *Addr) error {
	return pkgHandle.AddrDel(link, addr)
}

// AddrDel will delete an IP address from a link device.
// Equivalent to: `ip addr del $addr dev $link`
func (h *Handle) AddrDel(link Link, addr *Addr) error {
	req := h.newNetlinkRequest(syscall.RTM_DELADDR, syscall.NLM_F_ACK)
	return h.addrHandle(link, addr, req)
}

func (h *Handle) addrHandle(link Link, addr *Addr, req *nl.NetlinkRequest) error {
	base := link.Attrs()
	if addr.Label != "" && !strings.HasPrefix(addr.Label, base.Name) {
		return fmt.Errorf("label must begin with interface name")
	}
	h.ensureIndex(base)

	family := nl.GetIPFamily(addr.IP)

	msg := nl.NewIfAddrmsg(family)
	msg.Index = uint32(base.Index)
	msg.Scope = uint8(addr.Scope)
	prefixlen, _ := addr.Mask.Size()
	msg.Prefixlen = uint8(prefixlen)
	req.AddData(msg)

	var localAddrData []byte
	if family == FAMILY_V4 {
		localAddrData = addr.IP.To4()
	} else {
		localAddrData = addr.IP.To16()
	}

	localData := nl.NewRtAttr(syscall.IFA_LOCAL, localAddrData)
	req.AddData(localData)
	var peerAddrData []byte
	if addr.Peer != nil {
		if family == FAMILY_V4 {
			peerAddrData = addr.Peer.IP.To4()
		} else {
			peerAddrData = addr.Peer.IP.To16()
		}
	} else {
		peerAddrData = localAddrData
	}

	addressData := nl.NewRtAttr(syscall.IFA_ADDRESS, peerAddrData)
	req.AddData(addressData)

	if addr.Flags != 0 {
		if addr.Flags <= 0xff {
			msg.IfAddrmsg.Flags = uint8(addr.Flags)
		} else {
			b := make([]byte, 4)
			native.PutUint32(b, uint32(addr.Flags))
			flagsData := nl.NewRtAttr(IFA_FLAGS, b)
			req.AddData(flagsData)
		}
	}

	if addr.Broadcast != nil {
		req.AddData(nl.NewRtAttr(syscall.IFA_BROADCAST, addr.Broadcast))
	}

	if addr.Label != "" {
		labelData := nl.NewRtAttr(syscall.IFA_LABEL, nl.ZeroTerminated(addr.Label))
		req.AddData(labelData)
	}

	_, err := req.Execute(syscall.NETLINK_ROUTE, 0)
	return err
}

// AddrList gets a list of IP addresses in the system.
// Equivalent to: `ip addr show`.
// The list can be filtered by link and ip family.
func AddrList(link Link, family int) ([]Addr, error) {
	return pkgHandle.AddrList(link, family)
}

// AddrList gets a list of IP addresses in the system.
// Equivalent to: `ip addr show`.
// The list can be filtered by link and ip family.
func (h *Handle) AddrList(link Link, family int) ([]Addr, error) {
	req := h.newNetlinkRequest(syscall.RTM_GETADDR, syscall.NLM_F_DUMP)
	msg := nl.NewIfInfomsg(family)
	req.AddData(msg)

	msgs, err := req.Execute(syscall.NETLINK_ROUTE, syscall.RTM_NEWADDR)
	if err != nil {
		return nil, err
	}

	indexFilter := 0
	if link != nil {
		base := link.Attrs()
		h.ensureIndex(base)
		indexFilter = base.Index
	}

	var res []Addr
	for _, m := range msgs {
		addr, msgFamily, ifindex, err := parseAddr(m)
		if err != nil {
			return res, err
		}

		if link != nil && ifindex != indexFilter {
			// Ignore messages from other interfaces
			continue
		}

		if family != FAMILY_ALL && msgFamily != family {
			continue
		}

		res = append(res, addr)
	}

	return res, nil
}

func parseAddr(m []byte) (addr Addr, family, index int, err error) {
	msg := nl.DeserializeIfAddrmsg(m)

	family = -1
	index = -1

	attrs, err1 := nl.ParseRouteAttr(m[msg.Len():])
	if err1 != nil {
		err = err1
		return
	}

	family = int(msg.Family)
	index = int(msg.Index)

	var local, dst *net.IPNet
	for _, attr := range attrs {
		switch attr.Attr.Type {
		case syscall.IFA_ADDRESS:
			dst = &net.IPNet{
				IP:   attr.Value,
				Mask: net.CIDRMask(int(msg.Prefixlen), 8*len(attr.Value)),
			}
			addr.Peer = dst
		case syscall.IFA_LOCAL:
			local = &net.IPNet{
				IP:   attr.Value,
				Mask: net.CIDRMask(int(msg.Prefixlen), 8*len(attr.Value)),
			}
			addr.IPNet = local
		case syscall.IFA_LABEL:
			addr.Label = string(attr.Value[:len(attr.Value)-1])
		case IFA_FLAGS:
			addr.Flags = int(native.Uint32(attr.Value[0:4]))
		case nl.IFA_CACHEINFO:
			ci := nl.DeserializeIfaCacheInfo(attr.Value)
			addr.PreferedLft = int(ci.IfaPrefered)
			addr.ValidLft = int(ci.IfaValid)
		}
	}

	// IFA_LOCAL should be there but if not, fall back to IFA_ADDRESS
	if local != nil {
		addr.IPNet = local
	} else {
		addr.IPNet = dst
	}
	addr.Scope = int(msg.Scope)

	return
}

type AddrUpdate struct {
	LinkAddress net.IPNet
	LinkIndex   int
	NewAddr     bool // true=added false=deleted
}

// AddrSubscribe takes a chan down which notifications will be sent
// when addresses change.  Close the 'done' chan to stop subscription.
func AddrSubscribe(ch chan<- AddrUpdate, done <-chan struct{}) error {
	return addrSubscribe(netns.None(), netns.None(), ch, done)
}

// AddrSubscribeAt works like AddrSubscribe plus it allows the caller
// to choose the network namespace in which to subscribe (ns).
func AddrSubscribeAt(ns netns.NsHandle, ch chan<- AddrUpdate, done <-chan struct{}) error {
	return addrSubscribe(ns, netns.None(), ch, done)
}

func addrSubscribe(newNs, curNs netns.NsHandle, ch chan<- AddrUpdate, done <-chan struct{}) error {
	s, err := nl.SubscribeAt(newNs, curNs, syscall.NETLINK_ROUTE, syscall.RTNLGRP_IPV4_IFADDR, syscall.RTNLGRP_IPV6_IFADDR)
	if err != nil {
		return err
	}
	if done != nil {
		go func() {
			<-done
			s.Close()
		}()
	}
	go func() {
		defer close(ch)
		for {
			msgs, err := s.Receive()
			if err != nil {
				log.Printf("netlink.AddrSubscribe: Receive() error: %v", err)
				return
			}
			for _, m := range msgs {
				msgType := m.Header.Type
				if msgType != syscall.RTM_NEWADDR && msgType != syscall.RTM_DELADDR {
					log.Printf("netlink.AddrSubscribe: bad message type: %d", msgType)
					continue
				}

				addr, _, ifindex, err := parseAddr(m.Data)
				if err != nil {
					log.Printf("netlink.AddrSubscribe: could not parse address: %v", err)
					continue
				}

				ch <- AddrUpdate{LinkAddress: *addr.IPNet, LinkIndex: ifindex, NewAddr: msgType == syscall.RTM_NEWADDR}
			}
		}
	}()

	return nil
}
<commit_msg>AddrUpdate: Include flags, scope and lifetimes<commit_after>package netlink

import (
	"fmt"
	"log"
	"net"
	"strings"
	"syscall"

	"github.com/vishvananda/netlink/nl"
	"github.com/vishvananda/netns"
)

// IFA_FLAGS is a u32 attribute.
const IFA_FLAGS = 0x8

// AddrAdd will add an IP address to a link device.
// Equivalent to: `ip addr add $addr dev $link`
func AddrAdd(link Link, addr *Addr) error {
	return pkgHandle.AddrAdd(link, addr)
}

// AddrAdd will add an IP address to a link device.
// Equivalent to: `ip addr add $addr dev $link`
func (h *Handle) AddrAdd(link Link, addr *Addr) error {
	req := h.newNetlinkRequest(syscall.RTM_NEWADDR, syscall.NLM_F_CREATE|syscall.NLM_F_EXCL|syscall.NLM_F_ACK)
	return h.addrHandle(link, addr, req)
}

// AddrReplace will replace (or, if not present, add) an IP address on a link device.
// Equivalent to: `ip addr replace $addr dev $link`
func AddrReplace(link Link, addr *Addr) error {
	return pkgHandle.AddrReplace(link, addr)
}

// AddrReplace will replace (or, if not present, add) an IP address on a link device.
// Equivalent to: `ip addr replace $addr dev $link`
func (h *Handle) AddrReplace(link Link, addr *Addr) error {
	req := h.newNetlinkRequest(syscall.RTM_NEWADDR, syscall.NLM_F_CREATE|syscall.NLM_F_REPLACE|syscall.NLM_F_ACK)
	return h.addrHandle(link, addr, req)
}

// AddrDel will delete an IP address from a link device.
// Equivalent to: `ip addr del $addr dev $link`
func AddrDel(link Link, addr *Addr) error {
	return pkgHandle.AddrDel(link, addr)
}

// AddrDel will delete an IP address from a link device.
// Equivalent to: `ip addr del $addr dev $link`
func (h *Handle) AddrDel(link Link, addr *Addr) error {
	req := h.newNetlinkRequest(syscall.RTM_DELADDR, syscall.NLM_F_ACK)
	return h.addrHandle(link, addr, req)
}

func (h *Handle) addrHandle(link Link, addr *Addr, req *nl.NetlinkRequest) error {
	base := link.Attrs()
	if addr.Label != "" && !strings.HasPrefix(addr.Label, base.Name) {
		return fmt.Errorf("label must begin with interface name")
	}
	h.ensureIndex(base)

	family := nl.GetIPFamily(addr.IP)

	msg := nl.NewIfAddrmsg(family)
	msg.Index = uint32(base.Index)
	msg.Scope = uint8(addr.Scope)
	prefixlen, _ := addr.Mask.Size()
	msg.Prefixlen = uint8(prefixlen)
	req.AddData(msg)

	var localAddrData []byte
	if family == FAMILY_V4 {
		localAddrData = addr.IP.To4()
	} else {
		localAddrData = addr.IP.To16()
	}

	localData := nl.NewRtAttr(syscall.IFA_LOCAL, localAddrData)
	req.AddData(localData)
	var peerAddrData []byte
	if addr.Peer != nil {
		if family == FAMILY_V4 {
			peerAddrData = addr.Peer.IP.To4()
		} else {
			peerAddrData = addr.Peer.IP.To16()
		}
	} else {
		peerAddrData = localAddrData
	}

	addressData := nl.NewRtAttr(syscall.IFA_ADDRESS, peerAddrData)
	req.AddData(addressData)

	if addr.Flags != 0 {
		if addr.Flags <= 0xff {
			msg.IfAddrmsg.Flags = uint8(addr.Flags)
		} else {
			b := make([]byte, 4)
			native.PutUint32(b, uint32(addr.Flags))
			flagsData := nl.NewRtAttr(IFA_FLAGS, b)
			req.AddData(flagsData)
		}
	}

	if addr.Broadcast != nil {
		req.AddData(nl.NewRtAttr(syscall.IFA_BROADCAST, addr.Broadcast))
	}

	if addr.Label != "" {
		labelData := nl.NewRtAttr(syscall.IFA_LABEL, nl.ZeroTerminated(addr.Label))
		req.AddData(labelData)
	}

	_, err := req.Execute(syscall.NETLINK_ROUTE, 0)
	return err
}

// AddrList gets a list of IP addresses in the system.
// Equivalent to: `ip addr show`.
// The list can be filtered by link and ip family.
func AddrList(link Link, family int) ([]Addr, error) {
	return pkgHandle.AddrList(link, family)
}

// AddrList gets a list of IP addresses in the system.
// Equivalent to: `ip addr show`.
// The list can be filtered by link and ip family.
func (h *Handle) AddrList(link Link, family int) ([]Addr, error) {
	req := h.newNetlinkRequest(syscall.RTM_GETADDR, syscall.NLM_F_DUMP)
	msg := nl.NewIfInfomsg(family)
	req.AddData(msg)

	msgs, err := req.Execute(syscall.NETLINK_ROUTE, syscall.RTM_NEWADDR)
	if err != nil {
		return nil, err
	}

	indexFilter := 0
	if link != nil {
		base := link.Attrs()
		h.ensureIndex(base)
		indexFilter = base.Index
	}

	var res []Addr
	for _, m := range msgs {
		addr, msgFamily, ifindex, err := parseAddr(m)
		if err != nil {
			return res, err
		}

		if link != nil && ifindex != indexFilter {
			// Ignore messages from other interfaces
			continue
		}

		if family != FAMILY_ALL && msgFamily != family {
			continue
		}

		res = append(res, addr)
	}

	return res, nil
}

func parseAddr(m []byte) (addr Addr, family, index int, err error) {
	msg := nl.DeserializeIfAddrmsg(m)

	family = -1
	index = -1

	attrs, err1 := nl.ParseRouteAttr(m[msg.Len():])
	if err1 != nil {
		err = err1
		return
	}

	family = int(msg.Family)
	index = int(msg.Index)

	var local, dst *net.IPNet
	for _, attr := range attrs {
		switch attr.Attr.Type {
		case syscall.IFA_ADDRESS:
			dst = &net.IPNet{
				IP:   attr.Value,
				Mask: net.CIDRMask(int(msg.Prefixlen), 8*len(attr.Value)),
			}
			addr.Peer = dst
		case syscall.IFA_LOCAL:
			local = &net.IPNet{
				IP:   attr.Value,
				Mask: net.CIDRMask(int(msg.Prefixlen), 8*len(attr.Value)),
			}
			addr.IPNet = local
		case syscall.IFA_LABEL:
			addr.Label = string(attr.Value[:len(attr.Value)-1])
		case IFA_FLAGS:
			addr.Flags = int(native.Uint32(attr.Value[0:4]))
		case nl.IFA_CACHEINFO:
			ci := nl.DeserializeIfaCacheInfo(attr.Value)
			addr.PreferedLft = int(ci.IfaPrefered)
			addr.ValidLft = int(ci.IfaValid)
		}
	}

	// IFA_LOCAL should be there but if not, fall back to IFA_ADDRESS
	if local != nil {
		addr.IPNet = local
	} else {
		addr.IPNet = dst
	}
	addr.Scope = int(msg.Scope)

	return
}

type AddrUpdate struct {
	LinkAddress net.IPNet
	LinkIndex   int
	Flags       int
	Scope       int
	PreferedLft int
	ValidLft    int
	NewAddr     bool // true=added false=deleted
}

// AddrSubscribe takes a chan down which notifications will be sent
// when addresses change.  Close the 'done' chan to stop subscription.
func AddrSubscribe(ch chan<- AddrUpdate, done <-chan struct{}) error {
	return addrSubscribe(netns.None(), netns.None(), ch, done)
}

// AddrSubscribeAt works like AddrSubscribe plus it allows the caller
// to choose the network namespace in which to subscribe (ns).
func AddrSubscribeAt(ns netns.NsHandle, ch chan<- AddrUpdate, done <-chan struct{}) error {
	return addrSubscribe(ns, netns.None(), ch, done)
}

func addrSubscribe(newNs, curNs netns.NsHandle, ch chan<- AddrUpdate, done <-chan struct{}) error {
	s, err := nl.SubscribeAt(newNs, curNs, syscall.NETLINK_ROUTE, syscall.RTNLGRP_IPV4_IFADDR, syscall.RTNLGRP_IPV6_IFADDR)
	if err != nil {
		return err
	}
	if done != nil {
		go func() {
			<-done
			s.Close()
		}()
	}
	go func() {
		defer close(ch)
		for {
			msgs, err := s.Receive()
			if err != nil {
				log.Printf("netlink.AddrSubscribe: Receive() error: %v", err)
				return
			}
			for _, m := range msgs {
				msgType := m.Header.Type
				if msgType != syscall.RTM_NEWADDR && msgType != syscall.RTM_DELADDR {
					log.Printf("netlink.AddrSubscribe: bad message type: %d", msgType)
					continue
				}

				addr, _, ifindex, err := parseAddr(m.Data)
				if err != nil {
					log.Printf("netlink.AddrSubscribe: could not parse address: %v", err)
					continue
				}

				ch <- AddrUpdate{LinkAddress: *addr.IPNet,
					LinkIndex:   ifindex,
					NewAddr:     msgType == syscall.RTM_NEWADDR,
					Flags:       addr.Flags,
					Scope:       addr.Scope,
					PreferedLft: addr.PreferedLft,
					ValidLft:    addr.ValidLft}
			}
		}
	}()

	return nil
}
<|endoftext|>