<commit_before>// gowebcrawler is a concurrent Web Crawler that generates a JSON sitemap for a given root URL
package gowebcrawler

import (
	"encoding/json"
	"fmt"
	"github.com/PuerkitoBio/goquery"
	"net/http"
	"strings"
)

// A Page represents a web page's relation to other pages and the
// data needed to make a site map showing assets it depends on
type Page struct {
	Url      string
	Assets   []string
	Links    []string
	Children map[string]*Page
	parent   *Page
}

type Parser interface {
	Parse(string) (links []string, assets []string, err error)
}

// UrlParser implements Parser to extract relevant data from a page at a given URL
type UrlParser struct{}

type Crawler interface {
	Crawl(string, parser Parser) ([]byte, error)
}

// WebCrawler implements Crawler and generates a JSON site map from
// a starting domain. It takes care to not crawl other domains or
// get the same page more than once. Also supports a FetchLimit to limit
// the amount of total fetches made.
type WebCrawler struct {
	Parser     *UrlParser
	RootUrl    string
	FetchLimit int
}

type PageMessage struct {
	Page  *Page
	Error error
	Url   string
}

// Starts crawling from a given URL
func (w WebCrawler) Crawl(url string) ([]byte, error) {
	c := make(chan *PageMessage)

	// Make a slice of errors to append errors to
	// TODO: Make use of these or get rid of them
	var errors []error

	url = getAbsoluteUrl(w.RootUrl, url)
	page, err := w.fetchPage(nil, url)

	if err != nil {
		return nil, fmt.Errorf("%v: %v", err, url)
	}

	// Mark root url as requested and set the root page
	requestedUrls := make(map[string]bool)
	requestedUrls[url] = true
	rootPage := page

	go func() {
		c <- &PageMessage{Page: page, Url: url}
	}()

	for waiting := 1; waiting > 0; waiting-- {
		pageMsg := <-c

		if pageMsg.Error != nil {
			errors = append(errors, fmt.Errorf("%v: %v", pageMsg.Error, pageMsg.Url))
			continue
		}

		page := pageMsg.Page

		if page.parent != nil {
			page.parent.Children[page.Url] = page
		}

		// We've hit the fetch limit, don't send fetch any more but finish processing the ones in flight
		if w.FetchLimit != 0 && len(requestedUrls) >= w.FetchLimit {
			continue
		}

		// Fetch pages in goroutines without repeating fetches
		for _, l := range page.Links {
			l = getAbsoluteUrl(w.RootUrl, l)
			if requestedUrls[l] != true {
				// Mark as requested, and let the loop know to wait for one more
				requestedUrls[l] = true
				waiting++
				go func(link string) {
					result, err := w.fetchPage(page, link)
					c <- &PageMessage{Page: result, Error: err, Url: link}
				}(l)
			}
		}
	}

	b, jErr := json.MarshalIndent(rootPage, "", "  ")
	if jErr != nil {
		return nil, fmt.Errorf("Error generating JSON Site Map: %s", jErr)
	}

	return b, nil
}

func getAbsoluteUrl(rootUrl string, url string) string {
	if strings.HasPrefix(url, "/") && !strings.HasPrefix(url, "//") {
		return fmt.Sprint(rootUrl, url)
	}
	return url
}

// Fetches a page from it's parent and an absolute URL
func (w WebCrawler) fetchPage(parent *Page, url string) (*Page, error) {
	if !strings.HasPrefix(url, w.RootUrl) {
		return nil, fmt.Errorf("%s", "Url invalid or outside of allowed domain")
	}

	links, assets, err := w.Parser.Parse(url)
	if err != nil {
		return nil, err
	}

	page := Page{
		Url:      url,
		Assets:   assets,
		Links:    links,
		Children: make(map[string]*Page),
		parent:   parent,
	}

	return &page, nil
}

// Gets slices of links and assets from a goquery.Document
func GetAttributesFromDocument(doc *goquery.Document) (links []string, assets []string) {
	// Links
	links = doc.Find("a[href]").Map(func(_ int, s *goquery.Selection) string {
		href, _ := s.Attr("href")
		return href
	})

	// CSS and other "link" elements
	assets = doc.Find("link[href]").Map(func(i int, s *goquery.Selection) string {
		href, _ := s.Attr("href")
		return href
	})

	//Anything with the "src" attribute (media or scripts)
	assets = append(
		assets,
		doc.Find("[src]").Map(func(i int, s *goquery.Selection) string {
			src, _ := s.Attr("src")
			return src
		})...)

	return links, assets
}

// Grabs links and assets from a page at a URL
func (u UrlParser) Parse(url string) (links []string, assets []string, err error) {
	res, err := http.Get(url)
	if err != nil {
		return nil, nil, err
	}
	if res.StatusCode != 200 {
		return nil, nil, fmt.Errorf("Got a %d status code when getting URL [%s]", res.StatusCode, url)
	}

	doc, err := goquery.NewDocumentFromResponse(res)
	if err != nil {
		return nil, nil, err
	}

	links, assets = GetAttributesFromDocument(doc)
	return links, assets, nil
}
<commit_msg>Fix some comments.<commit_after>// gowebcrawler is a concurrent Web Crawler that generates a JSON sitemap for a given root URL
package gowebcrawler

import (
	"encoding/json"
	"fmt"
	"github.com/PuerkitoBio/goquery"
	"net/http"
	"strings"
)

// A Page represents a web page's relation to other pages and the
// data needed to make a site map showing assets it depends on
type Page struct {
	Url      string
	Assets   []string
	Links    []string
	Children map[string]*Page
	parent   *Page
}

type Parser interface {
	Parse(string) (links []string, assets []string, err error)
}

// UrlParser implements Parser to extract relevant data from a page at a given URL
type UrlParser struct{}

type Crawler interface {
	Crawl(string, parser Parser) ([]byte, error)
}

// WebCrawler implements Crawler and generates a JSON site map from
// a starting domain and path. It takes care to not crawl other domains or
// get the same page more than once. Also supports a FetchLimit to limit
// total fetches made.
type WebCrawler struct {
	Parser     *UrlParser
	RootUrl    string
	FetchLimit int
}

type PageMessage struct {
	Page  *Page
	Error error
	Url   string
}

// Starts crawling from a given URL or path.
func (w WebCrawler) Crawl(url string) ([]byte, error) {
	c := make(chan *PageMessage)

	// Make a slice of errors to append errors to
	// TODO: Make use of these or get rid of them
	var errors []error

	url = getAbsoluteUrl(w.RootUrl, url)
	page, err := w.fetchPage(nil, url)

	if err != nil {
		return nil, fmt.Errorf("%v: %v", err, url)
	}

	// Mark root url as requested and set the root page
	requestedUrls := make(map[string]bool)
	requestedUrls[url] = true
	rootPage := page

	go func() {
		c <- &PageMessage{Page: page, Url: url}
	}()

	for waiting := 1; waiting > 0; waiting-- {
		pageMsg := <-c

		if pageMsg.Error != nil {
			errors = append(errors, fmt.Errorf("%v: %v", pageMsg.Error, pageMsg.Url))
			continue
		}

		page := pageMsg.Page

		if page.parent != nil {
			page.parent.Children[page.Url] = page
		}

		// We've hit the fetch limit, don't fetch any more but finish processing the ones in flight
		if w.FetchLimit != 0 && len(requestedUrls) >= w.FetchLimit {
			continue
		}

		// Fetch pages in goroutines without repeating any
		for _, l := range page.Links {
			l = getAbsoluteUrl(w.RootUrl, l)
			if requestedUrls[l] != true {
				// Mark as requested, and let the loop know to wait for one more
				requestedUrls[l] = true
				waiting++
				go func(link string) {
					result, err := w.fetchPage(page, link)
					c <- &PageMessage{Page: result, Error: err, Url: link}
				}(l)
			}
		}
	}

	b, jErr := json.MarshalIndent(rootPage, "", "  ")
	if jErr != nil {
		return nil, fmt.Errorf("Error generating JSON Site Map: %s", jErr)
	}

	return b, nil
}

func getAbsoluteUrl(rootUrl string, url string) string {
	if strings.HasPrefix(url, "/") && !strings.HasPrefix(url, "//") {
		return fmt.Sprint(rootUrl, url)
	}
	return url
}

// Fetches a page from it's parent and an absolute URL
func (w WebCrawler) fetchPage(parent *Page, url string) (*Page, error) {
	if !strings.HasPrefix(url, w.RootUrl) {
		return nil, fmt.Errorf("%s", "Url invalid or outside of allowed domain")
	}

	links, assets, err := w.Parser.Parse(url)
	if err != nil {
		return nil, err
	}

	page := Page{
		Url:      url,
		Assets:   assets,
		Links:    links,
		Children: make(map[string]*Page),
		parent:   parent,
	}

	return &page, nil
}

// Gets slices of links and assets from a goquery.Document
func GetAttributesFromDocument(doc *goquery.Document) (links []string, assets []string) {
	// Links
	links = doc.Find("a[href]").Map(func(_ int, s *goquery.Selection) string {
		href, _ := s.Attr("href")
		return href
	})

	// CSS and other "link" elements
	assets = doc.Find("link[href]").Map(func(i int, s *goquery.Selection) string {
		href, _ := s.Attr("href")
		return href
	})

	//Anything with the "src" attribute (media or scripts)
	assets = append(
		assets,
		doc.Find("[src]").Map(func(i int, s *goquery.Selection) string {
			src, _ := s.Attr("src")
			return src
		})...)

	return links, assets
}

// Grabs links and assets from a page at a URL
func (u UrlParser) Parse(url string) (links []string, assets []string, err error) {
	res, err := http.Get(url)
	if err != nil {
		return nil, nil, err
	}
	if res.StatusCode != 200 {
		return nil, nil, fmt.Errorf("Got a %d status code when getting URL [%s]", res.StatusCode, url)
	}

	doc, err := goquery.NewDocumentFromResponse(res)
	if err != nil {
		return nil, nil, err
	}

	links, assets = GetAttributesFromDocument(doc)
	return links, assets, nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2017, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

package main

import (
	"errors"
	"fmt"
	"go/ast"
	"go/importer"
	"go/parser"
	"go/printer"
	"go/token"
	"go/types"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"text/template"
)

const mainFile = "goreduce_main.go"

var (
	mainTmpl = template.Must(template.New("main").Parse(`package main

func main() {
	{{ . }}()
}
`))
	rawPrinter = printer.Config{Mode: printer.RawFormat}
)

type reducer struct {
	dir     string
	logOut  io.Writer
	matchRe *regexp.Regexp

	fset     *token.FileSet
	pkg      *ast.Package
	files    []*ast.File
	file     *ast.File
	funcDecl *ast.FuncDecl
	origMain *ast.FuncDecl

	tconf types.Config
	info  *types.Info

	numUses map[types.Object]int

	outBin  string
	goArgs  []string
	dstFile *os.File

	didChange bool

	walker
}

var errNoReduction = fmt.Errorf("could not reduce program")

func reduce(dir, funcName, match string, logOut io.Writer, bflags ...string) error {
	r := &reducer{dir: dir, logOut: logOut}
	tdir, err := ioutil.TempDir("", "goreduce")
	if err != nil {
		return err
	}
	defer os.RemoveAll(tdir)
	r.tconf.Importer = importer.Default()
	if r.matchRe, err = regexp.Compile(match); err != nil {
		return err
	}
	r.fset = token.NewFileSet()
	pkgs, err := parser.ParseDir(r.fset, r.dir, nil, 0)
	if err != nil {
		return err
	}
	if len(pkgs) != 1 {
		return fmt.Errorf("expected 1 package, got %d", len(pkgs))
	}
	for _, pkg := range pkgs {
		r.pkg = pkg
	}
	for _, file := range r.pkg.Files {
		r.files = append(r.files, file)
	}
	r.file, r.funcDecl = findFunc(r.files, funcName)
	if r.file == nil {
		return fmt.Errorf("top-level func %s does not exist", funcName)
	}
	tfnames := make([]string, 0, len(r.files)+1)
	for _, file := range r.files {
		fname := r.fset.Position(file.Pos()).Filename
		tfname := filepath.Join(tdir, filepath.Base(fname))
		f, err := os.Create(tfname)
		if err != nil {
			return nil
		}
		if fd := delFunc(file, "main"); fd != nil && file == r.file {
			r.origMain = fd
		}
		file.Name.Name = "main"
		if err := rawPrinter.Fprint(f, r.fset, file); err != nil {
			return err
		}
		if file == r.file {
			r.dstFile = f
			defer r.dstFile.Close()
		} else if err := f.Close(); err != nil {
			return err
		}
		tfnames = append(tfnames, tfname)
	}
	mfname := filepath.Join(tdir, mainFile)
	mf, err := os.Create(mfname)
	if err != nil {
		return err
	}
	if err := mainTmpl.Execute(mf, funcName); err != nil {
		return err
	}
	if err := mf.Close(); err != nil {
		return err
	}
	tfnames = append(tfnames, mfname)
	r.outBin = filepath.Join(tdir, "bin")
	r.goArgs = []string{"build", "-o", r.outBin}
	r.goArgs = append(r.goArgs, buildFlags...)
	r.goArgs = append(r.goArgs, bflags...)
	r.goArgs = append(r.goArgs, tfnames...)
	// Check that the output matches before we apply any changes
	if err := r.checkRun(); err != nil {
		return err
	}
	if anyChanges := r.reduceLoop(); !anyChanges {
		return errNoReduction
	}
	fname := r.fset.Position(r.file.Pos()).Filename
	f, err := os.Create(fname)
	if err != nil {
		return err
	}
	r.file.Name.Name = r.pkg.Name
	if r.origMain != nil {
		r.file.Decls = append(r.file.Decls, r.origMain)
	}
	if err := printer.Fprint(f, r.fset, r.file); err != nil {
		return err
	}
	return f.Close()
}

func (r *reducer) logChange(node ast.Node, format string, a ...interface{}) {
	if *verbose {
		pos := r.fset.Position(node.Pos())
		fmt.Fprintf(r.logOut, "%s:%d: %s\n", pos.Filename, pos.Line,
			fmt.Sprintf(format, a...))
	}
}

func (r *reducer) checkRun() error {
	if err := r.buildAndRun(); err == nil {
		return fmt.Errorf("expected an error to occur")
	} else if s := err.Error(); !r.matchRe.MatchString(s) {
		return fmt.Errorf("error does not match:\n%s", s)
	}
	return nil
}

func (r *reducer) okChange() bool {
	if r.didChange {
		return false
	}
	// go/types catches most compile errors before writing
	// to disk and running the go tool. Since quite a lot of
	// changes are nonsensical, this is often a big win.
	if _, err := r.tconf.Check(r.dir, r.fset, r.files, nil); err != nil {
		if terr, ok := err.(types.Error); ok && terr.Soft {
			println("unexpected go/types soft error: " + terr.Msg)
		}
		return false
	}
	if err := r.dstFile.Truncate(0); err != nil {
		return false
	}
	if _, err := r.dstFile.Seek(0, 0); err != nil {
		return false
	}
	if err := rawPrinter.Fprint(r.dstFile, r.fset, r.file); err != nil {
		return false
	}
	if err := r.checkRun(); err != nil {
		return false
	}
	// Reduction worked
	r.didChange = true
	return true
}

func (r *reducer) reduceLoop() (anyChanges bool) {
	r.info = &types.Info{
		Defs: make(map[*ast.Ident]types.Object),
		Uses: make(map[*ast.Ident]types.Object),
	}
	for {
		if _, err := r.tconf.Check(r.dir, r.fset, r.files, r.info); err != nil {
			panic("types.Check should not error here")
		}
		r.fillUses()
		r.didChange = false
		r.walk(r.file, r.reduceNode)
		if !r.didChange {
			return
		}
		anyChanges = true
	}
}

func (r *reducer) fillUses() {
	r.numUses = make(map[types.Object]int)
	for _, obj := range r.info.Uses {
		if pkg := obj.Pkg(); pkg == nil || pkg.Name() != "main" {
			// builtin or declared outside of our pkg
			continue
		}
		switch obj.(type) {
		case *types.PkgName, *types.Var:
			r.numUses[obj]++
		}
	}
}

func findFunc(files []*ast.File, name string) (*ast.File, *ast.FuncDecl) {
	for _, file := range files {
		for _, decl := range file.Decls {
			fd, ok := decl.(*ast.FuncDecl)
			if ok && fd.Name.Name == name {
				return file, fd
			}
		}
	}
	return nil, nil
}

func delFunc(file *ast.File, name string) *ast.FuncDecl {
	for i, decl := range file.Decls {
		fd, ok := decl.(*ast.FuncDecl)
		if ok && fd.Name.Name == name {
			file.Decls = append(file.Decls[:i], file.Decls[i+1:]...)
			return fd
		}
	}
	return nil
}

func (r *reducer) buildAndRun() error {
	cmd := exec.Command("go", r.goArgs...)
	if out, err := cmd.CombinedOutput(); err != nil {
		if strings.HasPrefix(err.Error(), "exit status") {
			return errors.New(string(out))
		}
		return err
	}
	if out, err := exec.Command(r.outBin).CombinedOutput(); err != nil {
		if strings.HasPrefix(err.Error(), "exit status") {
			return errors.New(string(out))
		}
		return err
	}
	return nil
}
<commit_msg>Avoid converting exec output to string<commit_after>// Copyright (c) 2017, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

package main

import (
	"fmt"
	"go/ast"
	"go/importer"
	"go/parser"
	"go/printer"
	"go/token"
	"go/types"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"text/template"
)

const mainFile = "goreduce_main.go"

var (
	mainTmpl = template.Must(template.New("main").Parse(`package main

func main() {
	{{ . }}()
}
`))
	rawPrinter = printer.Config{Mode: printer.RawFormat}
)

type reducer struct {
	dir     string
	logOut  io.Writer
	matchRe *regexp.Regexp

	fset     *token.FileSet
	pkg      *ast.Package
	files    []*ast.File
	file     *ast.File
	funcDecl *ast.FuncDecl
	origMain *ast.FuncDecl

	tconf types.Config
	info  *types.Info

	numUses map[types.Object]int

	outBin  string
	goArgs  []string
	dstFile *os.File

	didChange bool

	walker
}

var errNoReduction = fmt.Errorf("could not reduce program")

func reduce(dir, funcName, match string, logOut io.Writer, bflags ...string) error {
	r := &reducer{dir: dir, logOut: logOut}
	tdir, err := ioutil.TempDir("", "goreduce")
	if err != nil {
		return err
	}
	defer os.RemoveAll(tdir)
	r.tconf.Importer = importer.Default()
	if r.matchRe, err = regexp.Compile(match); err != nil {
		return err
	}
	r.fset = token.NewFileSet()
	pkgs, err := parser.ParseDir(r.fset, r.dir, nil, 0)
	if err != nil {
		return err
	}
	if len(pkgs) != 1 {
		return fmt.Errorf("expected 1 package, got %d", len(pkgs))
	}
	for _, pkg := range pkgs {
		r.pkg = pkg
	}
	for _, file := range r.pkg.Files {
		r.files = append(r.files, file)
	}
	r.file, r.funcDecl = findFunc(r.files, funcName)
	if r.file == nil {
		return fmt.Errorf("top-level func %s does not exist", funcName)
	}
	tfnames := make([]string, 0, len(r.files)+1)
	for _, file := range r.files {
		fname := r.fset.Position(file.Pos()).Filename
		tfname := filepath.Join(tdir, filepath.Base(fname))
		f, err := os.Create(tfname)
		if err != nil {
			return nil
		}
		if fd := delFunc(file, "main"); fd != nil && file == r.file {
			r.origMain = fd
		}
		file.Name.Name = "main"
		if err := rawPrinter.Fprint(f, r.fset, file); err != nil {
			return err
		}
		if file == r.file {
			r.dstFile = f
			defer r.dstFile.Close()
		} else if err := f.Close(); err != nil {
			return err
		}
		tfnames = append(tfnames, tfname)
	}
	mfname := filepath.Join(tdir, mainFile)
	mf, err := os.Create(mfname)
	if err != nil {
		return err
	}
	if err := mainTmpl.Execute(mf, funcName); err != nil {
		return err
	}
	if err := mf.Close(); err != nil {
		return err
	}
	tfnames = append(tfnames, mfname)
	r.outBin = filepath.Join(tdir, "bin")
	r.goArgs = []string{"build", "-o", r.outBin}
	r.goArgs = append(r.goArgs, buildFlags...)
	r.goArgs = append(r.goArgs, bflags...)
	r.goArgs = append(r.goArgs, tfnames...)
	// Check that the output matches before we apply any changes
	if err := r.checkRun(); err != nil {
		return err
	}
	if anyChanges := r.reduceLoop(); !anyChanges {
		return errNoReduction
	}
	fname := r.fset.Position(r.file.Pos()).Filename
	f, err := os.Create(fname)
	if err != nil {
		return err
	}
	r.file.Name.Name = r.pkg.Name
	if r.origMain != nil {
		r.file.Decls = append(r.file.Decls, r.origMain)
	}
	if err := printer.Fprint(f, r.fset, r.file); err != nil {
		return err
	}
	return f.Close()
}

func (r *reducer) logChange(node ast.Node, format string, a ...interface{}) {
	if *verbose {
		pos := r.fset.Position(node.Pos())
		fmt.Fprintf(r.logOut, "%s:%d: %s\n", pos.Filename, pos.Line,
			fmt.Sprintf(format, a...))
	}
}

func (r *reducer) checkRun() error {
	out, err := r.buildAndRun()
	if err != nil {
		return err
	}
	if !r.matchRe.Match(out) {
		return fmt.Errorf("error does not match:\n%s", string(out))
	}
	return nil
}

func (r *reducer) okChange() bool {
	if r.didChange {
		return false
	}
	// go/types catches most compile errors before writing
	// to disk and running the go tool. Since quite a lot of
	// changes are nonsensical, this is often a big win.
	if _, err := r.tconf.Check(r.dir, r.fset, r.files, nil); err != nil {
		if terr, ok := err.(types.Error); ok && terr.Soft {
			println("unexpected go/types soft error: " + terr.Msg)
		}
		return false
	}
	if err := r.dstFile.Truncate(0); err != nil {
		return false
	}
	if _, err := r.dstFile.Seek(0, 0); err != nil {
		return false
	}
	if err := rawPrinter.Fprint(r.dstFile, r.fset, r.file); err != nil {
		return false
	}
	if err := r.checkRun(); err != nil {
		return false
	}
	// Reduction worked
	r.didChange = true
	return true
}

func (r *reducer) reduceLoop() (anyChanges bool) {
	r.info = &types.Info{
		Defs: make(map[*ast.Ident]types.Object),
		Uses: make(map[*ast.Ident]types.Object),
	}
	for {
		if _, err := r.tconf.Check(r.dir, r.fset, r.files, r.info); err != nil {
			panic("types.Check should not error here")
		}
		r.fillUses()
		r.didChange = false
		r.walk(r.file, r.reduceNode)
		if !r.didChange {
			return
		}
		anyChanges = true
	}
}

func (r *reducer) fillUses() {
	r.numUses = make(map[types.Object]int)
	for _, obj := range r.info.Uses {
		if pkg := obj.Pkg(); pkg == nil || pkg.Name() != "main" {
			// builtin or declared outside of our pkg
			continue
		}
		switch obj.(type) {
		case *types.PkgName, *types.Var:
			r.numUses[obj]++
		}
	}
}

func findFunc(files []*ast.File, name string) (*ast.File, *ast.FuncDecl) {
	for _, file := range files {
		for _, decl := range file.Decls {
			fd, ok := decl.(*ast.FuncDecl)
			if ok && fd.Name.Name == name {
				return file, fd
			}
		}
	}
	return nil, nil
}

func delFunc(file *ast.File, name string) *ast.FuncDecl {
	for i, decl := range file.Decls {
		fd, ok := decl.(*ast.FuncDecl)
		if ok && fd.Name.Name == name {
			file.Decls = append(file.Decls[:i], file.Decls[i+1:]...)
			return fd
		}
	}
	return nil
}

func (r *reducer) buildAndRun() ([]byte, error) {
	cmd := exec.Command("go", r.goArgs...)
	if out, err := cmd.CombinedOutput(); err != nil {
		if _, ok := err.(*exec.ExitError); ok {
			return out, nil
		}
		return nil, err
	}
	if out, err := exec.Command(r.outBin).CombinedOutput(); err != nil {
		if _, ok := err.(*exec.ExitError); ok {
			return out, nil
		}
		return nil, err
	}
	return nil, fmt.Errorf("expected an error to occur")
}
<|endoftext|>


<commit_before><commit_msg>working on go ptrs<commit_after><|endoftext|>


<commit_before><commit_msg>Fix bug where job deploys with count 0 hang.<commit_after><|endoftext|>


<commit_before>package main

import (
	"context"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"net/http"
	"os"
	"time"

	"github.com/prometheus/client_golang/api/prometheus"
	"github.com/prometheus/common/expfmt"
	"github.com/prometheus/common/model"
)

type Tag struct {
	Name  model.LabelName
	Value model.LabelValue
}

type Metric struct {
	Tags  []Tag
	Value float64
}

func CreateJSONMetrics(samples model.Vector) string {
	metrics := []Metric{}

	for _, sample := range samples {
		metric := Metric{}

		for name, value := range sample.Metric {
			tag := Tag{
				Name:  name,
				Value: value,
			}

			metric.Tags = append(metric.Tags, tag)
		}

		metric.Value = float64(sample.Value)

		metrics = append(metrics, metric)
	}

	jsonMetrics, _ := json.Marshal(metrics)

	return string(jsonMetrics)
}

func CreateGraphiteMetrics(samples model.Vector) string {
	metrics := ""

	for _, sample := range samples {
		name := sample.Metric["__name__"]

		value := sample.Value

		now := time.Now()
		timestamp := now.Unix()

		metric := fmt.Sprintf("%s %f %v\n", name, value, timestamp)

		metrics += metric
	}

	return metrics
}

func CreateInfluxMetrics(samples model.Vector) string {
	metrics := ""

	for _, sample := range samples {
		metric := string(sample.Metric["__name__"])

		for name, value := range sample.Metric {
			if name != "__name__" {
				metric += fmt.Sprintf(",%s=%s", name, value)
			}
		}

		value := sample.Value

		now := time.Now()
		timestamp := now.Unix()

		metric += fmt.Sprintf(" value=%f %v\n", value, timestamp)

		metrics += metric
	}

	return metrics
}

func OutputMetrics(samples model.Vector, outputFormat string) error {
	output := ""

	switch outputFormat {
	case "influx":
		output = CreateInfluxMetrics(samples)
	case "graphite":
		output = CreateGraphiteMetrics(samples)
	case "json":
		output = CreateJSONMetrics(samples)
	}

	fmt.Println(output)

	return nil
}

func QueryPrometheus(promURL string, queryString string) (model.Vector, error) {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	promConfig := prometheus.Config{Address: promURL}
	promClient, err := prometheus.New(promConfig)

	if err != nil {
		return nil, err
	}

	promQueryClient := prometheus.NewQueryAPI(promClient)

	promResponse, err := promQueryClient.Query(ctx, queryString, time.Now())

	if err != nil {
		return nil, err
	}

	if promResponse.Type() == model.ValVector {
		return promResponse.(model.Vector), nil
	}

	return nil, errors.New("unexpected response type")
}

func QueryExporter(exporterURL string) (model.Vector, error) {
	expResponse, err := http.Get(exporterURL)
	defer expResponse.Body.Close()

	if err != nil {
		return nil, err
	}

	if expResponse.StatusCode != http.StatusOK {
		return nil, errors.New("non OK HTTP response status")
	}

	var parser expfmt.TextParser

	metricFamilies, err := parser.TextToMetricFamilies(expResponse.Body)

	if err != nil {
		return nil, err
	}

	samples := model.Vector{}

	decodeOptions := &expfmt.DecodeOptions{
		Timestamp: model.Time(time.Now().Unix()),
	}

	for _, family := range metricFamilies {
		familySamples, _ := expfmt.ExtractSamples(decodeOptions, family)

		for _, sample := range familySamples {
			samples = append(samples, sample)
		}
	}

	return samples, nil
}

func main() {
	exporterURL := flag.String("exporter-url", "", "Prometheus exporter URL to pull metrics from")
	promURL := flag.String("prom-url", "http://localhost:9090", "Prometheus API URL")
	queryString := flag.String("prom-query", "up", "Prometheus API query string")
	outputFormat := flag.String("output-format", "influx", "The check output format to use for metrics {influx|graphite|json}")
	flag.Parse()

	samples := model.Vector{}
	var err error

	if *exporterURL != "" {
		samples, err = QueryExporter(*exporterURL)
	} else {
		samples, err = QueryPrometheus(*promURL, *queryString)
	}

	if err != nil {
		fmt.Errorf("%v", err)
		os.Exit(2)
	}

	err = OutputMetrics(samples, *outputFormat)

	if err != nil {
		fmt.Errorf("%v", err)
		os.Exit(2)
	}
}
<commit_msg>better exporter family series append<commit_after>package main

import (
	"context"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"net/http"
	"os"
	"time"

	"github.com/prometheus/client_golang/api/prometheus"
	"github.com/prometheus/common/expfmt"
	"github.com/prometheus/common/model"
)

type Tag struct {
	Name  model.LabelName
	Value model.LabelValue
}

type Metric struct {
	Tags  []Tag
	Value float64
}

func CreateJSONMetrics(samples model.Vector) string {
	metrics := []Metric{}

	for _, sample := range samples {
		metric := Metric{}

		for name, value := range sample.Metric {
			tag := Tag{
				Name:  name,
				Value: value,
			}

			metric.Tags = append(metric.Tags, tag)
		}

		metric.Value = float64(sample.Value)

		metrics = append(metrics, metric)
	}

	jsonMetrics, _ := json.Marshal(metrics)

	return string(jsonMetrics)
}

func CreateGraphiteMetrics(samples model.Vector) string {
	metrics := ""

	for _, sample := range samples {
		name := sample.Metric["__name__"]

		value := sample.Value

		now := time.Now()
		timestamp := now.Unix()

		metric := fmt.Sprintf("%s %f %v\n", name, value, timestamp)

		metrics += metric
	}

	return metrics
}

func CreateInfluxMetrics(samples model.Vector) string {
	metrics := ""

	for _, sample := range samples {
		metric := string(sample.Metric["__name__"])

		for name, value := range sample.Metric {
			if name != "__name__" {
				metric += fmt.Sprintf(",%s=%s", name, value)
			}
		}

		value := sample.Value

		now := time.Now()
		timestamp := now.Unix()

		metric += fmt.Sprintf(" value=%f %v\n", value, timestamp)

		metrics += metric
	}

	return metrics
}

func OutputMetrics(samples model.Vector, outputFormat string) error {
	output := ""

	switch outputFormat {
	case "influx":
		output = CreateInfluxMetrics(samples)
	case "graphite":
		output = CreateGraphiteMetrics(samples)
	case "json":
		output = CreateJSONMetrics(samples)
	}

	fmt.Println(output)

	return nil
}

func QueryPrometheus(promURL string, queryString string) (model.Vector, error) {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	promConfig := prometheus.Config{Address: promURL}
	promClient, err := prometheus.New(promConfig)

	if err != nil {
		return nil, err
	}

	promQueryClient := prometheus.NewQueryAPI(promClient)

	promResponse, err := promQueryClient.Query(ctx, queryString, time.Now())

	if err != nil {
		return nil, err
	}

	if promResponse.Type() == model.ValVector {
		return promResponse.(model.Vector), nil
	}

	return nil, errors.New("unexpected response type")
}

func QueryExporter(exporterURL string) (model.Vector, error) {
	expResponse, err := http.Get(exporterURL)
	defer expResponse.Body.Close()

	if err != nil {
		return nil, err
	}

	if expResponse.StatusCode != http.StatusOK {
		return nil, errors.New("exporter returned non OK HTTP response status")
	}

	var parser expfmt.TextParser

	metricFamilies, err := parser.TextToMetricFamilies(expResponse.Body)

	if err != nil {
		return nil, err
	}

	samples := model.Vector{}

	decodeOptions := &expfmt.DecodeOptions{
		Timestamp: model.Time(time.Now().Unix()),
	}

	for _, family := range metricFamilies {
		familySamples, _ := expfmt.ExtractSamples(decodeOptions, family)
		samples = append(samples, familySamples...)
	}

	return samples, nil
}

func main() {
	exporterURL := flag.String("exporter-url", "", "Prometheus exporter URL to pull metrics from")
	promURL := flag.String("prom-url", "http://localhost:9090", "Prometheus API URL")
	queryString := flag.String("prom-query", "up", "Prometheus API query string")
	outputFormat := flag.String("output-format", "influx", "The check output format to use for metrics {influx|graphite|json}")
	flag.Parse()

	samples := model.Vector{}
	var err error

	if *exporterURL != "" {
		samples, err = QueryExporter(*exporterURL)
	} else {
		samples, err = QueryPrometheus(*promURL, *queryString)
	}

	if err != nil {
		fmt.Errorf("%v", err)
		os.Exit(2)
	}

	err = OutputMetrics(samples, *outputFormat)

	if err != nil {
		fmt.Errorf("%v", err)
		os.Exit(2)
	}
}
<|endoftext|>


<commit_before>package terminal

import (
	"bufio"
	"bytes"
	"io"
	"os"
	"strconv"
	"strings"

	"github.com/elpinal/coco3/config"
	"github.com/elpinal/coco3/screen"

	"github.com/mattn/go-runewidth"
)

type Terminal struct {
	w              *bufio.Writer
	msg            string
	lastCursorLine int
}

func New(w io.Writer) *Terminal {
	return &Terminal{
		w: bufio.NewWriterSize(w, 32),
	}
}

func getwd() string {
	wd, _ := os.Getwd()
	if home := os.Getenv("HOME"); strings.HasPrefix(wd, home) {
		wd = strings.Replace(wd, home, "~", 1)
	}
	return wd
}

func (t *Terminal) Start(conf *config.Config, inCommandline bool, s []rune, pos int, hi *screen.Hi) {
	t.draw(conf, inCommandline, s, pos, false, hi)
}

func (t *Terminal) Refresh(conf *config.Config, inCommandline bool, s []rune, pos int, hi *screen.Hi) {
	t.draw(conf, inCommandline, s, pos, true, hi)
}

func (t *Terminal) draw(conf *config.Config, inCommandline bool, s []rune, pos int, refresh bool, hi *screen.Hi) {
	prompt := conf.Prompt
	if conf.PromptTmpl != nil {
		var buf bytes.Buffer
		conf.PromptTmpl.Execute(&buf, config.Info{WD: getwd()})
		prompt = buf.String()
	}
	if refresh {
		if t.lastCursorLine > 0 {
			t.w.WriteString("\033[")
			t.w.WriteString(strconv.Itoa(t.lastCursorLine))
			t.w.WriteString("A")
		}
	}
	count := strings.Count(prompt, "\n")
	if inCommandline {
		count += 1
	}
	t.lastCursorLine = count
	t.w.WriteString("\r\033[J")
	t.w.WriteString(prompt)
	i := strings.LastIndex(prompt, "\n") + 1
	promptWidth := runewidth.StringWidth(prompt[i:])
	if hi == nil {
		t.w.WriteString(string(s))
	} else {
		t.w.WriteString(string(s[:hi.Left]))
		t.w.WriteString("\033[7m")
		t.w.WriteString(string(s[hi.Left:hi.Right]))
		t.w.WriteString("\033[0m")
		t.w.WriteString(string(s[hi.Right:]))
	}
	if t.msg != "" {
		t.w.WriteString("\n\r")
		t.w.WriteString(t.msg)
		if !inCommandline {
			t.w.WriteString("\033[A")
		}
	}
	var drawPos int
	if inCommandline {
		drawPos = runewidth.StringWidth(t.msg[:pos])
	} else {
		drawPos = promptWidth + runesWidth(s[:pos])
	}
	t.w.WriteString("\033[")
	t.w.WriteString(strconv.Itoa(drawPos + 1))
	t.w.WriteString("G")
	t.w.Flush()
}

func runesWidth(s []rune) (width int) {
	for _, r := range s {
		width += runewidth.RuneWidth(r)
	}
	return width
}

func (t *Terminal) SetLastLine(msg string) {
	t.msg = msg
}
<commit_msg>Follow golint<commit_after>package terminal

import (
	"bufio"
	"bytes"
	"io"
	"os"
	"strconv"
	"strings"

	"github.com/elpinal/coco3/config"
	"github.com/elpinal/coco3/screen"

	"github.com/mattn/go-runewidth"
)

type Terminal struct {
	w              *bufio.Writer
	msg            string
	lastCursorLine int
}

func New(w io.Writer) *Terminal {
	return &Terminal{
		w: bufio.NewWriterSize(w, 32),
	}
}

func getwd() string {
	wd, _ := os.Getwd()
	if home := os.Getenv("HOME"); strings.HasPrefix(wd, home) {
		wd = strings.Replace(wd, home, "~", 1)
	}
	return wd
}

func (t *Terminal) Start(conf *config.Config, inCommandline bool, s []rune, pos int, hi *screen.Hi) {
	t.draw(conf, inCommandline, s, pos, false, hi)
}

func (t *Terminal) Refresh(conf *config.Config, inCommandline bool, s []rune, pos int, hi *screen.Hi) {
	t.draw(conf, inCommandline, s, pos, true, hi)
}

func (t *Terminal) draw(conf *config.Config, inCommandline bool, s []rune, pos int, refresh bool, hi *screen.Hi) {
	prompt := conf.Prompt
	if conf.PromptTmpl != nil {
		var buf bytes.Buffer
		conf.PromptTmpl.Execute(&buf, config.Info{WD: getwd()})
		prompt = buf.String()
	}
	if refresh {
		if t.lastCursorLine > 0 {
			t.w.WriteString("\033[")
			t.w.WriteString(strconv.Itoa(t.lastCursorLine))
			t.w.WriteString("A")
		}
	}
	count := strings.Count(prompt, "\n")
	if inCommandline {
		count++
	}
	t.lastCursorLine = count
	t.w.WriteString("\r\033[J")
	t.w.WriteString(prompt)
	i := strings.LastIndex(prompt, "\n") + 1
	promptWidth := runewidth.StringWidth(prompt[i:])
	if hi == nil {
		t.w.WriteString(string(s))
	} else {
		t.w.WriteString(string(s[:hi.Left]))
		t.w.WriteString("\033[7m")
		t.w.WriteString(string(s[hi.Left:hi.Right]))
		t.w.WriteString("\033[0m")
		t.w.WriteString(string(s[hi.Right:]))
	}
	if t.msg != "" {
		t.w.WriteString("\n\r")
		t.w.WriteString(t.msg)
		if !inCommandline {
			t.w.WriteString("\033[A")
		}
	}
	var drawPos int
	if inCommandline {
		drawPos = runewidth.StringWidth(t.msg[:pos])
	} else {
		drawPos = promptWidth + runesWidth(s[:pos])
	}
	t.w.WriteString("\033[")
	t.w.WriteString(strconv.Itoa(drawPos + 1))
	t.w.WriteString("G")
	t.w.Flush()
}

func runesWidth(s []rune) (width int) {
	for _, r := range s {
		width += runewidth.RuneWidth(r)
	}
	return width
}

func (t *Terminal) SetLastLine(msg string) {
	t.msg = msg
}
<|endoftext|>


<commit_before>package irmaserver

import (
	//TODO: use redigo instead of redis-go v8?
	"context"
	"encoding/json"
	"github.com/go-errors/errors"
	"strings"
	"sync"
	"time"

	"github.com/alexandrevicenzi/go-sse"
	"github.com/go-redis/redis/v8"
	"github.com/privacybydesign/gabi"
	"github.com/privacybydesign/gabi/big"
	irma "github.com/privacybydesign/irmago"
	"github.com/privacybydesign/irmago/internal/common"
	"github.com/privacybydesign/irmago/server"

	"github.com/sirupsen/logrus"
)

type session struct {
	//TODO: check if we can get rid of this Mutex for Redis
	sync.Mutex `json:-`
	//TODO: note somewhere that state with redis will not support sse for the moment
	sse         *sse.Server
	locked      bool
	sessions    sessionStore
	conf        *server.Configuration
	request     irma.SessionRequest
	context     context.Context
	toBeUpdated bool

	sessionData
}

type sessionData struct {
	Action             irma.Action
	Token              string
	ClientToken        string
	Version            *irma.ProtocolVersion `json:",omitempty"`
	Rrequest           irma.RequestorRequest
	LegacyCompatible   bool // if the request is convertible to pre-condiscon format
	ImplicitDisclosure irma.AttributeConDisCon
	Status             server.Status
	PrevStatus         server.Status
	ResponseCache      responseCache
	LastActive         time.Time
	Result             *server.SessionResult
	KssProofs          map[irma.SchemeManagerIdentifier]*gabi.ProofP
}

type responseCache struct {
	Message       []byte
	Response      []byte
	Status        int
	SessionStatus server.Status
}

type sessionStore interface {
	get(token string) (*session, error)
	clientGet(token string) (*session, error)
	add(session *session) error
	update(session *session) error
	deleteExpired()
	stop() error
}

type memorySessionStore struct {
	sync.RWMutex
	conf *server.Configuration

	requestor map[string]*session
	client    map[string]*session
}

type redisSessionStore struct {
	client *redis.Client
	conf   *server.Configuration
}

const (
	maxSessionLifetime  = 5 * time.Minute // After this a session is cancelled
	tokenLookupPrefix   = "token:"
	sessionLookupPrefix = "session:"
)

var (
	minProtocolVersion = irma.NewVersion(2, 4)
	maxProtocolVersion = irma.NewVersion(2, 7)
)

func (s *memorySessionStore) get(t string) (*session, error) {
	s.RLock()
	defer s.RUnlock()
	return s.requestor[t], nil
}

func (s *memorySessionStore) clientGet(t string) (*session, error) {
	s.RLock()
	defer s.RUnlock()
	return s.client[t], nil
}

func (s *memorySessionStore) add(session *session) error {
	s.Lock()
	defer s.Unlock()
	s.requestor[session.Token] = session
	s.client[session.ClientToken] = session
	return nil
}

func (s *memorySessionStore) update(session *session) error {
	session.updateSSE()
	return nil
}

func (s *memorySessionStore) stop() error {
	s.Lock()
	defer s.Unlock()
	for _, session := range s.requestor {
		if session.sse != nil {
			session.sse.CloseChannel("session/" + session.Token)
			session.sse.CloseChannel("session/" + session.ClientToken)
		}
	}
	return nil
}

func (s *memorySessionStore) deleteExpired() {
	// First check which sessions have expired
	// We don't need a write lock for this yet, so postpone that for actual deleting
	s.RLock()
	expired := make([]string, 0, len(s.requestor))
	for Token, session := range s.requestor {
		session.Lock()

		timeout := maxSessionLifetime
		if session.Status == server.StatusInitialized && session.Rrequest.Base().ClientTimeout != 0 {
			timeout = time.Duration(session.Rrequest.Base().ClientTimeout) * time.Second
		}

		if session.LastActive.Add(timeout).Before(time.Now()) {
			if !session.Status.Finished() {
				s.conf.Logger.WithFields(logrus.Fields{"session": session.Token}).Infof("Session expired")
				session.markAlive()
				session.setStatus(server.StatusTimeout)
			} else {
				s.conf.Logger.WithFields(logrus.Fields{"session": session.Token}).Infof("Deleting session")
				expired = append(expired, Token)
			}
		}
		session.Unlock()
	}
	s.RUnlock()

	// Using a write lock, delete the expired sessions
	s.Lock()
	for _, Token := range expired {
		session := s.requestor[Token]
		if session.sse != nil {
			session.sse.CloseChannel("session/" + session.Token)
			session.sse.CloseChannel("session/" + session.ClientToken)
		}
		delete(s.client, session.ClientToken)
		delete(s.requestor, Token)
	}
	s.Unlock()
}

// MarshalJSON marshals sessionData to be used in the Redis in-memory datastore.
func (s *sessionData) MarshalJSON() ([]byte, error) {
	return json.Marshal(*s)
}

// UnmarshalJSON unmarshals sessionData.
func (s *sessionData) UnmarshalJSON(data []byte) error {
	type rawSessionData sessionData

	var temp struct {
		Rrequest *json.RawMessage `json:",omitempty"`
		rawSessionData
	}

	if err := json.Unmarshal(data, &temp); err != nil {
		return err
	}

	*s = sessionData(temp.rawSessionData)

	if temp.Rrequest == nil {
		s.Rrequest = nil
		return errors.Errorf("temp.Rrequest == nil: %d \n", temp.Rrequest)
	}

	// unmarshal Rrequest
	ipR := &irma.IdentityProviderRequest{}
	spR := &irma.ServiceProviderRequest{}
	sigR := &irma.SignatureRequestorRequest{}

	if err := json.Unmarshal(*temp.Rrequest, ipR); err == nil && s.Action == "issuing" {
		s.Rrequest = ipR
	} else if err = json.Unmarshal(*temp.Rrequest, spR); err == nil && s.Action == "disclosing" {
		s.Rrequest = spR
	} else if err = json.Unmarshal(*temp.Rrequest, sigR); err == nil && s.Action == "signing" {
		s.Rrequest = sigR
	} else {
		return err
	}

	return nil
}

func (s *redisSessionStore) get(t string) (*session, error) {
	//TODO: input validation string?
	val, err := s.client.Get(context.Background(), tokenLookupPrefix+t).Result()
	if err == redis.Nil {
		return nil, nil
	} else if err != nil {
		_ = server.LogError(err)
		return nil, errors.New("redis error")
	}

	return s.clientGet(val)
}

func (s *redisSessionStore) clientGet(t string) (*session, error) {
	val, err := s.client.Get(context.Background(), sessionLookupPrefix+t).Result()
	if err == redis.Nil {
		return nil, nil
	} else if err != nil {
		_ = server.LogError(err)
		return nil, errors.New("redis error")
	}

	var session session
	session.conf = s.conf
	session.sessions = s
	if err := session.sessionData.UnmarshalJSON([]byte(val)); err != nil {
		_ = server.LogError(err)
		return nil, errors.New("redis error")
	}
	session.request = session.Rrequest.SessionRequest()

	if session.LastActive.Add(maxSessionLifetime).Before(time.Now()) && !session.Status.Finished() {
		s.conf.Logger.WithFields(logrus.Fields{"session": session.Token}).Infof("Session expired")
		session.markAlive()
		session.setStatus(server.StatusTimeout)
		_ = s.update(&session) // Worst case the TTL and status aren't updated. We won't deal with this error
	}

	return &session, nil
}

func (s *redisSessionStore) add(session *session) error {
	timeout := 2 * maxSessionLifetime // logic similar to memory store
	if session.Status == server.StatusInitialized && session.Rrequest.Base().ClientTimeout != 0 {
		timeout = time.Duration(session.Rrequest.Base().ClientTimeout) * time.Second
	} else if session.Status.Finished() {
		timeout = maxSessionLifetime
	}

	sessionJSON, err := session.sessionData.MarshalJSON()
	if err != nil {
		_ = server.LogError(err)
		return errors.New("redis error")
	}

	err = s.client.Set(session.context, tokenLookupPrefix+session.sessionData.Token, session.sessionData.ClientToken, timeout).Err()
	if err != nil {
		_ = server.LogError(err)
		return errors.New("redis error")
	}
	err = s.client.Set(session.context, sessionLookupPrefix+session.sessionData.ClientToken, sessionJSON, timeout).Err()
	if err != nil {
		_ = server.LogError(err)
		return errors.New("redis error")
	}

	return nil
}

func (s *redisSessionStore) update(session *session) error {
	return s.add(session)
}

func (s *redisSessionStore) stop() error {
	err := s.client.Close()
	if err != nil {
		return server.LogError(err)
	}

	return nil
}

func (s *redisSessionStore) deleteExpired() {
	// This method is not used as data is directly stored in Redis with a corresponding expiration time.
}

var one *big.Int = big.NewInt(1)

func (s *Server) newSession(action irma.Action, request irma.RequestorRequest, ctx context.Context) (*session, error) {
	token := common.NewSessionToken()
	clientToken := common.NewSessionToken()

	base := request.SessionRequest().Base()
	if s.conf.AugmentClientReturnURL && base.AugmentReturnURL && base.ClientReturnURL != "" {
		if strings.Contains(base.ClientReturnURL, "?") {
			base.ClientReturnURL += "&Token=" + token
		} else {
			base.ClientReturnURL += "?Token=" + token
		}
	}

	sd := sessionData{
		Action:      action,
		Rrequest:    request,
		LastActive:  time.Now(),
		Token:       token,
		ClientToken: clientToken,
		Status:      server.StatusInitialized,
		PrevStatus:  server.StatusInitialized,
		Result: &server.SessionResult{
			LegacySession: request.SessionRequest().Base().Legacy(),
			Token:         token,
			Type:          action,
			Status:        server.StatusInitialized,
		},
	}
	ses := &session{
		sessionData: sd,
		sessions:    s.sessions,
		sse:         s.serverSentEvents,
		conf:        s.conf,
		request:     request.SessionRequest(),
		context:     ctx,
	}

	s.conf.Logger.WithFields(logrus.Fields{"session": ses.Token}).Debug("New session started")
	nonce, _ := gabi.GenerateNonce()
	base.Nonce = nonce
	base.Context = one
	err := s.sessions.add(ses)
	if err != nil {
		return nil, err
	}

	return ses, nil
}
<commit_msg>Cleanup memorySessionStores update method from duplicated updateSSE call. The call has been moved to setStatus.<commit_after>package irmaserver

import (
	//TODO: use redigo instead of redis-go v8?
	"context"
	"encoding/json"
	"github.com/go-errors/errors"
	"strings"
	"sync"
	"time"

	"github.com/alexandrevicenzi/go-sse"
	"github.com/go-redis/redis/v8"
	"github.com/privacybydesign/gabi"
	"github.com/privacybydesign/gabi/big"
	irma "github.com/privacybydesign/irmago"
	"github.com/privacybydesign/irmago/internal/common"
	"github.com/privacybydesign/irmago/server"

	"github.com/sirupsen/logrus"
)

type session struct {
	//TODO: check if we can get rid of this Mutex for Redis
	sync.Mutex `json:-`
	//TODO: note somewhere that state with redis will not support sse for the moment
	sse         *sse.Server
	locked      bool
	sessions    sessionStore
	conf        *server.Configuration
	request     irma.SessionRequest
	context     context.Context
	toBeUpdated bool

	sessionData
}

type sessionData struct {
	Action             irma.Action
	Token              string
	ClientToken        string
	Version            *irma.ProtocolVersion `json:",omitempty"`
	Rrequest           irma.RequestorRequest
	LegacyCompatible   bool // if the request is convertible to pre-condiscon format
	ImplicitDisclosure irma.AttributeConDisCon
	Status             server.Status
	PrevStatus         server.Status
	ResponseCache      responseCache
	LastActive         time.Time
	Result             *server.SessionResult
	KssProofs          map[irma.SchemeManagerIdentifier]*gabi.ProofP
}

type responseCache struct {
	Message       []byte
	Response      []byte
	Status        int
	SessionStatus server.Status
}

type sessionStore interface {
	get(token string) (*session, error)
	clientGet(token string) (*session, error)
	add(session *session) error
	update(session *session) error
	deleteExpired()
	stop() error
}

type memorySessionStore struct {
	sync.RWMutex
	conf *server.Configuration

	requestor map[string]*session
	client    map[string]*session
}

type redisSessionStore struct {
	client *redis.Client
	conf   *server.Configuration
}

const (
	maxSessionLifetime  = 5 * time.Minute // After this a session is cancelled
	tokenLookupPrefix   = "token:"
	sessionLookupPrefix = "session:"
)

var (
	minProtocolVersion = irma.NewVersion(2, 4)
	maxProtocolVersion = irma.NewVersion(2, 7)
)

func (s *memorySessionStore) get(t string) (*session, error) {
	s.RLock()
	defer s.RUnlock()
	return s.requestor[t], nil
}

func (s *memorySessionStore) clientGet(t string) (*session, error) {
	s.RLock()
	defer s.RUnlock()
	return s.client[t], nil
}

func (s *memorySessionStore) add(session *session) error {
	s.Lock()
	defer s.Unlock()
	s.requestor[session.Token] = session
	s.client[session.ClientToken] = session
	return nil
}

func (s *memorySessionStore) update(_ *session) error {
	return nil
}

func (s *memorySessionStore) stop() error {
	s.Lock()
	defer s.Unlock()
	for _, session := range s.requestor {
		if session.sse != nil {
			session.sse.CloseChannel("session/" + session.Token)
			session.sse.CloseChannel("session/" + session.ClientToken)
		}
	}
	return nil
}

func (s *memorySessionStore) deleteExpired() {
	// First check which sessions have expired
	// We don't need a write lock for this yet, so postpone that for actual deleting
	s.RLock()
	expired := make([]string, 0, len(s.requestor))
	for Token, session := range s.requestor {
		session.Lock()

		timeout := maxSessionLifetime
		if session.Status == server.StatusInitialized && session.Rrequest.Base().ClientTimeout != 0 {
			timeout = time.Duration(session.Rrequest.Base().ClientTimeout) * time.Second
		}

		if session.LastActive.Add(timeout).Before(time.Now()) {
			if !session.Status.Finished() {
				s.conf.Logger.WithFields(logrus.Fields{"session": session.Token}).Infof("Session expired")
				session.markAlive()
				session.setStatus(server.StatusTimeout)
			} else {
				s.conf.Logger.WithFields(logrus.Fields{"session": session.Token}).Infof("Deleting session")
				expired = append(expired, Token)
			}
		}
		session.Unlock()
	}
	s.RUnlock()

	// Using a write lock, delete the expired sessions
	s.Lock()
	for _, Token := range expired {
		session := s.requestor[Token]
		if session.sse != nil {
			session.sse.CloseChannel("session/" + session.Token)
			session.sse.CloseChannel("session/" + session.ClientToken)
		}
		delete(s.client, session.ClientToken)
		delete(s.requestor, Token)
	}
	s.Unlock()
}

// MarshalJSON marshals sessionData to be used in the Redis in-memory datastore.
func (s *sessionData) MarshalJSON() ([]byte, error) {
	return json.Marshal(*s)
}

// UnmarshalJSON unmarshals sessionData.
func (s *sessionData) UnmarshalJSON(data []byte) error {
	type rawSessionData sessionData

	var temp struct {
		Rrequest *json.RawMessage `json:",omitempty"`
		rawSessionData
	}

	if err := json.Unmarshal(data, &temp); err != nil {
		return err
	}

	*s = sessionData(temp.rawSessionData)

	if temp.Rrequest == nil {
		s.Rrequest = nil
		return errors.Errorf("temp.Rrequest == nil: %d \n", temp.Rrequest)
	}

	// unmarshal Rrequest
	ipR := &irma.IdentityProviderRequest{}
	spR := &irma.ServiceProviderRequest{}
	sigR := &irma.SignatureRequestorRequest{}

	if err := json.Unmarshal(*temp.Rrequest, ipR); err == nil && s.Action == "issuing" {
		s.Rrequest = ipR
	} else if err = json.Unmarshal(*temp.Rrequest, spR); err == nil && s.Action == "disclosing" {
		s.Rrequest = spR
	} else if err = json.Unmarshal(*temp.Rrequest, sigR); err == nil && s.Action == "signing" {
		s.Rrequest = sigR
	} else {
		return err
	}

	return nil
}

func (s *redisSessionStore) get(t string) (*session, error) {
	//TODO: input validation string?
	val, err := s.client.Get(context.Background(), tokenLookupPrefix+t).Result()
	if err == redis.Nil {
		return nil, nil
	} else if err != nil {
		_ = server.LogError(err)
		return nil, errors.New("redis error")
	}

	return s.clientGet(val)
}

func (s *redisSessionStore) clientGet(t string) (*session, error) {
	val, err := s.client.Get(context.Background(), sessionLookupPrefix+t).Result()
	if err == redis.Nil {
		return nil, nil
	} else if err != nil {
		_ = server.LogError(err)
		return nil, errors.New("redis error")
	}

	var session session
	session.conf = s.conf
	session.sessions = s
	if err := session.sessionData.UnmarshalJSON([]byte(val)); err != nil {
		_ = server.LogError(err)
		return nil, errors.New("redis error")
	}
	session.request = session.Rrequest.SessionRequest()

	if session.LastActive.Add(maxSessionLifetime).Before(time.Now()) && !session.Status.Finished() {
		s.conf.Logger.WithFields(logrus.Fields{"session": session.Token}).Infof("Session expired")
		session.markAlive()
		session.setStatus(server.StatusTimeout)
		_ = s.update(&session) // Worst case the TTL and status aren't updated. We won't deal with this error
	}

	return &session, nil
}

func (s *redisSessionStore) add(session *session) error {
	timeout := 2 * maxSessionLifetime // logic similar to memory store
	if session.Status == server.StatusInitialized && session.Rrequest.Base().ClientTimeout != 0 {
		timeout = time.Duration(session.Rrequest.Base().ClientTimeout) * time.Second
	} else if session.Status.Finished() {
		timeout = maxSessionLifetime
	}

	sessionJSON, err := session.sessionData.MarshalJSON()
	if err != nil {
		_ = server.LogError(err)
		return errors.New("redis error")
	}

	err = s.client.Set(session.context, tokenLookupPrefix+session.sessionData.Token, session.sessionData.ClientToken, timeout).Err()
	if err != nil {
		_ = server.LogError(err)
		return errors.New("redis error")
	}
	err = s.client.Set(session.context, sessionLookupPrefix+session.sessionData.ClientToken, sessionJSON, timeout).Err()
	if err != nil {
		_ = server.LogError(err)
		return errors.New("redis error")
	}

	return nil
}

func (s *redisSessionStore) update(session *session) error {
	return s.add(session)
}

func (s *redisSessionStore) stop() error {
	err := s.client.Close()
	if err != nil {
		return server.LogError(err)
	}

	return nil
}

func (s *redisSessionStore) deleteExpired() {
	// This method is not used as data is directly stored in Redis with a corresponding expiration time.
}

var one *big.Int = big.NewInt(1)

func (s *Server) newSession(action irma.Action, request irma.RequestorRequest, ctx context.Context) (*session, error) {
	token := common.NewSessionToken()
	clientToken := common.NewSessionToken()

	base := request.SessionRequest().Base()
	if s.conf.AugmentClientReturnURL && base.AugmentReturnURL && base.ClientReturnURL != "" {
		if strings.Contains(base.ClientReturnURL, "?") {
			base.ClientReturnURL += "&Token=" + token
		} else {
			base.ClientReturnURL += "?Token=" + token
		}
	}

	sd := sessionData{
		Action:      action,
		Rrequest:    request,
		LastActive:  time.Now(),
		Token:       token,
		ClientToken: clientToken,
		Status:      server.StatusInitialized,
		PrevStatus:  server.StatusInitialized,
		Result: &server.SessionResult{
			LegacySession: request.SessionRequest().Base().Legacy(),
			Token:         token,
			Type:          action,
			Status:        server.StatusInitialized,
		},
	}
	ses := &session{
		sessionData: sd,
		sessions:    s.sessions,
		sse:         s.serverSentEvents,
		conf:        s.conf,
		request:     request.SessionRequest(),
		context:     ctx,
	}

	s.conf.Logger.WithFields(logrus.Fields{"session": ses.Token}).Debug("New session started")
	nonce, _ := gabi.GenerateNonce()
	base.Nonce = nonce
	base.Context = one
	err := s.sessions.add(ses)
	if err != nil {
		return nil, err
	}

	return ses, nil
}
<|endoftext|>


<commit_before>// Package twilio provides bindings for Twilio's REST APIs.
package twilio

import (
	"os"
	"time"

	"github.com/twilio/twilio-go/client"
	AccountsV1 "github.com/twilio/twilio-go/rest/accounts/v1"
	ApiV2010 "github.com/twilio/twilio-go/rest/api/v2010"
	AutopilotV1 "github.com/twilio/twilio-go/rest/autopilot/v1"
	BulkexportsV1 "github.com/twilio/twilio-go/rest/bulkexports/v1"
	ChatV1 "github.com/twilio/twilio-go/rest/chat/v1"
	ChatV2 "github.com/twilio/twilio-go/rest/chat/v2"
	ConversationsV1 "github.com/twilio/twilio-go/rest/conversations/v1"
	EventsV1 "github.com/twilio/twilio-go/rest/events/v1"
	FaxV1 "github.com/twilio/twilio-go/rest/fax/v1"
	FlexV1 "github.com/twilio/twilio-go/rest/flex/v1"
	InsightsV1 "github.com/twilio/twilio-go/rest/insights/v1"
	Ip_MessagingV1 "github.com/twilio/twilio-go/rest/ip_messaging/v1"
	Ip_MessagingV2 "github.com/twilio/twilio-go/rest/ip_messaging/v2"
	LookupsV1 "github.com/twilio/twilio-go/rest/lookups/v1"
	MessagingV1 "github.com/twilio/twilio-go/rest/messaging/v1"
	MonitorV1 "github.com/twilio/twilio-go/rest/monitor/v1"
	NotifyV1 "github.com/twilio/twilio-go/rest/notify/v1"
	NumbersV2 "github.com/twilio/twilio-go/rest/numbers/v2"
	PricingV1 "github.com/twilio/twilio-go/rest/pricing/v1"
	PricingV2 "github.com/twilio/twilio-go/rest/pricing/v2"
	ProxyV1 "github.com/twilio/twilio-go/rest/proxy/v1"
	ServerlessV1 "github.com/twilio/twilio-go/rest/serverless/v1"
	StudioV1 "github.com/twilio/twilio-go/rest/studio/v1"
	StudioV2 "github.com/twilio/twilio-go/rest/studio/v2"
	SupersimV1 "github.com/twilio/twilio-go/rest/supersim/v1"
	SyncV1 "github.com/twilio/twilio-go/rest/sync/v1"
	TaskrouterV1 "github.com/twilio/twilio-go/rest/taskrouter/v1"
	TrunkingV1 "github.com/twilio/twilio-go/rest/trunking/v1"
	TrusthubV1 "github.com/twilio/twilio-go/rest/trusthub/v1"
	VerifyV2 "github.com/twilio/twilio-go/rest/verify/v2"
	VideoV1 "github.com/twilio/twilio-go/rest/video/v1"
	VoiceV1 "github.com/twilio/twilio-go/rest/voice/v1"
	WirelessV1 "github.com/twilio/twilio-go/rest/wireless/v1"
)

// RestClient provides access to Twilio services.
type RestClient struct {
	*client.Client
	AccountsV1      *AccountsV1.DefaultApiService
	ApiV2010        *ApiV2010.DefaultApiService
	AutopilotV1     *AutopilotV1.DefaultApiService
	BulkexportsV1   *BulkexportsV1.DefaultApiService
	ChatV1          *ChatV1.DefaultApiService
	ChatV2          *ChatV2.DefaultApiService
	ConversationsV1 *ConversationsV1.DefaultApiService
	EventsV1        *EventsV1.DefaultApiService
	FaxV1           *FaxV1.DefaultApiService
	FlexV1          *FlexV1.DefaultApiService
	InsightsV1      *InsightsV1.DefaultApiService
	Ip_MessagingV1  *Ip_MessagingV1.DefaultApiService
	Ip_MessagingV2  *Ip_MessagingV2.DefaultApiService
	LookupsV1       *LookupsV1.DefaultApiService
	MessagingV1     *MessagingV1.DefaultApiService
	MonitorV1       *MonitorV1.DefaultApiService
	NotifyV1        *NotifyV1.DefaultApiService
	NumbersV2       *NumbersV2.DefaultApiService
	PricingV1       *PricingV1.DefaultApiService
	PricingV2       *PricingV2.DefaultApiService
	ProxyV1         *ProxyV1.DefaultApiService
	ServerlessV1    *ServerlessV1.DefaultApiService
	StudioV1        *StudioV1.DefaultApiService
	StudioV2        *StudioV2.DefaultApiService
	SupersimV1      *SupersimV1.DefaultApiService
	SyncV1          *SyncV1.DefaultApiService
	TaskrouterV1    *TaskrouterV1.DefaultApiService
	TrunkingV1      *TrunkingV1.DefaultApiService
	TrusthubV1      *TrusthubV1.DefaultApiService
	VerifyV2        *VerifyV2.DefaultApiService
	VideoV1         *VideoV1.DefaultApiService
	VoiceV1         *VoiceV1.DefaultApiService
	WirelessV1      *WirelessV1.DefaultApiService
}

// Meta holds relevant pagination resources.
type Meta struct {
	FirstPageURL    *string `json:"first_page_url"`
	Key             *string `json:"key"`
	LastPageURL     *string `json:"last_page_url,omitempty"`
	NextPageURL     *string `json:"next_page_url"`
	Page            *int    `json:"page"`
	PageSize        *int    `json:"page_size"`
	PreviousPageURL *string `json:"previous_page_url"`
	URL             *string `json:"url"`
}

type RestClientParams struct {
	AccountSid string
}

// NewRestClientWithParams provides an initialized Twilio RestClient with params.
func NewRestClientWithParams(username string, password string, params RestClientParams) *RestClient {
	c := &RestClient{
		Client: &client.Client{
			Credentials: client.NewCredentials(username, password),
			BaseURL:     "twilio.com",
			Edge:        os.Getenv("TWILIO_EDGE"),
			Region:      os.Getenv("TWILIO_REGION"),
			AccountSid:  username,
		},
	}

	if params.AccountSid != "" {
		c.AccountSid = params.AccountSid
	}

	c.AccountsV1 = AccountsV1.NewDefaultApiService(c.Client)
	c.ApiV2010 = ApiV2010.NewDefaultApiService(c.Client)
	c.AutopilotV1 = AutopilotV1.NewDefaultApiService(c.Client)
	c.BulkexportsV1 = BulkexportsV1.NewDefaultApiService(c.Client)
	c.ChatV1 = ChatV1.NewDefaultApiService(c.Client)
	c.ChatV2 = ChatV2.NewDefaultApiService(c.Client)
	c.ConversationsV1 = ConversationsV1.NewDefaultApiService(c.Client)
	c.EventsV1 = EventsV1.NewDefaultApiService(c.Client)
	c.FaxV1 = FaxV1.NewDefaultApiService(c.Client)
	c.FlexV1 = FlexV1.NewDefaultApiService(c.Client)
	c.InsightsV1 = InsightsV1.NewDefaultApiService(c.Client)
	c.Ip_MessagingV1 = Ip_MessagingV1.NewDefaultApiService(c.Client)
	c.Ip_MessagingV2 = Ip_MessagingV2.NewDefaultApiService(c.Client)
	c.LookupsV1 = LookupsV1.NewDefaultApiService(c.Client)
	c.MessagingV1 = MessagingV1.NewDefaultApiService(c.Client)
	c.MonitorV1 = MonitorV1.NewDefaultApiService(c.Client)
	c.NotifyV1 = NotifyV1.NewDefaultApiService(c.Client)
	c.NumbersV2 = NumbersV2.NewDefaultApiService(c.Client)
	c.PricingV1 = PricingV1.NewDefaultApiService(c.Client)
	c.PricingV2 = PricingV2.NewDefaultApiService(c.Client)
	c.ProxyV1 = ProxyV1.NewDefaultApiService(c.Client)
	c.ServerlessV1 = ServerlessV1.NewDefaultApiService(c.Client)
	c.StudioV1 = StudioV1.NewDefaultApiService(c.Client)
	c.StudioV2 = StudioV2.NewDefaultApiService(c.Client)
	c.SupersimV1 = SupersimV1.NewDefaultApiService(c.Client)
	c.SyncV1 = SyncV1.NewDefaultApiService(c.Client)
	c.TaskrouterV1 = TaskrouterV1.NewDefaultApiService(c.Client)
	c.TrunkingV1 = TrunkingV1.NewDefaultApiService(c.Client)
	c.TrusthubV1 = TrusthubV1.NewDefaultApiService(c.Client)
	c.VerifyV2 = VerifyV2.NewDefaultApiService(c.Client)
	c.VideoV1 = VideoV1.NewDefaultApiService(c.Client)
	c.VoiceV1 = VoiceV1.NewDefaultApiService(c.Client)
	c.WirelessV1 = WirelessV1.NewDefaultApiService(c.Client)

	return c
}

// NewRestClient provides an initialized Twilio RestClient.
func NewRestClient(username string, password string) *RestClient {
	return NewRestClientWithParams(username, password, RestClientParams{
		AccountSid: username,
	})
}

// SetTimeout sets the Timeout for Twilio HTTP requests.
func (c *RestClient) SetTimeout(timeout time.Duration) {
	c.Client.SetTimeout(timeout)
}

// SetEdge sets the Edge for the Twilio request.
func (c *RestClient) SetEdge(edge string) {
	c.Client.Edge = edge
}

// SetRegion sets the Region for the Twilio request. Defaults to "us1" if an edge is provided.
func (c *RestClient) SetRegion(region string) {
	c.Client.Region = region
}
<commit_msg>chore!: rename Ip_MessagingVx to IpMessagingVx (#79)<commit_after>// Package twilio provides bindings for Twilio's REST APIs.
package twilio

import (
	"os"
	"time"

	"github.com/twilio/twilio-go/client"
	AccountsV1 "github.com/twilio/twilio-go/rest/accounts/v1"
	ApiV2010 "github.com/twilio/twilio-go/rest/api/v2010"
	AutopilotV1 "github.com/twilio/twilio-go/rest/autopilot/v1"
	BulkexportsV1 "github.com/twilio/twilio-go/rest/bulkexports/v1"
	ChatV1 "github.com/twilio/twilio-go/rest/chat/v1"
	ChatV2 "github.com/twilio/twilio-go/rest/chat/v2"
	ConversationsV1 "github.com/twilio/twilio-go/rest/conversations/v1"
	EventsV1 "github.com/twilio/twilio-go/rest/events/v1"
	FaxV1 "github.com/twilio/twilio-go/rest/fax/v1"
	FlexV1 "github.com/twilio/twilio-go/rest/flex/v1"
	InsightsV1 "github.com/twilio/twilio-go/rest/insights/v1"
	IpMessagingV1 "github.com/twilio/twilio-go/rest/ip_messaging/v1"
	IpMessagingV2 "github.com/twilio/twilio-go/rest/ip_messaging/v2"
	LookupsV1 "github.com/twilio/twilio-go/rest/lookups/v1"
	MessagingV1 "github.com/twilio/twilio-go/rest/messaging/v1"
	MonitorV1 "github.com/twilio/twilio-go/rest/monitor/v1"
	NotifyV1 "github.com/twilio/twilio-go/rest/notify/v1"
	NumbersV2 "github.com/twilio/twilio-go/rest/numbers/v2"
	PricingV1 "github.com/twilio/twilio-go/rest/pricing/v1"
	PricingV2 "github.com/twilio/twilio-go/rest/pricing/v2"
	ProxyV1 "github.com/twilio/twilio-go/rest/proxy/v1"
	ServerlessV1 "github.com/twilio/twilio-go/rest/serverless/v1"
	StudioV1 "github.com/twilio/twilio-go/rest/studio/v1"
	StudioV2 "github.com/twilio/twilio-go/rest/studio/v2"
	SupersimV1 "github.com/twilio/twilio-go/rest/supersim/v1"
	SyncV1 "github.com/twilio/twilio-go/rest/sync/v1"
	TaskrouterV1 "github.com/twilio/twilio-go/rest/taskrouter/v1"
	TrunkingV1 "github.com/twilio/twilio-go/rest/trunking/v1"
	TrusthubV1 "github.com/twilio/twilio-go/rest/trusthub/v1"
	VerifyV2 "github.com/twilio/twilio-go/rest/verify/v2"
	VideoV1 "github.com/twilio/twilio-go/rest/video/v1"
	VoiceV1 "github.com/twilio/twilio-go/rest/voice/v1"
	WirelessV1 "github.com/twilio/twilio-go/rest/wireless/v1"
)

// RestClient provides access to Twilio services.
type RestClient struct {
	*client.Client
	AccountsV1      *AccountsV1.DefaultApiService
	ApiV2010        *ApiV2010.DefaultApiService
	AutopilotV1     *AutopilotV1.DefaultApiService
	BulkexportsV1   *BulkexportsV1.DefaultApiService
	ChatV1          *ChatV1.DefaultApiService
	ChatV2          *ChatV2.DefaultApiService
	ConversationsV1 *ConversationsV1.DefaultApiService
	EventsV1        *EventsV1.DefaultApiService
	FaxV1           *FaxV1.DefaultApiService
	FlexV1          *FlexV1.DefaultApiService
	InsightsV1      *InsightsV1.DefaultApiService
	IpMessagingV1   *IpMessagingV1.DefaultApiService
	IpMessagingV2   *IpMessagingV2.DefaultApiService
	LookupsV1       *LookupsV1.DefaultApiService
	MessagingV1     *MessagingV1.DefaultApiService
	MonitorV1       *MonitorV1.DefaultApiService
	NotifyV1        *NotifyV1.DefaultApiService
	NumbersV2       *NumbersV2.DefaultApiService
	PricingV1       *PricingV1.DefaultApiService
	PricingV2       *PricingV2.DefaultApiService
	ProxyV1         *ProxyV1.DefaultApiService
	ServerlessV1    *ServerlessV1.DefaultApiService
	StudioV1        *StudioV1.DefaultApiService
	StudioV2        *StudioV2.DefaultApiService
	SupersimV1      *SupersimV1.DefaultApiService
	SyncV1          *SyncV1.DefaultApiService
	TaskrouterV1    *TaskrouterV1.DefaultApiService
	TrunkingV1      *TrunkingV1.DefaultApiService
	TrusthubV1      *TrusthubV1.DefaultApiService
	VerifyV2        *VerifyV2.DefaultApiService
	VideoV1         *VideoV1.DefaultApiService
	VoiceV1         *VoiceV1.DefaultApiService
	WirelessV1      *WirelessV1.DefaultApiService
}

// Meta holds relevant pagination resources.
type Meta struct {
	FirstPageURL    *string `json:"first_page_url"`
	Key             *string `json:"key"`
	LastPageURL     *string `json:"last_page_url,omitempty"`
	NextPageURL     *string `json:"next_page_url"`
	Page            *int    `json:"page"`
	PageSize        *int    `json:"page_size"`
	PreviousPageURL *string `json:"previous_page_url"`
	URL             *string `json:"url"`
}

type RestClientParams struct {
	AccountSid string
}

// NewRestClientWithParams provides an initialized Twilio RestClient with params.
func NewRestClientWithParams(username string, password string, params RestClientParams) *RestClient {
	c := &RestClient{
		Client: &client.Client{
			Credentials: client.NewCredentials(username, password),
			BaseURL:     "twilio.com",
			Edge:        os.Getenv("TWILIO_EDGE"),
			Region:      os.Getenv("TWILIO_REGION"),
			AccountSid:  username,
		},
	}

	if params.AccountSid != "" {
		c.AccountSid = params.AccountSid
	}

	c.AccountsV1 = AccountsV1.NewDefaultApiService(c.Client)
	c.ApiV2010 = ApiV2010.NewDefaultApiService(c.Client)
	c.AutopilotV1 = AutopilotV1.NewDefaultApiService(c.Client)
	c.BulkexportsV1 = BulkexportsV1.NewDefaultApiService(c.Client)
	c.ChatV1 = ChatV1.NewDefaultApiService(c.Client)
	c.ChatV2 = ChatV2.NewDefaultApiService(c.Client)
	c.ConversationsV1 = ConversationsV1.NewDefaultApiService(c.Client)
	c.EventsV1 = EventsV1.NewDefaultApiService(c.Client)
	c.FaxV1 = FaxV1.NewDefaultApiService(c.Client)
	c.FlexV1 = FlexV1.NewDefaultApiService(c.Client)
	c.InsightsV1 = InsightsV1.NewDefaultApiService(c.Client)
	c.IpMessagingV1 = IpMessagingV1.NewDefaultApiService(c.Client)
	c.IpMessagingV2 = IpMessagingV2.NewDefaultApiService(c.Client)
	c.LookupsV1 = LookupsV1.NewDefaultApiService(c.Client)
	c.MessagingV1 = MessagingV1.NewDefaultApiService(c.Client)
	c.MonitorV1 = MonitorV1.NewDefaultApiService(c.Client)
	c.NotifyV1 = NotifyV1.NewDefaultApiService(c.Client)
	c.NumbersV2 = NumbersV2.NewDefaultApiService(c.Client)
	c.PricingV1 = PricingV1.NewDefaultApiService(c.Client)
	c.PricingV2 = PricingV2.NewDefaultApiService(c.Client)
	c.ProxyV1 = ProxyV1.NewDefaultApiService(c.Client)
	c.ServerlessV1 = ServerlessV1.NewDefaultApiService(c.Client)
	c.StudioV1 = StudioV1.NewDefaultApiService(c.Client)
	c.StudioV2 = StudioV2.NewDefaultApiService(c.Client)
	c.SupersimV1 = SupersimV1.NewDefaultApiService(c.Client)
	c.SyncV1 = SyncV1.NewDefaultApiService(c.Client)
	c.TaskrouterV1 = TaskrouterV1.NewDefaultApiService(c.Client)
	c.TrunkingV1 = TrunkingV1.NewDefaultApiService(c.Client)
	c.TrusthubV1 = TrusthubV1.NewDefaultApiService(c.Client)
	c.VerifyV2 = VerifyV2.NewDefaultApiService(c.Client)
	c.VideoV1 = VideoV1.NewDefaultApiService(c.Client)
	c.VoiceV1 = VoiceV1.NewDefaultApiService(c.Client)
	c.WirelessV1 = WirelessV1.NewDefaultApiService(c.Client)

	return c
}

// NewRestClient provides an initialized Twilio RestClient.
func NewRestClient(username string, password string) *RestClient {
	return NewRestClientWithParams(username, password, RestClientParams{
		AccountSid: username,
	})
}

// SetTimeout sets the Timeout for Twilio HTTP requests.
func (c *RestClient) SetTimeout(timeout time.Duration) {
	c.Client.SetTimeout(timeout)
}

// SetEdge sets the Edge for the Twilio request.
func (c *RestClient) SetEdge(edge string) {
	c.Client.Edge = edge
}

// SetRegion sets the Region for the Twilio request. Defaults to "us1" if an edge is provided.
func (c *RestClient) SetRegion(region string) {
	c.Client.Region = region
}
<|endoftext|>


<commit_before>package service

import (
	"github.com/QubitProducts/bamboo/Godeps/_workspace/src/github.com/samuel/go-zookeeper/zk"
	conf "github.com/QubitProducts/bamboo/configuration"
	"net/url"
	"log"
)

type ZKStorage struct {
	conn *zk.Conn
	conf conf.Zookeeper
	acl []zk.ACL
}

func NewZKStorage(conn *zk.Conn, conf conf.Zookeeper) (s *ZKStorage, err error) {
	s = &ZKStorage{
		conn: conn,
		conf: conf,
		acl: defaultACL(),
	}
	err = s.ensurePathExists()
	return s, err
}

func (z *ZKStorage) All() (services []Service, err error) {
	err = z.ensurePathExists()
	if err != nil {
		return
	}

	keys, _, err := z.conn.Children(z.conf.Path)
	if err != nil {
		return
	}

	services = make([]Service, 0, len(keys))
	for _, childPath := range keys {
		body, _, err := z.conn.Get(z.conf.Path + "/" + childPath)
		if err != nil {
			return nil, err
		}

		path, err := unescapePath(childPath)
		if err != nil {
			return nil, err
		}

		// We tolerate being unable to decode a service body, as may be new version running simultaneously
		repr, err := ParseServiceRepr(body, path)
		if err != nil {
			log.Printf("Failed to parse service at %v: %v", path, err)
			continue
		}

		services = append(services, repr.Service())
	}

	return
}

func (z *ZKStorage) Upsert(service Service) (err error) {
	repr := NewV2ServiceRepr(service.Id, service.Config)

	body, err := repr.Serialize()
	if err != nil {
		return
	}

	err = z.ensurePathExists()
	if err != nil {
		return err
	}

	path := z.servicePath(service.Id)

	ok, _, err := z.conn.Exists(path)
	if err != nil {
		return
	}

	if ok {
		_, err = z.conn.Set(path, body, -1)
		if err != nil {
			log.Print("Failed to set path", err)
			return
		}

		// Trigger an event on the parent
		_, err = z.conn.Set(z.conf.Path, []byte{}, -1)
		if err != nil {
			log.Print("Failed to trigger event on parent", err)
			err = nil
		}

	} else {
		_, err = z.conn.Create(path, body, 0, z.acl)
		if err != nil {
			log.Print("Failed to set create", err)
			return
		}
	}
	return
}

func (z *ZKStorage) Delete(serviceId string) error {
	path := z.servicePath(serviceId)
	return z.conn.Delete(path, -1)
}

func (z *ZKStorage) servicePath(id string) string {
	return z.conf.Path + "/" + escapePath(id)
}

func (z *ZKStorage) ensurePathExists() error {
	pathExists, _, _ := z.conn.Exists(z.conf.Path)
	if pathExists {
		return nil
	}

	// This is a fairly rare, and fairly critical, operation, so I'm going to be verbose
	log.Print("Creating base zk path", z.conf.Path)
	_, err := z.conn.Create(z.conf.Path, []byte{}, 0, z.acl)
	if err != nil {
		log.Print("Failed to create base zk path", err)
	}

	return err
}

func defaultACL() []zk.ACL {
	return []zk.ACL{zk.ACL{Perms: zk.PermAll, Scheme: "world", ID: "anyone"}}
}

func escapePath(path string) string {
	return url.QueryEscape(path)
}

func unescapePath(path string) (string, error) {
	return url.QueryUnescape(path)
}
<commit_msg>Use MakeV2ServiceRepr over NewV2ServiceRepr<commit_after>package service

import (
	"github.com/QubitProducts/bamboo/Godeps/_workspace/src/github.com/samuel/go-zookeeper/zk"
	conf "github.com/QubitProducts/bamboo/configuration"
	"net/url"
	"log"
)

type ZKStorage struct {
	conn *zk.Conn
	conf conf.Zookeeper
	acl []zk.ACL
}

func NewZKStorage(conn *zk.Conn, conf conf.Zookeeper) (s *ZKStorage, err error) {
	s = &ZKStorage{
		conn: conn,
		conf: conf,
		acl: defaultACL(),
	}
	err = s.ensurePathExists()
	return s, err
}

func (z *ZKStorage) All() (services []Service, err error) {
	err = z.ensurePathExists()
	if err != nil {
		return
	}

	keys, _, err := z.conn.Children(z.conf.Path)
	if err != nil {
		return
	}

	services = make([]Service, 0, len(keys))
	for _, childPath := range keys {
		body, _, err := z.conn.Get(z.conf.Path + "/" + childPath)
		if err != nil {
			return nil, err
		}

		path, err := unescapePath(childPath)
		if err != nil {
			return nil, err
		}

		// We tolerate being unable to decode a service body, as may be new version running simultaneously
		repr, err := ParseServiceRepr(body, path)
		if err != nil {
			log.Printf("Failed to parse service at %v: %v", path, err)
			continue
		}

		services = append(services, repr.Service())
	}

	return
}

func (z *ZKStorage) Upsert(service Service) (err error) {
	repr := MakeV2ServiceRepr(service)

	body, err := repr.Serialize()
	if err != nil {
		return
	}

	err = z.ensurePathExists()
	if err != nil {
		return err
	}

	path := z.servicePath(service.Id)

	ok, _, err := z.conn.Exists(path)
	if err != nil {
		return
	}

	if ok {
		_, err = z.conn.Set(path, body, -1)
		if err != nil {
			log.Print("Failed to set path", err)
			return
		}

		// Trigger an event on the parent
		_, err = z.conn.Set(z.conf.Path, []byte{}, -1)
		if err != nil {
			log.Print("Failed to trigger event on parent", err)
			err = nil
		}

	} else {
		_, err = z.conn.Create(path, body, 0, z.acl)
		if err != nil {
			log.Print("Failed to set create", err)
			return
		}
	}
	return
}

func (z *ZKStorage) Delete(serviceId string) error {
	path := z.servicePath(serviceId)
	return z.conn.Delete(path, -1)
}

func (z *ZKStorage) servicePath(id string) string {
	return z.conf.Path + "/" + escapePath(id)
}

func (z *ZKStorage) ensurePathExists() error {
	pathExists, _, _ := z.conn.Exists(z.conf.Path)
	if pathExists {
		return nil
	}

	// This is a fairly rare, and fairly critical, operation, so I'm going to be verbose
	log.Print("Creating base zk path", z.conf.Path)
	_, err := z.conn.Create(z.conf.Path, []byte{}, 0, z.acl)
	if err != nil {
		log.Print("Failed to create base zk path", err)
	}

	return err
}

func defaultACL() []zk.ACL {
	return []zk.ACL{zk.ACL{Perms: zk.PermAll, Scheme: "world", ID: "anyone"}}
}

func escapePath(path string) string {
	return url.QueryEscape(path)
}

func unescapePath(path string) (string, error) {
	return url.QueryUnescape(path)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"

	"github.com/makebyte/mutago"
)

func main() {
	f, err := mutago.Open("ok.mp3")
	defer f.Close()
	if err != nil {
		fmt.Println(err)
	}

	val, err := f.Get("TALB")
	if err != nil {
		fmt.Println(err)
	}

	fmt.Println(val)
}
<commit_msg>Updated example<commit_after>package main

import (
	"fmt"

	"github.com/makebyte/mutago"
)

func main() {
	f, err := mutago.Open("ok.mp3")
	defer f.Close()
	if err != nil {
		fmt.Println(err)
	}

	val, err := f.Get("TIT2") // Get title value
	if err != nil {
		fmt.Println(err)
	}

	fmt.Println(val)

	tags := f.List() // Get all available tags (Not values)
	for _, y := range tags {
		fmt.Println(y) // Print all tags
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2015 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package testing

import (
	"math/rand"
	"reflect"
	"strconv"
	"testing"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/resource"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/fields"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/labels"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/runtime"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/types"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
	"github.com/fsouza/go-dockerclient"
	"github.com/google/gofuzz"

	"speter.net/go/exp/math/dec/inf"
)

// FuzzerFor can randomly populate api objects that are destined for version.
func FuzzerFor(t *testing.T, version string, src rand.Source) *fuzz.Fuzzer {
	f := fuzz.New().NilChance(.5).NumElements(1, 1)
	if src != nil {
		f.RandSource(src)
	}
	f.Funcs(
		func(j *runtime.PluginBase, c fuzz.Continue) {
			// Do nothing; this struct has only a Kind field and it must stay blank in memory.
		},
		func(j *runtime.TypeMeta, c fuzz.Continue) {
			// We have to customize the randomization of TypeMetas because their
			// APIVersion and Kind must remain blank in memory.
			j.APIVersion = ""
			j.Kind = ""
		},
		func(j *api.TypeMeta, c fuzz.Continue) {
			// We have to customize the randomization of TypeMetas because their
			// APIVersion and Kind must remain blank in memory.
			j.APIVersion = ""
			j.Kind = ""
		},
		func(j *api.ObjectMeta, c fuzz.Continue) {
			j.Name = c.RandString()
			j.ResourceVersion = strconv.FormatUint(c.RandUint64(), 10)
			j.SelfLink = c.RandString()
			j.UID = types.UID(c.RandString())
			j.GenerateName = c.RandString()

			var sec, nsec int64
			c.Fuzz(&sec)
			c.Fuzz(&nsec)
			j.CreationTimestamp = util.Unix(sec, nsec).Rfc3339Copy()
		},
		func(j *api.ObjectReference, c fuzz.Continue) {
			// We have to customize the randomization of TypeMetas because their
			// APIVersion and Kind must remain blank in memory.
			j.APIVersion = c.RandString()
			j.Kind = c.RandString()
			j.Namespace = c.RandString()
			j.Name = c.RandString()
			j.ResourceVersion = strconv.FormatUint(c.RandUint64(), 10)
			j.FieldPath = c.RandString()
		},
		func(j *api.ListMeta, c fuzz.Continue) {
			j.ResourceVersion = strconv.FormatUint(c.RandUint64(), 10)
			j.SelfLink = c.RandString()
		},
		func(j *api.ListOptions, c fuzz.Continue) {
			// TODO: add some parsing
			j.LabelSelector, _ = labels.Parse("a=b")
			j.FieldSelector, _ = fields.ParseSelector("a=b")
		},
		func(j *api.PodPhase, c fuzz.Continue) {
			statuses := []api.PodPhase{api.PodPending, api.PodRunning, api.PodFailed, api.PodUnknown}
			*j = statuses[c.Rand.Intn(len(statuses))]
		},
		func(j *api.PodTemplateSpec, c fuzz.Continue) {
			// TODO: v1beta1/2 can't round trip a nil template correctly, fix by having v1beta1/2
			// conversion compare converted object to nil via DeepEqual
			j.ObjectMeta = api.ObjectMeta{}
			c.Fuzz(&j.ObjectMeta)
			j.ObjectMeta = api.ObjectMeta{Labels: j.ObjectMeta.Labels}
			j.Spec = api.PodSpec{}
			c.Fuzz(&j.Spec)
		},
		func(j *api.Binding, c fuzz.Continue) {
			c.Fuzz(&j.ObjectMeta)
			j.Target.Name = c.RandString()
		},
		func(j *api.ReplicationControllerSpec, c fuzz.Continue) {
			c.FuzzNoCustom(j)   // fuzz self without calling this function again
			j.TemplateRef = nil // this is required for round trip
		},
		func(j *api.ReplicationControllerStatus, c fuzz.Continue) {
			// only replicas round trips
			j.Replicas = int(c.RandUint64())
		},
		func(j *api.List, c fuzz.Continue) {
			c.FuzzNoCustom(j) // fuzz self without calling this function again
			if j.Items == nil {
				j.Items = []runtime.Object{}
			}
		},
		func(j *runtime.Object, c fuzz.Continue) {
			if c.RandBool() {
				*j = &runtime.Unknown{
					TypeMeta: runtime.TypeMeta{Kind: "Something", APIVersion: "unknown"},
					RawJSON:  []byte(`{"apiVersion":"unknown","kind":"Something","someKey":"someValue"}`),
				}
			} else {
				types := []runtime.Object{&api.Pod{}, &api.ReplicationController{}}
				t := types[c.Rand.Intn(len(types))]
				c.Fuzz(t)
				*j = t
			}
		},
		func(pb map[docker.Port][]docker.PortBinding, c fuzz.Continue) {
			// This is necessary because keys with nil values get omitted.
			// TODO: Is this a bug?
			pb[docker.Port(c.RandString())] = []docker.PortBinding{
				{c.RandString(), c.RandString()},
				{c.RandString(), c.RandString()},
			}
		},
		func(pm map[string]docker.PortMapping, c fuzz.Continue) {
			// This is necessary because keys with nil values get omitted.
			// TODO: Is this a bug?
			pm[c.RandString()] = docker.PortMapping{
				c.RandString(): c.RandString(),
			}
		},
		func(q *resource.Quantity, c fuzz.Continue) {
			// Real Quantity fuzz testing is done elsewhere;
			// this limited subset of functionality survives
			// round-tripping to v1beta1/2.
			q.Amount = &inf.Dec{}
			q.Format = resource.DecimalExponent
			//q.Amount.SetScale(inf.Scale(-c.Intn(12)))
			q.Amount.SetUnscaled(c.Int63n(1000))
		},
		func(p *api.PullPolicy, c fuzz.Continue) {
			policies := []api.PullPolicy{api.PullAlways, api.PullNever, api.PullIfNotPresent}
			*p = policies[c.Rand.Intn(len(policies))]
		},
		func(rp *api.RestartPolicy, c fuzz.Continue) {
			policies := []api.RestartPolicy{api.RestartPolicyAlways, api.RestartPolicyNever, api.RestartPolicyOnFailure}
			*rp = policies[c.Rand.Intn(len(policies))]
		},
		func(vs *api.VolumeSource, c fuzz.Continue) {
			// Exactly one of the fields must be set.
			v := reflect.ValueOf(vs).Elem()
			i := int(c.RandUint64() % uint64(v.NumField()))
			v = v.Field(i).Addr()
			// Use a new fuzzer which cannot populate nil to ensure one field will be set.
			fuzz.New().NilChance(0).NumElements(1, 1).Fuzz(v.Interface())
		},
		func(d *api.DNSPolicy, c fuzz.Continue) {
			policies := []api.DNSPolicy{api.DNSClusterFirst, api.DNSDefault}
			*d = policies[c.Rand.Intn(len(policies))]
		},
		func(p *api.Protocol, c fuzz.Continue) {
			protocols := []api.Protocol{api.ProtocolTCP, api.ProtocolUDP}
			*p = protocols[c.Rand.Intn(len(protocols))]
		},
		func(p *api.AffinityType, c fuzz.Continue) {
			types := []api.AffinityType{api.AffinityTypeClientIP, api.AffinityTypeNone}
			*p = types[c.Rand.Intn(len(types))]
		},
		func(ct *api.Container, c fuzz.Continue) {
			c.FuzzNoCustom(ct)                                          // fuzz self without calling this function again
			ct.TerminationMessagePath = "/" + ct.TerminationMessagePath // Must be non-empty
		},
		func(e *api.Event, c fuzz.Continue) {
			c.FuzzNoCustom(e) // fuzz self without calling this function again
			// Fix event count to 1, otherwise, if a v1beta1 or v1beta2 event has a count set arbitrarily, it's count is ignored
			if e.FirstTimestamp.IsZero() {
				e.Count = 1
			} else {
				c.Fuzz(&e.Count)
			}
		},
		func(s *api.Secret, c fuzz.Continue) {
			c.FuzzNoCustom(s) // fuzz self without calling this function again
			s.Type = api.SecretTypeOpaque
		},
		func(pv *api.PersistentVolume, c fuzz.Continue) {
			c.FuzzNoCustom(pv) // fuzz self without calling this function again
			pv.Status.Phase = api.VolumePending
		},
		func(pvc *api.PersistentVolumeClaim, c fuzz.Continue) {
			c.FuzzNoCustom(pvc) // fuzz self without calling this function again
			pvc.Status.Phase = api.ClaimPending
		},
		func(s *api.NamespaceSpec, c fuzz.Continue) {
			s.Finalizers = []api.FinalizerName{api.FinalizerKubernetes}
		},
		func(s *api.NamespaceStatus, c fuzz.Continue) {
			s.Phase = api.NamespaceActive
		},
		func(http *api.HTTPGetAction, c fuzz.Continue) {
			c.FuzzNoCustom(http)        // fuzz self without calling this function again
			http.Path = "/" + http.Path // can't be blank
		},
		func(ss *api.ServiceSpec, c fuzz.Continue) {
			c.FuzzNoCustom(ss) // fuzz self without calling this function again
			if len(ss.Ports) == 0 {
				// There must be at least 1 port.
				ss.Ports = append(ss.Ports, api.ServicePort{})
				c.Fuzz(&ss.Ports[0])
			}
			for i := range ss.Ports {
				switch ss.Ports[i].TargetPort.Kind {
				case util.IntstrInt:
					ss.Ports[i].TargetPort.IntVal = 1 + ss.Ports[i].TargetPort.IntVal%65535 // non-zero
				case util.IntstrString:
					ss.Ports[i].TargetPort.StrVal = "x" + ss.Ports[i].TargetPort.StrVal // non-empty
				}
			}
		},
		func(n *api.Node, c fuzz.Continue) {
			c.FuzzNoCustom(n)
			n.Spec.ExternalID = "external"
		},
	)
	return f
}
<commit_msg>addressed feedback.  added opt-in cmd line flag<commit_after>/*
Copyright 2015 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package testing

import (
	"math/rand"
	"reflect"
	"strconv"
	"testing"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/resource"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/fields"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/labels"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/runtime"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/types"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
	"github.com/fsouza/go-dockerclient"
	"github.com/google/gofuzz"

	"speter.net/go/exp/math/dec/inf"
)

// FuzzerFor can randomly populate api objects that are destined for version.
func FuzzerFor(t *testing.T, version string, src rand.Source) *fuzz.Fuzzer {
	f := fuzz.New().NilChance(.5).NumElements(1, 1)
	if src != nil {
		f.RandSource(src)
	}
	f.Funcs(
		func(j *runtime.PluginBase, c fuzz.Continue) {
			// Do nothing; this struct has only a Kind field and it must stay blank in memory.
		},
		func(j *runtime.TypeMeta, c fuzz.Continue) {
			// We have to customize the randomization of TypeMetas because their
			// APIVersion and Kind must remain blank in memory.
			j.APIVersion = ""
			j.Kind = ""
		},
		func(j *api.TypeMeta, c fuzz.Continue) {
			// We have to customize the randomization of TypeMetas because their
			// APIVersion and Kind must remain blank in memory.
			j.APIVersion = ""
			j.Kind = ""
		},
		func(j *api.ObjectMeta, c fuzz.Continue) {
			j.Name = c.RandString()
			j.ResourceVersion = strconv.FormatUint(c.RandUint64(), 10)
			j.SelfLink = c.RandString()
			j.UID = types.UID(c.RandString())
			j.GenerateName = c.RandString()

			var sec, nsec int64
			c.Fuzz(&sec)
			c.Fuzz(&nsec)
			j.CreationTimestamp = util.Unix(sec, nsec).Rfc3339Copy()
		},
		func(j *api.ObjectReference, c fuzz.Continue) {
			// We have to customize the randomization of TypeMetas because their
			// APIVersion and Kind must remain blank in memory.
			j.APIVersion = c.RandString()
			j.Kind = c.RandString()
			j.Namespace = c.RandString()
			j.Name = c.RandString()
			j.ResourceVersion = strconv.FormatUint(c.RandUint64(), 10)
			j.FieldPath = c.RandString()
		},
		func(j *api.ListMeta, c fuzz.Continue) {
			j.ResourceVersion = strconv.FormatUint(c.RandUint64(), 10)
			j.SelfLink = c.RandString()
		},
		func(j *api.ListOptions, c fuzz.Continue) {
			// TODO: add some parsing
			j.LabelSelector, _ = labels.Parse("a=b")
			j.FieldSelector, _ = fields.ParseSelector("a=b")
		},
		func(j *api.PodPhase, c fuzz.Continue) {
			statuses := []api.PodPhase{api.PodPending, api.PodRunning, api.PodFailed, api.PodUnknown}
			*j = statuses[c.Rand.Intn(len(statuses))]
		},
		func(j *api.PodTemplateSpec, c fuzz.Continue) {
			// TODO: v1beta1/2 can't round trip a nil template correctly, fix by having v1beta1/2
			// conversion compare converted object to nil via DeepEqual
			j.ObjectMeta = api.ObjectMeta{}
			c.Fuzz(&j.ObjectMeta)
			j.ObjectMeta = api.ObjectMeta{Labels: j.ObjectMeta.Labels}
			j.Spec = api.PodSpec{}
			c.Fuzz(&j.Spec)
		},
		func(j *api.Binding, c fuzz.Continue) {
			c.Fuzz(&j.ObjectMeta)
			j.Target.Name = c.RandString()
		},
		func(j *api.ReplicationControllerSpec, c fuzz.Continue) {
			c.FuzzNoCustom(j)   // fuzz self without calling this function again
			j.TemplateRef = nil // this is required for round trip
		},
		func(j *api.ReplicationControllerStatus, c fuzz.Continue) {
			// only replicas round trips
			j.Replicas = int(c.RandUint64())
		},
		func(j *api.List, c fuzz.Continue) {
			c.FuzzNoCustom(j) // fuzz self without calling this function again
			if j.Items == nil {
				j.Items = []runtime.Object{}
			}
		},
		func(j *runtime.Object, c fuzz.Continue) {
			if c.RandBool() {
				*j = &runtime.Unknown{
					TypeMeta: runtime.TypeMeta{Kind: "Something", APIVersion: "unknown"},
					RawJSON:  []byte(`{"apiVersion":"unknown","kind":"Something","someKey":"someValue"}`),
				}
			} else {
				types := []runtime.Object{&api.Pod{}, &api.ReplicationController{}}
				t := types[c.Rand.Intn(len(types))]
				c.Fuzz(t)
				*j = t
			}
		},
		func(pb map[docker.Port][]docker.PortBinding, c fuzz.Continue) {
			// This is necessary because keys with nil values get omitted.
			// TODO: Is this a bug?
			pb[docker.Port(c.RandString())] = []docker.PortBinding{
				{c.RandString(), c.RandString()},
				{c.RandString(), c.RandString()},
			}
		},
		func(pm map[string]docker.PortMapping, c fuzz.Continue) {
			// This is necessary because keys with nil values get omitted.
			// TODO: Is this a bug?
			pm[c.RandString()] = docker.PortMapping{
				c.RandString(): c.RandString(),
			}
		},
		func(q *resource.Quantity, c fuzz.Continue) {
			// Real Quantity fuzz testing is done elsewhere;
			// this limited subset of functionality survives
			// round-tripping to v1beta1/2.
			q.Amount = &inf.Dec{}
			q.Format = resource.DecimalExponent
			//q.Amount.SetScale(inf.Scale(-c.Intn(12)))
			q.Amount.SetUnscaled(c.Int63n(1000))
		},
		func(p *api.PullPolicy, c fuzz.Continue) {
			policies := []api.PullPolicy{api.PullAlways, api.PullNever, api.PullIfNotPresent}
			*p = policies[c.Rand.Intn(len(policies))]
		},
		func(rp *api.RestartPolicy, c fuzz.Continue) {
			policies := []api.RestartPolicy{api.RestartPolicyAlways, api.RestartPolicyNever, api.RestartPolicyOnFailure}
			*rp = policies[c.Rand.Intn(len(policies))]
		},
		func(vs *api.VolumeSource, c fuzz.Continue) {
			// Exactly one of the fields must be set.
			v := reflect.ValueOf(vs).Elem()
			i := int(c.RandUint64() % uint64(v.NumField()))
			v = v.Field(i).Addr()
			// Use a new fuzzer which cannot populate nil to ensure one field will be set.
			fuzz.New().NilChance(0).NumElements(1, 1).Fuzz(v.Interface())
		},
		func(d *api.DNSPolicy, c fuzz.Continue) {
			policies := []api.DNSPolicy{api.DNSClusterFirst, api.DNSDefault}
			*d = policies[c.Rand.Intn(len(policies))]
		},
		func(p *api.Protocol, c fuzz.Continue) {
			protocols := []api.Protocol{api.ProtocolTCP, api.ProtocolUDP}
			*p = protocols[c.Rand.Intn(len(protocols))]
		},
		func(p *api.AffinityType, c fuzz.Continue) {
			types := []api.AffinityType{api.AffinityTypeClientIP, api.AffinityTypeNone}
			*p = types[c.Rand.Intn(len(types))]
		},
		func(ct *api.Container, c fuzz.Continue) {
			c.FuzzNoCustom(ct)                                          // fuzz self without calling this function again
			ct.TerminationMessagePath = "/" + ct.TerminationMessagePath // Must be non-empty
		},
		func(e *api.Event, c fuzz.Continue) {
			c.FuzzNoCustom(e) // fuzz self without calling this function again
			// Fix event count to 1, otherwise, if a v1beta1 or v1beta2 event has a count set arbitrarily, it's count is ignored
			if e.FirstTimestamp.IsZero() {
				e.Count = 1
			} else {
				c.Fuzz(&e.Count)
			}
		},
		func(s *api.Secret, c fuzz.Continue) {
			c.FuzzNoCustom(s) // fuzz self without calling this function again
			s.Type = api.SecretTypeOpaque
		},
		func(pv *api.PersistentVolume, c fuzz.Continue) {
			c.FuzzNoCustom(pv) // fuzz self without calling this function again
			types := []api.PersistentVolumePhase{api.VolumePending, api.VolumeBound, api.VolumeReleased, api.VolumeAvailable}
			pv.Status.Phase = types[c.Rand.Intn(len(types))]
		},
		func(pvc *api.PersistentVolumeClaim, c fuzz.Continue) {
			c.FuzzNoCustom(pvc) // fuzz self without calling this function again
			types := []api.PersistentVolumeClaimPhase{api.ClaimBound, api.ClaimPending}
			pvc.Status.Phase = types[c.Rand.Intn(len(types))]
		},
		func(s *api.NamespaceSpec, c fuzz.Continue) {
			s.Finalizers = []api.FinalizerName{api.FinalizerKubernetes}
		},
		func(s *api.NamespaceStatus, c fuzz.Continue) {
			s.Phase = api.NamespaceActive
		},
		func(http *api.HTTPGetAction, c fuzz.Continue) {
			c.FuzzNoCustom(http)        // fuzz self without calling this function again
			http.Path = "/" + http.Path // can't be blank
		},
		func(ss *api.ServiceSpec, c fuzz.Continue) {
			c.FuzzNoCustom(ss) // fuzz self without calling this function again
			if len(ss.Ports) == 0 {
				// There must be at least 1 port.
				ss.Ports = append(ss.Ports, api.ServicePort{})
				c.Fuzz(&ss.Ports[0])
			}
			for i := range ss.Ports {
				switch ss.Ports[i].TargetPort.Kind {
				case util.IntstrInt:
					ss.Ports[i].TargetPort.IntVal = 1 + ss.Ports[i].TargetPort.IntVal%65535 // non-zero
				case util.IntstrString:
					ss.Ports[i].TargetPort.StrVal = "x" + ss.Ports[i].TargetPort.StrVal // non-empty
				}
			}
		},
		func(n *api.Node, c fuzz.Continue) {
			c.FuzzNoCustom(n)
			n.Spec.ExternalID = "external"
		},
	)
	return f
}
<|endoftext|>


<commit_before>package whois

import (
	// "github.com/codegangsta/cli"
	// "bytes"
	"encoding/csv"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"regexp"
	"strings"
	"sync"
)

type Result struct {
	Emails []string          `json:"emails"`
	Data   map[string]string `json:"data"`
	Raw    string            `json:"raw"`
}

func ParseCSV(f io.Reader) [][]string {
	csvReader := csv.NewReader(f)
	cont, _ := csvReader.ReadAll()
	fmt.Printf("Content: %#v", cont)
	return cont
}

func Retrieve(query string) (*Result, error) {
	jwhois := fmt.Sprintf("./whois.sh")
	q := cleanDomain(query)
	cmd := exec.Command(jwhois, q)

	record, e := cmd.Output()
	check(e)

	sRecord := strings.TrimSpace(string(record))
	lines := strings.Split(sRecord, "\n")
	ourMap := toMap(lines)
	emailArray := emails(sRecord)

	result := &Result{
		Emails: emailArray,
		Data:   ourMap,
		Raw:    sRecord,
	}

	fmt.Printf("STRUCT: %#v", result)

	return result, nil
}

func AsyncRetrieve(domain string, messages chan<- string, wg *sync.WaitGroup) {
	rec, err := Retrieve(domain)
	if err == nil {
		emails := strings.Join(rec.Emails, " ")
		output := strings.Join([]string{domain, emails}, ", ")
		messages <- output
	} else {
		fmt.Println(err)
	}
	wg.Done()
}

func RetrieveJSON(query string) string {
	record, err := Retrieve(query)
	check(err)

	js, _ := json.Marshal(&record)
	return string(js)
}

func emails(rawRecord string) []string {
	lines := strings.Split(rawRecord, "\n")

	hash := toMap(lines)

	emails := getEmails(hash)

	if len(emails) == 0 {
		emails = grepEmails(rawRecord)
		fmt.Printf("slice - %#v\n", emails)
	}

	return emails
}

func grepEmails(c string) []string {
	emailRegex := "[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\\.[a-zA-Z0-9-.]+"
	r, _ := regexp.Compile(emailRegex)
	return r.FindAllString(c, -1)
}

func getEmails(h map[string]string) []string {

	fmt.Printf("emails - %#v\n", h)
	s := make([]string, 0)
	for k, v := range h {
		match, _ := regexp.MatchString("email", k)
		if match {
			s = append(s, v)
		}
	}
	return s

}

func dPrint(s ...interface{}) {
	if os.Getenv("WHOIS_DEBUG") != "" {
		fmt.Printf("%#v", s)
	}
}

func toMap(lines []string) map[string]string {

	hash := make(map[string]string)

	for _, line := range lines {
		if strings.Contains(line, ": ") {
			a := strings.SplitN(line, ":", 2)

			key := strings.ToLower(strings.TrimSpace(a[0]))
			val := strings.TrimSpace(a[1])

			hash[key] = val
		}
	}
	return hash
}

func getData(file string) string {
	dat, err := ioutil.ReadFile(file)
	check(err)

	return string(dat)
}

func check(e error) {
	if e != nil {
		fmt.Printf("ERROR: %#v", e)
		panic(1)
	}
}
<commit_msg>Improve error checking from whois<commit_after>package whois

import (
	// "github.com/codegangsta/cli"
	// "bytes"
	"encoding/csv"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"regexp"
	"strings"
	"sync"
)

type Result struct {
	Emails []string          `json:"emails"`
	Data   map[string]string `json:"data"`
	Raw    string            `json:"raw"`
}

func ParseCSV(f io.Reader) [][]string {
	csvReader := csv.NewReader(f)
	cont, _ := csvReader.ReadAll()
	fmt.Printf("Content: %#v", cont)
	return cont
}

func Retrieve(query string) (*Result, error) {
	jwhois := fmt.Sprintf("./whois.sh")
	q := cleanDomain(query)
	cmd := exec.Command(jwhois, q)

	record, e := cmd.Output()
	var sRecord string
	var ourMap map[string]string
	var emailArray []string
	if e != nil {
		sRecord = ""
		ourMap  = make(map [string]string)
		emailArray = []string{}
	} else {
		sRecord = strings.TrimSpace(string(record))
		lines   := strings.Split(sRecord, "\n")
		ourMap  = toMap(lines)
		emailArray = emails(sRecord)
	}

	result := &Result{
		Emails: emailArray,
		Data:   ourMap,
		Raw:    sRecord,
	}

	fmt.Printf("STRUCT: %#v", result)

	return result, nil
}

func AsyncRetrieve(domain string, messages chan<- string, wg *sync.WaitGroup) {
	rec, err := Retrieve(domain)
	if err == nil {
		emails := strings.Join(rec.Emails, " ")
		output := strings.Join([]string{domain, emails}, ", ")
		messages <- output
	} else {
		fmt.Println(err)
	}
	wg.Done()
}

func RetrieveJSON(query string) string {
	record, err := Retrieve(query)
	check(err)

	js, _ := json.Marshal(&record)
	return string(js)
}

func emails(rawRecord string) []string {
	lines := strings.Split(rawRecord, "\n")

	hash := toMap(lines)

	emails := getEmails(hash)

	if len(emails) == 0 {
		emails = grepEmails(rawRecord)
		fmt.Printf("slice - %#v\n", emails)
	}

	return emails
}

func grepEmails(c string) []string {
	emailRegex := "[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\\.[a-zA-Z0-9-.]+"
	r, _ := regexp.Compile(emailRegex)
	return r.FindAllString(c, -1)
}

func getEmails(h map[string]string) []string {

	fmt.Printf("emails - %#v\n", h)
	s := make([]string, 0)
	for k, v := range h {
		match, _ := regexp.MatchString("email", k)
		if match {
			s = append(s, v)
		}
	}
	return s

}

func dPrint(s ...interface{}) {
	if os.Getenv("WHOIS_DEBUG") != "" {
		fmt.Printf("%#v", s)
	}
}

func toMap(lines []string) map[string]string {

	hash := make(map[string]string)

	for _, line := range lines {
		if strings.Contains(line, ": ") {
			a := strings.SplitN(line, ":", 2)

			key := strings.ToLower(strings.TrimSpace(a[0]))
			val := strings.TrimSpace(a[1])

			hash[key] = val
		}
	}
	return hash
}

func getData(file string) string {
	dat, err := ioutil.ReadFile(file)
	check(err)

	return string(dat)
}

func check(e error) {
	if e != nil {
		fmt.Printf("ERROR: %#v", e)
		panic(1)
	}
}
<|endoftext|>