<commit_before>package main

import (
	"archive/tar"
	"bytes"
	"compress/gzip"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/appc/spec/aci"
	"github.com/appc/spec/schema"
	"github.com/appc/spec/schema/types"
)

var Debug bool

func warn(s string, i ...interface{}) {
	s = fmt.Sprintf(s, i...)
	fmt.Fprintln(os.Stderr, strings.TrimSuffix(s, "\n"))
}

func die(s string, i ...interface{}) {
	warn(s, i...)
	os.Exit(1)
}

func debug(i ...interface{}) {
	if Debug {
		s := fmt.Sprint(i...)
		fmt.Fprintln(os.Stderr, strings.TrimSuffix(s, "\n"))
	}
}

type StringVector []string

func (v *StringVector) String() string {
	return `"` + strings.Join(*v, `" "`) + `"`
}

func (v *StringVector) Set(str string) error {
	*v = append(*v, str)
	return nil
}

func main() {
	var (
		execOpts            StringVector
		goDefaultBinaryDesc string
		goBinaryOpt         string
	)

	// Find the go binary
	gocmd, err := exec.LookPath("go")

	flag.Var(&execOpts, "exec", "Parameters passed to app, can be used multiple times")
	if err != nil {
		goDefaultBinaryDesc = "Go binary to use (default: none found in $PATH, so it must be provided)"
	} else {
		goDefaultBinaryDesc = "Go binary to use (default: whatever go in $PATH)"
	}
	flag.StringVar(&goBinaryOpt, "go-binary", gocmd, goDefaultBinaryDesc)
	flag.Parse()
	if os.Getenv("GOPATH") != "" {
		warn("GOPATH envvar is ignored")
	}
	if os.Getenv("GOROOT") != "" {
		warn("GOROOT envvar is ignored, use --go-binary=\"$GOROOT/bin/go\" option instead")
	}
	if os.Getenv("GOACI_DEBUG") != "" {
		Debug = true
	}

	if goBinaryOpt == "" {
		die("go binary not found")
	}

	// Set up a temporary directory for everything (gopath and builds)
	tmpdir, err := ioutil.TempDir("", "goaci")
	if err != nil {
		die("error setting up temporary directory: %v", err)
	}
	defer os.RemoveAll(tmpdir)

	// Scratch build dir for aci
	acidir := filepath.Join(tmpdir, "aci")

	// Be explicit with gobin
	gobin := filepath.Join(tmpdir, "bin")

	// Construct args for a go get that does a static build
	args := []string{
		goBinaryOpt,
		"get",
		"-a",
		"-tags", "netgo",
		"-ldflags", "'-w'",
		// 1.4
		"-installsuffix", "cgo",
	}

	// Extract the package name (which is the last arg).
	var ns string
	for _, arg := range os.Args[1:] {
		// TODO(jonboulle): try to pass the other args on to go get?
		//		args = append(args, arg)
		ns = arg
	}

	name, err := types.NewACName(ns)
	// TODO(jonboulle): could this ever actually happen?
	if err != nil {
		die("bad app name: %v", err)
	}
	args = append(args, ns)

	// Use the last component, e.g. example.com/my/app --> app
	ofn := filepath.Base(ns) + ".aci"
	mode := os.O_CREATE | os.O_WRONLY | os.O_TRUNC
	of, err := os.OpenFile(ofn, mode, 0644)
	if err != nil {
		die("error opening output file: %v", err)
	}

	cmd := exec.Cmd{
		Env: []string{
			"GOPATH=" + tmpdir,
			"GOBIN=" + gobin,
			"CGO_ENABLED=0",
			"PATH=" + os.Getenv("PATH"),
		},
		Path:   goBinaryOpt,
		Args:   args,
		Stderr: os.Stderr,
		Stdout: os.Stdout,
	}
	debug("env:", cmd.Env)
	debug("running command:", strings.Join(cmd.Args, " "))
	if err := cmd.Run(); err != nil {
		die("error running go: %v", err)
	}

	// Check that we got 1 binary from the go get command
	fi, err := ioutil.ReadDir(gobin)
	if err != nil {
		die(err.Error())
	}
	switch {
	case len(fi) < 1:
		die("no binaries found in gobin")
	case len(fi) > 1:
		debug(fmt.Sprint(fi))
		die("can't handle multiple binaries")
	}
	fn := fi[0].Name()
	debug("found binary: ", fn)

	// Set up rootfs for ACI layout
	rfs := filepath.Join(acidir, "rootfs")
	err = os.MkdirAll(rfs, 0755)
	if err != nil {
		die(err.Error())
	}

	// Move the binary into the rootfs
	ep := filepath.Join(rfs, fn)
	err = os.Rename(filepath.Join(gobin, fn), ep)
	if err != nil {
		die(err.Error())
	}
	debug("moved binary to:", ep)

	exec := []string{filepath.Join("/", fn)}
	exec = append(exec, execOpts...)
	// Build the ACI
	im := schema.ImageManifest{
		ACKind:    types.ACKind("ImageManifest"),
		ACVersion: schema.AppContainerVersion,
		Name:      *name,
		App: &types.App{
			Exec:  exec,
			User:  "0",
			Group: "0",
		},
	}
	debug(im)

	gw := gzip.NewWriter(of)
	tr := tar.NewWriter(gw)

	defer func() {
		tr.Close()
		gw.Close()
		of.Close()
	}()

	iw := aci.NewImageWriter(im, tr)
	err = filepath.Walk(acidir, aci.BuildWalker(acidir, iw))
	if err != nil {
		die(err.Error())
	}
	err = iw.Close()
	if err != nil {
		die(err.Error())
	}
	fmt.Println("Wrote", of.Name())
}

// strip replaces all characters that are not [a-Z_] with _
func strip(in string) string {
	out := bytes.Buffer{}
	for _, c := range in {
		if !strings.ContainsRune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTVWXYZ0123456789_", c) {
			c = '_'
		}
		if _, err := out.WriteRune(c); err != nil {
			panic(err)
		}
	}
	return out.String()
}
<commit_msg>Allow specifying custom GOPATH<commit_after>package main

import (
	"archive/tar"
	"bytes"
	"compress/gzip"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/appc/spec/aci"
	"github.com/appc/spec/schema"
	"github.com/appc/spec/schema/types"
)

var Debug bool

func warn(s string, i ...interface{}) {
	s = fmt.Sprintf(s, i...)
	fmt.Fprintln(os.Stderr, strings.TrimSuffix(s, "\n"))
}

func die(s string, i ...interface{}) {
	warn(s, i...)
	os.Exit(1)
}

func debug(i ...interface{}) {
	if Debug {
		s := fmt.Sprint(i...)
		fmt.Fprintln(os.Stderr, strings.TrimSuffix(s, "\n"))
	}
}

type StringVector []string

func (v *StringVector) String() string {
	return `"` + strings.Join(*v, `" "`) + `"`
}

func (v *StringVector) Set(str string) error {
	*v = append(*v, str)
	return nil
}

func main() {
	var (
		execOpts            StringVector
		goDefaultBinaryDesc string
		goBinaryOpt         string
		goPathOpt           string
	)

	// Find the go binary
	gocmd, err := exec.LookPath("go")

	flag.Var(&execOpts, "exec", "Parameters passed to app, can be used multiple times")
	if err != nil {
		goDefaultBinaryDesc = "Go binary to use (default: none found in $PATH, so it must be provided)"
	} else {
		goDefaultBinaryDesc = "Go binary to use (default: whatever go in $PATH)"
	}
	flag.StringVar(&goBinaryOpt, "go-binary", gocmd, goDefaultBinaryDesc)
	flag.StringVar(&goPathOpt, "go-path", "", "Custom GOPATH (default: a temporary directory)")
	flag.Parse()
	if os.Getenv("GOPATH") != "" {
		warn("GOPATH envvar is ignored, use --go-path=\"$GOPATH\" option instead")
	}
	if os.Getenv("GOROOT") != "" {
		warn("GOROOT envvar is ignored, use --go-binary=\"$GOROOT/bin/go\" option instead")
	}
	if os.Getenv("GOACI_DEBUG") != "" {
		Debug = true
	}

	if goBinaryOpt == "" {
		die("go binary not found")
	}

	// Set up a temporary directory for everything (gopath and builds)
	tmpdir, err := ioutil.TempDir("", "goaci")
	if err != nil {
		die("error setting up temporary directory: %v", err)
	}
	defer os.RemoveAll(tmpdir)
	if goPathOpt == "" {
		goPathOpt = tmpdir
	}

	// Scratch build dir for aci
	acidir := filepath.Join(tmpdir, "aci")

	// Let's put final binary in tmpdir
	gobin := filepath.Join(tmpdir, "bin")

	// Construct args for a go get that does a static build
	args := []string{
		goBinaryOpt,
		"get",
		"-a",
		"-tags", "netgo",
		"-ldflags", "'-w'",
		// 1.4
		"-installsuffix", "cgo",
	}

	// Extract the package name (which is the last arg).
	var ns string
	for _, arg := range os.Args[1:] {
		// TODO(jonboulle): try to pass the other args on to go get?
		//		args = append(args, arg)
		ns = arg
	}

	name, err := types.NewACName(ns)
	// TODO(jonboulle): could this ever actually happen?
	if err != nil {
		die("bad app name: %v", err)
	}
	args = append(args, ns)

	// Use the last component, e.g. example.com/my/app --> app
	ofn := filepath.Base(ns) + ".aci"
	mode := os.O_CREATE | os.O_WRONLY | os.O_TRUNC
	of, err := os.OpenFile(ofn, mode, 0644)
	if err != nil {
		die("error opening output file: %v", err)
	}

	cmd := exec.Cmd{
		Env: []string{
			"GOPATH=" + goPathOpt,
			"GOBIN=" + gobin,
			"CGO_ENABLED=0",
			"PATH=" + os.Getenv("PATH"),
		},
		Path:   goBinaryOpt,
		Args:   args,
		Stderr: os.Stderr,
		Stdout: os.Stdout,
	}
	debug("env:", cmd.Env)
	debug("running command:", strings.Join(cmd.Args, " "))
	if err := cmd.Run(); err != nil {
		die("error running go: %v", err)
	}

	// Check that we got 1 binary from the go get command
	fi, err := ioutil.ReadDir(gobin)
	if err != nil {
		die(err.Error())
	}
	switch {
	case len(fi) < 1:
		die("no binaries found in gobin")
	case len(fi) > 1:
		debug(fmt.Sprint(fi))
		die("can't handle multiple binaries")
	}
	fn := fi[0].Name()
	debug("found binary: ", fn)

	// Set up rootfs for ACI layout
	rfs := filepath.Join(acidir, "rootfs")
	err = os.MkdirAll(rfs, 0755)
	if err != nil {
		die(err.Error())
	}

	// Move the binary into the rootfs
	ep := filepath.Join(rfs, fn)
	err = os.Rename(filepath.Join(gobin, fn), ep)
	if err != nil {
		die(err.Error())
	}
	debug("moved binary to:", ep)

	exec := []string{filepath.Join("/", fn)}
	exec = append(exec, execOpts...)
	// Build the ACI
	im := schema.ImageManifest{
		ACKind:    types.ACKind("ImageManifest"),
		ACVersion: schema.AppContainerVersion,
		Name:      *name,
		App: &types.App{
			Exec:  exec,
			User:  "0",
			Group: "0",
		},
	}
	debug(im)

	gw := gzip.NewWriter(of)
	tr := tar.NewWriter(gw)

	defer func() {
		tr.Close()
		gw.Close()
		of.Close()
	}()

	iw := aci.NewImageWriter(im, tr)
	err = filepath.Walk(acidir, aci.BuildWalker(acidir, iw))
	if err != nil {
		die(err.Error())
	}
	err = iw.Close()
	if err != nil {
		die(err.Error())
	}
	fmt.Println("Wrote", of.Name())
}

// strip replaces all characters that are not [a-Z_] with _
func strip(in string) string {
	out := bytes.Buffer{}
	for _, c := range in {
		if !strings.ContainsRune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTVWXYZ0123456789_", c) {
			c = '_'
		}
		if _, err := out.WriteRune(c); err != nil {
			panic(err)
		}
	}
	return out.String()
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"code.google.com/p/go.net/html" //Tokenizer für HTML
	"database/sql"
	"fmt"
	_ "github.com/mattn/go-sqlite3"
	"io"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"
)

const MaxOutstanding_Fetcher = 100 // Maximale Anzahl gleichzeitger Fetcher
const DEBUG = 1                    // Debugausgabe an/aus
const MaxLinkDepth = 3             // Maximale Linktiefe

type URLINDEX struct {
	URL   string
	WORDS map[string]int
}
type HTTPRESP struct {
	URL       string
	LINKDEPTH int
	FD        io.Reader
}
type URL struct {
	URL       string
	LINKDEPTH int
}

var crwldurls map[string]bool // globale URL Map - um doppeltes HTTP GET zu vermeiden

//Channels
var chan_urls = make(chan URL, 100000) // buffered channel of strings
//bei aktueller implementierung des Parser aufrufs, müsste der io_reader channel nicht buffered sein
var chan_ioreaders = make(chan HTTPRESP, 10)   // buffered channel of structs
var chan_urlindexes = make(chan URLINDEX, 100) // buffered channel of structs

//Semaphore Channel
var sem_Fetcher = make(chan int, MaxOutstanding_Fetcher)

func debugausgabe(msg string) {
	if DEBUG == 1 {
		fmt.Printf("%s DEBUG: %s\n", time.Now(), msg)
	}
}

func handleFetcher(url URL) {
	<-sem_Fetcher // Eine Ressource verbrauchen: Lock falls bereits alle verbraucht
	fetchURL(url)
	sem_Fetcher <- 1 // Eine Ressource wieder freigeben
}

func starten() {
	// Initiales setzen der Ressourcen
	for i := 0; i < MaxOutstanding_Fetcher; i++ {
		sem_Fetcher <- 1
	}
	// Endless (as long as the channel is not empty) Fetcher Spawning
	for {
		go handleFetcher(<-chan_urls)
	}
}

// parseHTML bekommt eine komplette HTML Seite
// und legt eine Map mit Wörtern und (viele) einzelne Links in entsprechende Channels
func parseHtml(a HTTPRESP) {
	//start := time.Now()
	d := html.NewTokenizer(a.FD)
	var words map[string]int
	words = make(map[string]int)

	for {
		// token type
		tokenType := d.Next()
		// ErrorToken kommt (auch) beim Ende der Daten
		if tokenType == html.ErrorToken {
			chan_urlindexes <- URLINDEX{a.URL, words} // WORD-Map in den Channel legen
			//fmt.Printf("Parse-Dauer : [%.2fs]  URL: %s\n", time.Since(start).Seconds(), a.URL)
			return
		}
		token := d.Token()
		switch tokenType {
		case html.StartTagToken: // <tag>
			// Links finden
			if token.Data == "a" {
				for _, element := range token.Attr {
					if element.Key == "href" {
						// Link normalisieren
						ref_url, err := url.Parse(element.Val)         // geparste URL
						base_url, _ := url.Parse(a.URL)                // Basis URL der geparsten Seite
						comp_url := base_url.ResolveReference(ref_url) // zusammengesetzte url oder falls ref_url==absoluteurl->ref_url
						// Nur Links die nicht in der globalen Link Map sind
						if err == nil && comp_url.Scheme == "http" && crwldurls[comp_url.String()] != true && a.LINKDEPTH < MaxLinkDepth {
							crwldurls[url.URL] = true                            //URL in die globale URL Liste aufnehmen damit sie nicht nochmal in den Work Queue kommt.
							chan_urls <- URL{comp_url.String(), a.LINKDEPTH + 1} // Die URL in den Channel legen und Linktiefe hochzählen
						}
					}
				}
			}

		case html.TextToken: // text between start and end tag
			//Map mit Wörtern erstellen
			temp := strings.Fields(token.Data) //Aufteilen in Einzelne Wörter, trennen bei Whitespace
			for _, element := range temp {
				//TODO: einzelne Örter noch besser von Sonderzeichen trennen z.b. mit TRIM()
				words[element] = words[element] + 1
			}

			//fmt.Printf("%q\n", temp)
		case html.EndTagToken: // </tag>
		case html.SelfClosingTagToken: // <tag/>

		}
	}
}

// fetchURL Bekommt eine URL
// lädt die Seite herunter
// und legt ein struct vom Typ HTTPRESP in den Channel chan_ioreaders
func fetchURL(url URL) {
	start := time.Now()
	response, err := http.Get(url.URL)
	if err != nil {
		fmt.Printf("Fehler: %s beim HTTP GET von: %s\n", err, url.URL)
		return
	}
	//fmt.Printf("%T", response.Body)
	chan_ioreaders <- HTTPRESP{url.URL, url.LINKDEPTH, response.Body}
	fmt.Printf("HTTP GET Dauer: [%.2fs]  URL: %s Tiefe: %d\n", time.Since(start).Seconds(), url.URL, url.LINKDEPTH)
	//TODO io.reader channel umstellen auf direkte übergabe des structs
	go parseHtml(<-chan_ioreaders) // Für jeden Fetcher wird ein Parser aufgerufen, der unabhängig läuft
	return
}

//save holt sich daten (structs vom Typ URLINDEX) aus dem Channel chan_urlindexes
// und schreibt die Daten in eine Datei "output.txt"
func save() {

	//fo, err := os.OpenFile("output.txt", os.O_APPEND, 0777)
	fo, err := os.Create("output.txt")
	if err != nil {
		panic(err)
	}
	// close fo on exit and check for its returned error
	defer func() {
		if err := fo.Close(); err != nil {
			panic(err)
		}
	}()
	// make a write buffer
	w := bufio.NewWriter(fo)
	for {
		data, ok := <-chan_urlindexes

		if ok == false {
			return // Abbruch wenn der Channel geschlossen ist
		}

		fmt.Fprintf(w, "\nURL: %s\nMAP:\n%v\n\n\n\n", data.URL, data.WORDS)
		w.Flush()
	}
}

//writeDB holt sich daten (structs vom Typ URLINDEX) aus dem Channel chan_urlindexes
// und schreibt die Daten in eine sqlite3 Datenbank: crwld.db
func writeDB() {
	os.Remove("./crwld.db")

	db, err := sql.Open("sqlite3", "./crwld.db")
	if err != nil {
		fmt.Println(err)
		return
	}
	defer db.Close()

	//Neue Tabelle(n) erstellen
	sqls := []string{
		"create table data (url text not null, word text, count int)",
		"delete from data",
	}
	for _, sql := range sqls {
		_, err = db.Exec(sql)
		if err != nil {
			fmt.Printf("%q: %s\n", err, sql)
			return
		}
	}

	// Endlosworker
	for {
		index, ok := <-chan_urlindexes //Neue Arbeit aus dem Channel holen

		if ok == false {
			return // Abbruch wenn der Channel geschlossen ist
		}

		tx, err := db.Begin()
		if err != nil {
			fmt.Println(err)
			return
		}

		stmt, err := tx.Prepare("insert into data(url, word, count) values(?, ?, ?)") //Maske für das SQL Statement setzen
		if err != nil {
			fmt.Println(err)
			return
		}
		defer stmt.Close()

		// Die gesamte MAP durchlaufen und für jeden Key ein SQl Statement zusammensetzen
		for element := range index.WORDS {
			_, err = stmt.Exec(index.URL, element, index.WORDS[element])
			if err != nil {
				fmt.Println(err)
				return
			}
		}
		tx.Commit()
	}
}

func main() {
	// "http://www.rsdk.net/test2/" das letzte / ist wichtig für die Auflösung von relativen URLs
	// aber am besten eine "echte" Startseite wie z.B. index.html angeben.
	starturl := URL{"http://www.rsdk.net/test2/index.html", 0} // Start URL mit Linktiefe 0 festlegen
	chan_urls <- starturl                                      // URL in den Channel legen
	crwldurls = make(map[string]bool)
	//go save() // File Writer starten
	debugausgabe("Starte DB Writer")
	go writeDB() //DB Writer starten
	debugausgabe("Starte Crawler")
	starten() // Crawler starten

}
<commit_msg>Crawler beendet sich selbst wenn es keine Schreibarbeit mehr gibt<commit_after>package main

import (
	"bufio"
	"code.google.com/p/go.net/html" //Tokenizer für HTML
	"database/sql"
	"fmt"
	_ "github.com/mattn/go-sqlite3"
	"io"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"
)

const MaxOutstanding_Fetcher = 40 // Maximale Anzahl gleichzeitger Fetcher
const DEBUG = 1                   // Debugausgabe an/aus
const MaxLinkDepth = 2            // Maximale Linktiefe
var stop bool

type URLINDEX struct {
	URL   string
	WORDS map[string]int
}
type HTTPRESP struct {
	URL       string
	LINKDEPTH int
	FD        io.Reader
}
type URL struct {
	URL       string
	LINKDEPTH int
}

var crwldurls map[string]bool // globale URL Map - um doppeltes HTTP GET zu vermeiden

//Channels
var chan_urls = make(chan URL, 100000) // buffered channel of strings
//bei aktueller implementierung des Parser aufrufs, müsste der io_reader channel nicht buffered sein
var chan_ioreaders = make(chan HTTPRESP, 10)   // buffered channel of structs
var chan_urlindexes = make(chan URLINDEX, 100) // buffered channel of structs

//Semaphore Channel
var sem_Fetcher = make(chan int, MaxOutstanding_Fetcher)

func debugausgabe(msg string) {
	if DEBUG == 1 {
		fmt.Printf("%s DEBUG: %s\n", time.Now(), msg)
	}
}

func handleFetcher(url URL) {
	<-sem_Fetcher // Eine Ressource verbrauchen: Lock falls bereits alle verbraucht
	fetchURL(url)
	sem_Fetcher <- 1 // Eine Ressource wieder freigeben
}

func starten() {
	// Initiales setzen der Ressourcen
	for i := 0; i < MaxOutstanding_Fetcher; i++ {
		sem_Fetcher <- 1
	}
	waittime, _ := time.ParseDuration("300ms")
	// Endless (as long as the channel is not empty) Fetcher Spawning
	for {
		select {
		case element := <-chan_urls: // Neue Arbeit aus dem Channel holen
			go handleFetcher(element)
		default:
			//keine arbeit da
			if stop {
				return
			}
			time.Sleep(waittime)

		}
	}
}

// parseHTML bekommt eine komplette HTML Seite
// und legt eine Map mit Wörtern und (viele) einzelne Links in entsprechende Channels
func parseHtml(a HTTPRESP) {
	//start := time.Now()
	d := html.NewTokenizer(a.FD)
	var words map[string]int
	words = make(map[string]int)

	for {
		// token type
		tokenType := d.Next()
		// ErrorToken kommt (auch) beim Ende der Daten
		if tokenType == html.ErrorToken {
			chan_urlindexes <- URLINDEX{a.URL, words} // WORD-Map in den Channel legen
			//fmt.Printf("Parse-Dauer : [%.2fs]  URL: %s\n", time.Since(start).Seconds(), a.URL)
			return
		}
		token := d.Token()
		switch tokenType {
		case html.StartTagToken: // <tag>
			// Links finden
			if token.Data == "a" {
				for _, element := range token.Attr {
					if element.Key == "href" {
						// Link normalisieren
						ref_url, err := url.Parse(element.Val)         // geparste URL
						base_url, _ := url.Parse(a.URL)                // Basis URL der geparsten Seite
						comp_url := base_url.ResolveReference(ref_url) // zusammengesetzte url oder falls ref_url==absoluteurl->ref_url
						// Nur Links die nicht in der globalen Link Map sind
						if err == nil && comp_url.Scheme == "http" && crwldurls[comp_url.String()] != true && a.LINKDEPTH < MaxLinkDepth {
							crwldurls[comp_url.String()] = true                  //URL in die globale URL Liste aufnehmen damit sie nicht nochmal in den Work Queue kommt.
							chan_urls <- URL{comp_url.String(), a.LINKDEPTH + 1} // Die URL in den Channel legen und Linktiefe hochzählen
						}
					}
				}
			}

		case html.TextToken: // text between start and end tag
			//Map mit Wörtern erstellen
			temp := strings.Fields(token.Data) //Aufteilen in Einzelne Wörter, trennen bei Whitespace
			for _, element := range temp {
				//TODO: einzelne Örter noch besser von Sonderzeichen trennen z.b. mit TRIM()
				words[element] = words[element] + 1
			}

			//fmt.Printf("%q\n", temp)
		case html.EndTagToken: // </tag>
		case html.SelfClosingTagToken: // <tag/>

		}
	}
}

// fetchURL Bekommt eine URL
// lädt die Seite herunter
// und legt ein struct vom Typ HTTPRESP in den Channel chan_ioreaders
func fetchURL(url URL) {
	start := time.Now()
	response, err := http.Get(url.URL)
	if err != nil {
		fmt.Printf("Fehler: %s beim HTTP GET von: %s\n", err, url.URL)
		return
	}
	//fmt.Printf("%T", response.Body)
	chan_ioreaders <- HTTPRESP{url.URL, url.LINKDEPTH, response.Body}
	fmt.Printf("HTTP GET Dauer: [%.2fs]  URL: %s Tiefe: %d\n", time.Since(start).Seconds(), url.URL, url.LINKDEPTH)
	//TODO io.reader channel umstellen auf direkte übergabe des structs
	go parseHtml(<-chan_ioreaders) // Für jeden Fetcher wird ein Parser aufgerufen, der unabhängig läuft
	return
}

//save holt sich daten (structs vom Typ URLINDEX) aus dem Channel chan_urlindexes
// und schreibt die Daten in eine Datei "output.txt"
func save() {

	//fo, err := os.OpenFile("output.txt", os.O_APPEND, 0777)
	fo, err := os.Create("output.txt")
	if err != nil {
		panic(err)
	}
	// close fo on exit and check for its returned error
	defer func() {
		if err := fo.Close(); err != nil {
			panic(err)
		}
	}()
	// make a write buffer
	w := bufio.NewWriter(fo)
	for {
		data, ok := <-chan_urlindexes

		if ok == false {
			return // Abbruch wenn der Channel geschlossen ist
		}

		fmt.Fprintf(w, "\nURL: %s\nMAP:\n%v\n\n\n\n", data.URL, data.WORDS)
		w.Flush()
	}
}

//writeDB holt sich daten (structs vom Typ URLINDEX) aus dem Channel chan_urlindexes
// und schreibt die Daten in eine sqlite3 Datenbank: crwld.db
func writeDB() {
	os.Remove("./crwld.db")

	db, err := sql.Open("sqlite3", "./crwld.db")
	if err != nil {
		fmt.Println(err)
		return
	}
	defer db.Close()

	//Neue Tabelle(n) erstellen
	sqls := []string{
		"create table data (url text not null, word text, count int)",
		"delete from data",
	}
	for _, sql := range sqls {
		_, err = db.Exec(sql)
		if err != nil {
			fmt.Printf("%q: %s\n", err, sql)
			return
		}
	}
	start := time.Now() // Initialiseren
	waittime, _ := time.ParseDuration("300ms")
	// Endlosworker
	for {
		var index URLINDEX
		select {
		case index = <-chan_urlindexes: // Neue Arbeit aus dem Channel holen
			start = time.Now() // zu diesem Zeitpunkt gab es das letzte mal Arbeit
		default:
			//keine arbeit da
			if time.Since(start).Seconds() > 3 {
				stop = true // wenn länger als 3 Sekunden auf neue Schreibarbeit gewartet wurde -->beenden
			}
			time.Sleep(waittime) //kurz abwarten
			continue
		}

		tx, err := db.Begin()
		if err != nil {
			fmt.Println(err)
			return
		}

		stmt, err := tx.Prepare("insert into data(url, word, count) values(?, ?, ?)") //Maske für das SQL Statement setzen
		if err != nil {
			fmt.Println(err)
			return
		}
		defer stmt.Close()

		// Die gesamte MAP durchlaufen und für jeden Key ein SQl Statement zusammensetzen
		for element := range index.WORDS {
			_, err = stmt.Exec(index.URL, element, index.WORDS[element])
			if err != nil {
				fmt.Println(err)
				return
			}
		}
		tx.Commit()
	}
}

func main() {
	stop = false
	// "http://www.rsdk.net/test2/" das letzte / ist wichtig für die Auflösung von relativen URLs
	// aber am besten eine "echte" Startseite wie z.B. index.html angeben.
	starturl := URL{"http://www.rsdk.net/test2/index.html", 0} // Start URL mit Linktiefe 0 festlegen
	chan_urls <- starturl                                      // URL in den Channel legen
	crwldurls = make(map[string]bool)
	//go save() // File Writer starten
	debugausgabe("Starte DB Writer")
	go writeDB() //DB Writer starten
	debugausgabe("Starte Crawler")
	starten() // Crawler starten

}
<|endoftext|>


<commit_before>package main

/*
#cgo LDFLAGS: -lX11
#include <X11/Xlib.h>
*/
import "C"

import (
	"fmt"
	"gopkg.in/yaml.v2"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"time"

	"github.com/gotk3/gotk3/gtk"
)

type gobarConfig struct {
	Listen   string `yaml:"listen"`
	CssPath  string `yaml:"css_path"`
	Position struct {
		X int `yaml:"x"`
		Y int `yaml:"y"`
	} `yaml:"position"`
	BarSize struct {
		X int `yaml:"x"`
		Y int `yaml:"y"`
	} `yaml:"bar_size"`
	Actions map[string]struct {
		Command  string  `yaml:"command"`
		Value    string  `yaml:"value"`
		Label    string  `yaml:"label"`
		Duration string  `yaml:"duration"`
		Min      float64 `yaml:"min"`
		Max      float64 `yaml:"max"`
	} `yaml:"actions"`
}

func getLabelHandler(label *gtk.Label, bar *gtk.LevelBar, win *gtk.Window) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		var err error
		min := 0.0
		max := 100.0
		level := 50.0
		duration, _ := time.ParseDuration("700ms")
		labelText := "label string xD"

		vars := r.URL.Query()
		r.Body.Close()

		if val, ok := vars["label"]; ok {
			labelText = val[0]
		}
		if val, ok := vars["min"]; ok {
			min, err = strconv.ParseFloat(val[0], 64)
			if err != nil {
				log.Println("Got a wrong min value", val, err)
			}
		}
		if val, ok := vars["max"]; ok {
			max, err = strconv.ParseFloat(val[0], 64)
			if err != nil {
				log.Println("Got a wrong max value", val, err)
			}
		}
		if val, ok := vars["level"]; ok {
			level, err = strconv.ParseFloat(val[0], 64)
			if err != nil {
				log.Println("Got a wrong level value", val, err)
			}
		}
		if val, ok := vars["duration"]; ok {
			duration, err = time.ParseDuration(val[0])
			if err != nil {
				log.Println("Got a wrong duration value", val, err)
			}
		}

		_, _ = w.Write([]byte("OK\n"))

		go func() {
			bar.SetMinValue(min)
			bar.SetMaxValue(max)
			bar.SetValue(level)
			label.SetLabel(labelText)

			win.ShowAll()

			time.Sleep(duration)

			win.Resize(10, 10)
			win.Hide()
		}()
	}
}

func main() {
	var config gobarConfig

	if len(os.Args) != 2 {
		log.Fatalln("Usage:", os.Args[0], "<configuration file>")
	}

	data, err := ioutil.ReadFile(os.Args[1])
	if err != nil {
		log.Fatalln("Error reading configuration file:", err)
	}

	if err := yaml.Unmarshal(data, &config); err != nil {
		log.Fatalln("Error parsing configuration file:", err)
	}

	C.XInitThreads()
	gtk.Init(nil)

	win, err := gtk.WindowNew(gtk.WINDOW_TOPLEVEL)
	if err != nil {
		log.Fatal("Unable to create window:", err)
	}
	win.Connect("destroy", func() {
		gtk.MainQuit()
	})

	grid, err := gtk.GridNew()
	if err != nil {
		log.Fatal("Unable to create grid:", err)
	}

	css, err := gtk.CssProviderNew()
	if err != nil {
		log.Fatal("Unable to create css provider:", err)
	}

	lb := label()
	bar := levelBar(config.BarSize.X, config.BarSize.Y)

	grid.SetOrientation(gtk.ORIENTATION_HORIZONTAL)
	grid.Add(bar)
	grid.Add(lb)

	css.LoadFromPath(config.CssPath)

	style_context, err := lb.GetStyleContext()
	if err != nil {
		log.Fatal("Unable to get label style context:", err)
	}
	style_context.AddProvider(css, gtk.STYLE_PROVIDER_PRIORITY_USER)

	win.SetWMClass("gobar", "gobar")
	win.SetTitle("gobar")
	win.Add(grid)
	win.SetAcceptFocus(false)
	win.Move(config.Position.X, config.Position.Y)

	http.HandleFunc("/api/v1/bar", getLabelHandler(lb, bar, win))

	for key, value := range config.Actions {
		key := key
		value := value
		http.HandleFunc("/api/v1/action/"+key, func(w http.ResponseWriter, r *http.Request) {
			r.Body.Close()
			err := exec.Command("sh", "-c", value.Command).Run()
			if err != nil {
				log.Println("Command:", value.Command, "error:", err)
				fmt.Fprintln(w, "error:", err)
				return
			}
			out, err := exec.Command("sh", "-c", value.Value).Output()
			if err != nil {
				log.Println("Command:", value.Command, "error:", err)
				fmt.Fprintln(w, "error:", err)
				return
			}
			val, err := strconv.ParseFloat(strings.TrimSpace(string(out)), 64)
			if err != nil {
				log.Println("Error parsing as float", string(out), "error:", err)
				fmt.Fprintln(w, "error:", err)
				return
			}
			duration, err := time.ParseDuration(value.Duration)
			if err != nil {
				log.Println("Error paring as duration", duration, "error:", err)
				fmt.Fprintln(w, "error:", err)
			}

			fmt.Fprintln(w, "status: OK\nvalue:", val)

			go func() {
				lb.SetLabel(value.Label)

				bar.SetMinValue(value.Min)
				bar.SetMaxValue(value.Max)
				bar.SetValue(val)

				win.ShowAll()

				time.Sleep(duration)

				win.Resize(10, 10)
				win.Hide()
			}()
		})
	}

	go gtk.Main()

	http.ListenAndServe(config.Listen, nil)
}

func label() *gtk.Label {
	label, err := gtk.LabelNew("xD")

	if err != nil {
		log.Fatal("Unable to create label:", err)
	}

	return label
}

func levelBar(x, y int) *gtk.LevelBar {
	lb, err := gtk.LevelBarNew()

	if err != nil {
		log.Fatal("Unable to create level bar:", err)
	}

	lb.SetSizeRequest(x, y)

	return lb
}
<commit_msg>welp… C works.<commit_after>package main

/*
#cgo LDFLAGS: -lX11
#cgo pkg-config: gtk+-3.0
#include <X11/Xlib.h>
#include <gtk/gtk.h>

void setup_css(char *path) {
  GtkCssProvider *provider;
  GdkDisplay *display;
  GdkScreen *screen;
  provider = gtk_css_provider_new();
  display = gdk_display_get_default();
  screen = gdk_display_get_default_screen(display);
  gtk_style_context_add_provider_for_screen(screen,
      GTK_STYLE_PROVIDER(provider),
      GTK_STYLE_PROVIDER_PRIORITY_APPLICATION);
  gtk_css_provider_load_from_path(GTK_CSS_PROVIDER(provider), path, NULL);
  g_object_unref(provider);
}
*/
import "C"

import (
	"fmt"
	"gopkg.in/yaml.v2"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"time"

	"github.com/gotk3/gotk3/gtk"
)

type gobarConfig struct {
	Listen   string `yaml:"listen"`
	CssPath  string `yaml:"css_path"`
	Position struct {
		X int `yaml:"x"`
		Y int `yaml:"y"`
	} `yaml:"position"`
	BarSize struct {
		X int `yaml:"x"`
		Y int `yaml:"y"`
	} `yaml:"bar_size"`
	Actions map[string]struct {
		Command  string  `yaml:"command"`
		Value    string  `yaml:"value"`
		Label    string  `yaml:"label"`
		Duration string  `yaml:"duration"`
		Min      float64 `yaml:"min"`
		Max      float64 `yaml:"max"`
	} `yaml:"actions"`
}

func getLabelHandler(label *gtk.Label, bar *gtk.LevelBar, win *gtk.Window) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		var err error
		min := 0.0
		max := 100.0
		level := 50.0
		duration, _ := time.ParseDuration("700ms")
		labelText := "label string xD"

		vars := r.URL.Query()
		r.Body.Close()

		if val, ok := vars["label"]; ok {
			labelText = val[0]
		}
		if val, ok := vars["min"]; ok {
			min, err = strconv.ParseFloat(val[0], 64)
			if err != nil {
				log.Println("Got a wrong min value", val, err)
			}
		}
		if val, ok := vars["max"]; ok {
			max, err = strconv.ParseFloat(val[0], 64)
			if err != nil {
				log.Println("Got a wrong max value", val, err)
			}
		}
		if val, ok := vars["level"]; ok {
			level, err = strconv.ParseFloat(val[0], 64)
			if err != nil {
				log.Println("Got a wrong level value", val, err)
			}
		}
		if val, ok := vars["duration"]; ok {
			duration, err = time.ParseDuration(val[0])
			if err != nil {
				log.Println("Got a wrong duration value", val, err)
			}
		}

		_, _ = w.Write([]byte("OK\n"))

		go func() {
			bar.SetMinValue(min)
			bar.SetMaxValue(max)
			bar.SetValue(level)
			label.SetLabel(labelText)

			win.ShowAll()

			time.Sleep(duration)

			win.Resize(10, 10)
			win.Hide()
		}()
	}
}

func main() {
	var config gobarConfig

	if len(os.Args) != 2 {
		log.Fatalln("Usage:", os.Args[0], "<configuration file>")
	}

	data, err := ioutil.ReadFile(os.Args[1])
	if err != nil {
		log.Fatalln("Error reading configuration file:", err)
	}

	if err := yaml.Unmarshal(data, &config); err != nil {
		log.Fatalln("Error parsing configuration file:", err)
	}

	C.XInitThreads()
	gtk.Init(nil)

	C.setup_css(C.CString(config.CssPath))

	win, err := gtk.WindowNew(gtk.WINDOW_TOPLEVEL)
	if err != nil {
		log.Fatal("Unable to create window:", err)
	}
	win.Connect("destroy", func() {
		gtk.MainQuit()
	})

	grid, err := gtk.GridNew()
	if err != nil {
		log.Fatal("Unable to create grid:", err)
	}

	lb := label()
	bar := levelBar(config.BarSize.X, config.BarSize.Y)

	grid.SetOrientation(gtk.ORIENTATION_HORIZONTAL)
	grid.Add(bar)
	grid.Add(lb)

	win.SetWMClass("gobar", "gobar")
	win.SetTitle("gobar")
	win.Add(grid)
	win.SetAcceptFocus(false)
	win.Move(config.Position.X, config.Position.Y)

	http.HandleFunc("/api/v1/bar", getLabelHandler(lb, bar, win))

	for key, value := range config.Actions {
		key := key
		value := value
		http.HandleFunc("/api/v1/action/"+key, func(w http.ResponseWriter, r *http.Request) {
			r.Body.Close()
			err := exec.Command("sh", "-c", value.Command).Run()
			if err != nil {
				log.Println("Command:", value.Command, "error:", err)
				fmt.Fprintln(w, "error:", err)
				return
			}
			out, err := exec.Command("sh", "-c", value.Value).Output()
			if err != nil {
				log.Println("Command:", value.Command, "error:", err)
				fmt.Fprintln(w, "error:", err)
				return
			}
			val, err := strconv.ParseFloat(strings.TrimSpace(string(out)), 64)
			if err != nil {
				log.Println("Error parsing as float", string(out), "error:", err)
				fmt.Fprintln(w, "error:", err)
				return
			}
			duration, err := time.ParseDuration(value.Duration)
			if err != nil {
				log.Println("Error paring as duration", duration, "error:", err)
				fmt.Fprintln(w, "error:", err)
			}

			fmt.Fprintln(w, "status: OK\nvalue:", val)

			go func() {
				lb.SetLabel(value.Label)

				bar.SetMinValue(value.Min)
				bar.SetMaxValue(value.Max)
				bar.SetValue(val)

				win.ShowAll()

				time.Sleep(duration)

				win.Resize(10, 10)
				win.Hide()
			}()
		})
	}

	go gtk.Main()

	http.ListenAndServe(config.Listen, nil)
}

func label() *gtk.Label {
	label, err := gtk.LabelNew("xD")

	if err != nil {
		log.Fatal("Unable to create label:", err)
	}

	return label
}

func levelBar(x, y int) *gtk.LevelBar {
	lb, err := gtk.LevelBarNew()

	if err != nil {
		log.Fatal("Unable to create level bar:", err)
	}

	lb.SetSizeRequest(x, y)

	return lb
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/musl/libgofr"
	"github.com/nfnt/resize"
	"image"
	"image/png"
	"log"
	"net/http"
	"runtime"
	"strconv"
	"time"
)

var id_chan = make(chan int, 1)

func finish(w http.ResponseWriter, status int, message string) {
	w.WriteHeader(status)
	fmt.Fprintf(w, message)
}

func logDuration(message string, start time.Time) {
	end := time.Now()
	log.Printf("%s: %v\n", message, end.Sub(start))
}

func route_png(w http.ResponseWriter, r *http.Request) {
	id := <-id_chan
	start := time.Now()
	defer logDuration(fmt.Sprintf("%08d %s", id, r.URL.Path), start)

	if r.Method != "GET" {
		finish(w, http.StatusMethodNotAllowed, "Method not allowed.")
		return
	}

	q := r.URL.Query()

	s, err := strconv.Atoi(q.Get("s"))
	if err != nil {
		s = 1
	}

	width, err := strconv.Atoi(q.Get("w"))
	if err != nil {
		finish(w, http.StatusUnprocessableEntity, "Invalid width")
		return
	}

	height, err := strconv.Atoi(q.Get("h"))
	if err != nil {
		finish(w, http.StatusUnprocessableEntity, "Invalid height")
		return
	}

	iterations, err := strconv.Atoi(q.Get("i"))
	if err != nil {
		finish(w, http.StatusUnprocessableEntity, "Invalid i")
		return
	}

	er, err := strconv.ParseFloat(q.Get("e"), 64)
	if err != nil {
		finish(w, http.StatusUnprocessableEntity, "Invalid e")
		return
	}

	rmin, err := strconv.ParseFloat(q.Get("rmin"), 64)
	if err != nil {
		finish(w, http.StatusUnprocessableEntity, "Invalid rmin")
		return
	}

	imin, err := strconv.ParseFloat(q.Get("imin"), 64)
	if err != nil {
		finish(w, http.StatusUnprocessableEntity, "Invalid rmin")
		return
	}

	rmax, err := strconv.ParseFloat(q.Get("rmax"), 64)
	if err != nil {
		finish(w, http.StatusUnprocessableEntity, "Invalid rmax")
		return
	}

	imax, err := strconv.ParseFloat(q.Get("imax"), 64)
	if err != nil {
		finish(w, http.StatusUnprocessableEntity, "Invalid rmin")
		return
	}

	c := q.Get("c")
	hex := q.Get("m")

	p := gofr.Parameters{
		ImageWidth:   width * s,
		ImageHeight:  height * s,
		MaxI:         iterations,
		EscapeRadius: er,
		Min:          complex(rmin, imin),
		Max:          complex(rmax, imax),
		ColorFunc:    c,
		MemberColor:  hex,
	}

	log.Printf("%08d rendering\n", id)

	// TODO: Check parameters and set reasonable bounds on what we can
	// quickly calculate.

	img := image.NewNRGBA64(image.Rect(0, 0, p.ImageWidth, p.ImageHeight))
	n := runtime.NumCPU()
	contexts := gofr.MakeContexts(img, n, &p)
	gofr.Render(n, contexts, gofr.Mandelbrot)

	scaled_img := resize.Resize(uint(width), uint(height), image.Image(img), resize.Lanczos3)

	w.Header().Set("Content-Type", "image/png")
	png.Encode(w, scaled_img)
}

func main() {
	fs := http.FileServer(http.Dir("static"))
	bind_addr := "0.0.0.0:8000"

	go func() {
		for i := 0; ; i++ {
			id_chan <- i
		}
	}()

	http.Handle("/", fs)
	http.HandleFunc("/png", route_png)
	log.Printf("Listening on: %s\n", bind_addr)
	log.Fatal(http.ListenAndServe(bind_addr, nil))
}
<commit_msg>Add version constant<commit_after>package main

import (
	"fmt"
	"github.com/musl/libgofr"
	"github.com/nfnt/resize"
	"image"
	"image/png"
	"log"
	"net/http"
	"runtime"
	"strconv"
	"time"
)

const Version = "0.0.2"

var id_chan = make(chan int, 1)

func finish(w http.ResponseWriter, status int, message string) {
	w.WriteHeader(status)
	fmt.Fprintf(w, message)
}

func logDuration(message string, start time.Time) {
	end := time.Now()
	log.Printf("%s: %v\n", message, end.Sub(start))
}

func route_png(w http.ResponseWriter, r *http.Request) {
	id := <-id_chan
	start := time.Now()
	defer logDuration(fmt.Sprintf("%08d %s", id, r.URL.Path), start)

	if r.Method != "GET" {
		finish(w, http.StatusMethodNotAllowed, "Method not allowed.")
		return
	}

	q := r.URL.Query()

	s, err := strconv.Atoi(q.Get("s"))
	if err != nil {
		s = 1
	}

	width, err := strconv.Atoi(q.Get("w"))
	if err != nil {
		finish(w, http.StatusUnprocessableEntity, "Invalid width")
		return
	}

	height, err := strconv.Atoi(q.Get("h"))
	if err != nil {
		finish(w, http.StatusUnprocessableEntity, "Invalid height")
		return
	}

	iterations, err := strconv.Atoi(q.Get("i"))
	if err != nil {
		finish(w, http.StatusUnprocessableEntity, "Invalid i")
		return
	}

	er, err := strconv.ParseFloat(q.Get("e"), 64)
	if err != nil {
		finish(w, http.StatusUnprocessableEntity, "Invalid e")
		return
	}

	rmin, err := strconv.ParseFloat(q.Get("rmin"), 64)
	if err != nil {
		finish(w, http.StatusUnprocessableEntity, "Invalid rmin")
		return
	}

	imin, err := strconv.ParseFloat(q.Get("imin"), 64)
	if err != nil {
		finish(w, http.StatusUnprocessableEntity, "Invalid rmin")
		return
	}

	rmax, err := strconv.ParseFloat(q.Get("rmax"), 64)
	if err != nil {
		finish(w, http.StatusUnprocessableEntity, "Invalid rmax")
		return
	}

	imax, err := strconv.ParseFloat(q.Get("imax"), 64)
	if err != nil {
		finish(w, http.StatusUnprocessableEntity, "Invalid rmin")
		return
	}

	c := q.Get("c")
	hex := q.Get("m")

	p := gofr.Parameters{
		ImageWidth:   width * s,
		ImageHeight:  height * s,
		MaxI:         iterations,
		EscapeRadius: er,
		Min:          complex(rmin, imin),
		Max:          complex(rmax, imax),
		ColorFunc:    c,
		MemberColor:  hex,
	}

	log.Printf("%08d rendering\n", id)

	// TODO: Check parameters and set reasonable bounds on what we can
	// quickly calculate.

	img := image.NewNRGBA64(image.Rect(0, 0, p.ImageWidth, p.ImageHeight))
	n := runtime.NumCPU()
	contexts := gofr.MakeContexts(img, n, &p)
	gofr.Render(n, contexts, gofr.Mandelbrot)

	scaled_img := resize.Resize(uint(width), uint(height), image.Image(img), resize.Lanczos3)

	w.Header().Set("Content-Type", "image/png")
	png.Encode(w, scaled_img)
}

func main() {
	fs := http.FileServer(http.Dir("static"))
	bind_addr := "0.0.0.0:8000"

	go func() {
		for i := 0; ; i++ {
			id_chan <- i
		}
	}()

	http.Handle("/", fs)
	http.HandleFunc("/png", route_png)
	log.Printf("Listening on: %s\n", bind_addr)
	log.Fatal(http.ListenAndServe(bind_addr, nil))
}
<|endoftext|>


<commit_before>package goreq

import (
	"bufio"
	"bytes"
	"compress/gzip"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"reflect"
	"strings"
	"time"
)

type Request struct {
	headers      []headerTuple
	Method       string
	Uri          string
	Body         interface{}
	QueryString  interface{}
	Timeout      time.Duration
	ContentType  string
	Accept       string
	Host         string
	UserAgent    string
	Insecure     bool
	MaxRedirects int
	Compressed   bool
}

type Response struct {
	StatusCode    int
	ContentLength int64
	Body          Body
	Header        http.Header
}

type headerTuple struct {
	name  string
	value string
}

type Body struct {
	io.ReadCloser
}

type Error struct {
	timeout bool
	Err     error
}

func (e *Error) Timeout() bool {
	return e.timeout
}

func (e *Error) Error() string {
	return e.Err.Error()
}

func (b *Body) FromJsonTo(o interface{}) error {
	if body, err := ioutil.ReadAll(b); err != nil {
		return err
	} else if err := json.Unmarshal(body, o); err != nil {
		return err
	}

	return nil
}

func (b *Body) ToString() (string, error) {
	body, err := ioutil.ReadAll(b)
	if err != nil {
		return "", err
	}
	return string(body), nil
}

func paramParse(query interface{}) (string, error) {
	var (
		v = &url.Values{}
		s = reflect.ValueOf(query)
		t = reflect.TypeOf(query)
	)

	switch query.(type) {
	case url.Values:
		return query.(url.Values).Encode(), nil
	default:
		for i := 0; i < s.NumField(); i++ {
			v.Add(strings.ToLower(t.Field(i).Name), fmt.Sprintf("%v", s.Field(i).Interface()))
		}
		return v.Encode(), nil
	}
}

func prepareRequestBody(b interface{}) (io.Reader, error) {
	switch b.(type) {
	case string:
		// treat is as text
		return strings.NewReader(b.(string)), nil
	case io.Reader:
		// treat is as text
		return b.(io.Reader), nil
	case []byte:
		//treat as byte array
		return bytes.NewReader(b.([]byte)), nil
	case nil:
		return nil, nil
	default:
		// try to jsonify it
		j, err := json.Marshal(b)
		if err == nil {
			return bytes.NewReader(j), nil
		}
		return nil, err
	}
}

func newResponse(res *http.Response) *Response {
	return &Response{StatusCode: res.StatusCode, ContentLength: res.ContentLength, Header: res.Header, Body: Body{res.Body}}
}

var defaultDialer = &net.Dialer{Timeout: 1000 * time.Millisecond}
var defaultTransport = &http.Transport{Dial: defaultDialer.Dial}
var defaultClient = &http.Client{Transport: defaultTransport}

func SetConnectTimeout(duration time.Duration) {
	defaultDialer.Timeout = duration
}

func (r *Request) AddHeader(name string, value string) {
	if r.headers == nil {
		r.headers = []headerTuple{}
	}
	r.headers = append(r.headers, headerTuple{name: name, value: value})
}

func (r Request) Do() (*Response, error) {
	var req *http.Request
	var er error

	if r.Insecure {
		defaultTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
	} else if defaultTransport.TLSClientConfig != nil {
		// the default TLS client (when transport.TLSClientConfig==nil) is
		// already set to verify, so do nothing in that case
		defaultTransport.TLSClientConfig.InsecureSkipVerify = false
	}

	b, e := prepareRequestBody(r.Body)
	if e != nil {
		// there was a problem marshaling the body
		return nil, &Error{Err: e}
	}

	if r.QueryString != nil {
		param, e := paramParse(r.QueryString)
		if e != nil {
			return nil, &Error{Err: e}
		}
		r.Uri = r.Uri + "?" + param
	}

	if r.Compressed && b != nil {
		gzipBuffer := bytes.NewBuffer([]byte{})
		readBuffer := bufio.NewReader(b)
		gzipWriter := gzip.NewWriter(gzipBuffer)
		_, e = readBuffer.WriteTo(gzipWriter)
		gzipWriter.Close()
		if e != nil {
			fmt.Println("error: ", e)
			return nil, &Error{Err: e}
		}
		req, er = http.NewRequest(r.Method, r.Uri, gzipBuffer)
	} else {
		req, er = http.NewRequest(r.Method, r.Uri, b)
	}

	if er != nil {
		// we couldn't parse the URL.
		return nil, &Error{Err: er}
	}

	// add headers to the request
	req.Host = r.Host
	req.Header.Add("User-Agent", r.UserAgent)
	req.Header.Add("Content-Type", r.ContentType)
	req.Header.Add("Accept", r.Accept)
	if r.Compressed {
		req.Header.Add("Content-Encoding", "gzip")
		req.Header.Add("Accept-Encoding", "gzip")
	}
	if r.headers != nil {
		for _, header := range r.headers {
			req.Header.Add(header.name, header.value)
		}
	}

	timeout := false
	var timer *time.Timer
	if r.Timeout > 0 {
		timer = time.AfterFunc(r.Timeout, func() {
			defaultTransport.CancelRequest(req)
			timeout = true
		})
	}

	res, err := defaultClient.Do(req)
	if timer != nil {
		timer.Stop()
	}

	if err != nil {
		if op, ok := err.(*net.OpError); !timeout && ok {
			timeout = op.Timeout()
		}
		return nil, &Error{timeout: timeout, Err: err}
	}

	if isRedirect(res.StatusCode) && r.MaxRedirects > 0 {
		loc, _ := res.Location()
		r.MaxRedirects--
		r.Uri = loc.String()
		return r.Do()
	}

	if r.Compressed && strings.Contains(res.Header.Get("Content-Encoding"), "gzip") {
		gzipReader, err := gzip.NewReader(res.Body)
		defer res.Body.Close()
		if err != nil {
			return nil, &Error{Err: err}
		}
		return &Response{StatusCode: res.StatusCode, ContentLength: res.ContentLength, Header: res.Header, Body: Body{gzipReader}}, nil
	} else {
		return &Response{StatusCode: res.StatusCode, ContentLength: res.ContentLength, Header: res.Header, Body: Body{res.Body}}, nil
	}
	return newResponse(res), nil
}

func isRedirect(status int) bool {
	switch status {
	case http.StatusMovedPermanently:
		return true
	case http.StatusFound:
		return true
	case http.StatusSeeOther:
		return true
	case http.StatusTemporaryRedirect:
		return true
	default:
		return false
	}
}
<commit_msg>remove useless newResponse call and method<commit_after>package goreq

import (
	"bufio"
	"bytes"
	"compress/gzip"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"reflect"
	"strings"
	"time"
)

type Request struct {
	headers      []headerTuple
	Method       string
	Uri          string
	Body         interface{}
	QueryString  interface{}
	Timeout      time.Duration
	ContentType  string
	Accept       string
	Host         string
	UserAgent    string
	Insecure     bool
	MaxRedirects int
	Compressed   bool
}

type Response struct {
	StatusCode    int
	ContentLength int64
	Body          Body
	Header        http.Header
}

type headerTuple struct {
	name  string
	value string
}

type Body struct {
	io.ReadCloser
}

type Error struct {
	timeout bool
	Err     error
}

func (e *Error) Timeout() bool {
	return e.timeout
}

func (e *Error) Error() string {
	return e.Err.Error()
}

func (b *Body) FromJsonTo(o interface{}) error {
	if body, err := ioutil.ReadAll(b); err != nil {
		return err
	} else if err := json.Unmarshal(body, o); err != nil {
		return err
	}

	return nil
}

func (b *Body) ToString() (string, error) {
	body, err := ioutil.ReadAll(b)
	if err != nil {
		return "", err
	}
	return string(body), nil
}

func paramParse(query interface{}) (string, error) {
	var (
		v = &url.Values{}
		s = reflect.ValueOf(query)
		t = reflect.TypeOf(query)
	)

	switch query.(type) {
	case url.Values:
		return query.(url.Values).Encode(), nil
	default:
		for i := 0; i < s.NumField(); i++ {
			v.Add(strings.ToLower(t.Field(i).Name), fmt.Sprintf("%v", s.Field(i).Interface()))
		}
		return v.Encode(), nil
	}
}

func prepareRequestBody(b interface{}) (io.Reader, error) {
	switch b.(type) {
	case string:
		// treat is as text
		return strings.NewReader(b.(string)), nil
	case io.Reader:
		// treat is as text
		return b.(io.Reader), nil
	case []byte:
		//treat as byte array
		return bytes.NewReader(b.([]byte)), nil
	case nil:
		return nil, nil
	default:
		// try to jsonify it
		j, err := json.Marshal(b)
		if err == nil {
			return bytes.NewReader(j), nil
		}
		return nil, err
	}
}

var defaultDialer = &net.Dialer{Timeout: 1000 * time.Millisecond}
var defaultTransport = &http.Transport{Dial: defaultDialer.Dial}
var defaultClient = &http.Client{Transport: defaultTransport}

func SetConnectTimeout(duration time.Duration) {
	defaultDialer.Timeout = duration
}

func (r *Request) AddHeader(name string, value string) {
	if r.headers == nil {
		r.headers = []headerTuple{}
	}
	r.headers = append(r.headers, headerTuple{name: name, value: value})
}

func (r Request) Do() (*Response, error) {
	var req *http.Request
	var er error

	if r.Insecure {
		defaultTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
	} else if defaultTransport.TLSClientConfig != nil {
		// the default TLS client (when transport.TLSClientConfig==nil) is
		// already set to verify, so do nothing in that case
		defaultTransport.TLSClientConfig.InsecureSkipVerify = false
	}

	b, e := prepareRequestBody(r.Body)
	if e != nil {
		// there was a problem marshaling the body
		return nil, &Error{Err: e}
	}

	if r.QueryString != nil {
		param, e := paramParse(r.QueryString)
		if e != nil {
			return nil, &Error{Err: e}
		}
		r.Uri = r.Uri + "?" + param
	}

	if r.Compressed && b != nil {
		gzipBuffer := bytes.NewBuffer([]byte{})
		readBuffer := bufio.NewReader(b)
		gzipWriter := gzip.NewWriter(gzipBuffer)
		_, e = readBuffer.WriteTo(gzipWriter)
		gzipWriter.Close()
		if e != nil {
			fmt.Println("error: ", e)
			return nil, &Error{Err: e}
		}
		req, er = http.NewRequest(r.Method, r.Uri, gzipBuffer)
	} else {
		req, er = http.NewRequest(r.Method, r.Uri, b)
	}

	if er != nil {
		// we couldn't parse the URL.
		return nil, &Error{Err: er}
	}

	// add headers to the request
	req.Host = r.Host
	req.Header.Add("User-Agent", r.UserAgent)
	req.Header.Add("Content-Type", r.ContentType)
	req.Header.Add("Accept", r.Accept)
	if r.Compressed {
		req.Header.Add("Content-Encoding", "gzip")
		req.Header.Add("Accept-Encoding", "gzip")
	}
	if r.headers != nil {
		for _, header := range r.headers {
			req.Header.Add(header.name, header.value)
		}
	}

	timeout := false
	var timer *time.Timer
	if r.Timeout > 0 {
		timer = time.AfterFunc(r.Timeout, func() {
			defaultTransport.CancelRequest(req)
			timeout = true
		})
	}

	res, err := defaultClient.Do(req)
	if timer != nil {
		timer.Stop()
	}

	if err != nil {
		if op, ok := err.(*net.OpError); !timeout && ok {
			timeout = op.Timeout()
		}
		return nil, &Error{timeout: timeout, Err: err}
	}

	if isRedirect(res.StatusCode) && r.MaxRedirects > 0 {
		loc, _ := res.Location()
		r.MaxRedirects--
		r.Uri = loc.String()
		return r.Do()
	}

	if r.Compressed && strings.Contains(res.Header.Get("Content-Encoding"), "gzip") {
		gzipReader, err := gzip.NewReader(res.Body)
		defer res.Body.Close()
		if err != nil {
			return nil, &Error{Err: err}
		}
		return &Response{StatusCode: res.StatusCode, ContentLength: res.ContentLength, Header: res.Header, Body: Body{gzipReader}}, nil
	} else {
		return &Response{StatusCode: res.StatusCode, ContentLength: res.ContentLength, Header: res.Header, Body: Body{res.Body}}, nil
	}
}

func isRedirect(status int) bool {
	switch status {
	case http.StatusMovedPermanently:
		return true
	case http.StatusFound:
		return true
	case http.StatusSeeOther:
		return true
	case http.StatusTemporaryRedirect:
		return true
	default:
		return false
	}
}
<|endoftext|>


<commit_before>//go:generate bitfanDoc
// Receive event on a ws connection
package websocketinput

import (
	"bytes"
	"io"
	"log"
	"net/http"
	"os"
	"time"

	"github.com/gorilla/websocket"
	"github.com/vjeantet/bitfan/codecs"
	"github.com/vjeantet/bitfan/processors"
)

const (
	// Time allowed to write a message to the peer.
	writeWait = 10 * time.Second

	// Time allowed to read the next pong message from the peer.
	pongWait = 60 * time.Second

	// Send pings to peer with this period. Must be less than pongWait.
	pingPeriod = (pongWait * 9) / 10

	// Maximum message size allowed from peer.
	maxMessageSize = 512
)

// Receive event on a ws connection
func New() processors.Processor {
	return &processor{opt: &options{}}
}

type options struct {
	processors.CommonOptions `mapstructure:",squash"`

	// The codec used for outputed data.
	// @Default "json"
	// @Type codec
	Codec codecs.CodecCollection

	// URI path
	// @Default "wsin"
	Uri string
}

// Reads events from standard input
type processor struct {
	processors.Base

	opt  *options
	q    chan bool
	host string

	wsupgrader websocket.Upgrader

	Hub *Hub
}

func (p *processor) Configure(ctx processors.ProcessorContext, conf map[string]interface{}) error {
	defaults := options{
		Codec: codecs.CodecCollection{
			Dec: codecs.New("json", nil, ctx.Log(), ctx.ConfigWorkingLocation()),
		},
		Uri: "wsin",
	}
	p.opt = &defaults
	err := p.ConfigureAndValidate(ctx, conf, p.opt)
	if err != nil {
		return err
	}

	if p.host, err = os.Hostname(); err != nil {
		p.Logger.Warnf("can not get hostname : %v", err)
	}

	p.wsupgrader = websocket.Upgrader{
		ReadBufferSize:  1024,
		WriteBufferSize: 1024,
		CheckOrigin: func(r *http.Request) bool {
			return true
		},
	}

	return err
}

func (p *processor) Start(e processors.IPacket) error {

	p.Hub = newHub(p.wellcome)
	go p.Hub.run()
	p.WebHook.Add(p.opt.Uri, p.HttpHandler)
	return nil
}

func (p *processor) Stop(e processors.IPacket) error {
	p.Hub.stop()
	return nil
}

func (p *processor) processMessage(m []byte) {
	r := bytes.NewReader(m)

	var dec codecs.Decoder
	var err error
	if dec, err = p.opt.Codec.NewDecoder(r); err != nil {
		p.Logger.Errorln("decoder error : ", err.Error())
		return
	}

	for dec.More() {
		var record interface{}
		if err = dec.Decode(&record); err != nil {
			if err == io.EOF {
				p.Logger.Debugln("error while decoding : ", err)
			} else {
				p.Logger.Errorln("error while decoding : ", err)
				break
			}
		}

		var e processors.IPacket
		switch v := record.(type) {
		case nil:
			e = p.NewPacket(string(m), map[string]interface{}{})
		case string:
			e = p.NewPacket(v, map[string]interface{}{})
		case map[string]interface{}:
			e = p.NewPacket("", v)
		case []interface{}:
			e = p.NewPacket("", map[string]interface{}{
				"request": v,
			})
		default:
			p.Logger.Errorf("Unknow structure %#v", v)
			continue
		}

		p.opt.ProcessCommonOptions(e.Fields())
		p.Send(e)
	}
}

// Handle Request received by bitfan for this agent (url hook should be registered during p.Start)
func (p *processor) HttpHandler(w http.ResponseWriter, r *http.Request) {
	conn, err := p.wsupgrader.Upgrade(w, r, nil)

	if err != nil {
		p.Logger.Errorf("websocket upgrade - %s", err.Error())
		return
	}

	client := &Client{
		hub:       p.Hub,
		conn:      conn,
		onMessage: p.processMessage,
	}
	p.Hub.register <- client

	go client.writePump()
	go client.readPump()
}

func (p *processor) wellcome() [][]byte {
	return [][]byte{}
}

// Client is a middleman between the websocket connection and the hub.
type Client struct {
	hub *Hub

	// The websocket connection.
	conn *websocket.Conn

	onMessage func([]byte)

	done chan struct{}
}

// readPump pumps messages from the websocket connection to the hub.

// The application runs readPump in a per-connection goroutine. The application
// ensures that there is at most one reader on a connection by executing all
// reads from this goroutine.
func (c *Client) readPump() {
	defer func() {
		c.conn.Close()
		close(c.done)
		c.hub.unregister <- c
	}()
	c.conn.SetReadLimit(maxMessageSize)
	c.conn.SetReadDeadline(time.Now().Add(pongWait))
	c.conn.SetPongHandler(func(string) error { c.conn.SetReadDeadline(time.Now().Add(pongWait)); return nil })
	for {
		_, message, err := c.conn.ReadMessage()
		if err != nil {
			if websocket.IsUnexpectedCloseError(err, websocket.CloseGoingAway) {
				log.Printf("IsUnexpectedCloseError error: %v", err)
			}
			break
		}

		// Build PACKET
		c.onMessage(message)
	}

}

// writePump pumps messages from the hub to the websocket connection.
//
// A goroutine running writePump is started for each connection. The
// application ensures that there is at most one writer to a connection by
// executing all writes from this goroutine.
func (c *Client) writePump() {
	ticker := time.NewTicker(pingPeriod)
	defer func() {
		ticker.Stop()
		c.conn.Close()
	}()
	for {
		select {
		case <-ticker.C:
			c.conn.SetWriteDeadline(time.Now().Add(writeWait))
			if err := c.conn.WriteMessage(websocket.PingMessage, []byte{}); err != nil {
				return
			}
		case <-c.done:
			return
		}
	}
}

// hub maintains the set of active clients and Broadcasts messages to the
// clients.
type Hub struct {
	// Registered clients.
	clients map[*Client]bool

	// Register requests from the clients.
	register chan *Client

	// Unregister requests from clients.
	unregister chan *Client

	done chan struct{}
}

func newHub(wellcomeMessage func() [][]byte) *Hub {
	return &Hub{
		register:   make(chan *Client),
		unregister: make(chan *Client),
		clients:    make(map[*Client]bool),
		done:       make(chan struct{}),
	}
}

func (h *Hub) stop() {
	for c, _ := range h.clients {
		h.unregister <- c
	}
	close(h.done)
}

func (h *Hub) run() {
	for {
		select {
		case <-h.done:
			return
		case client := <-h.register:
			client.done = make(chan struct{}, 1)
			h.clients[client] = true

		case client := <-h.unregister:
			if _, ok := h.clients[client]; ok {
				client.conn.Close()
				delete(h.clients, client)
			}
		}
	}
}
<commit_msg>processor : websocket input - add option maxMessageSize<commit_after>//go:generate bitfanDoc
// Receive event on a ws connection
package websocketinput

import (
	"bytes"
	"io"
	"log"
	"net/http"
	"os"
	"time"

	"github.com/gorilla/websocket"
	"github.com/vjeantet/bitfan/codecs"
	"github.com/vjeantet/bitfan/processors"
)

const (
	// Time allowed to write a message to the peer.
	writeWait = 10 * time.Second

	// Time allowed to read the next pong message from the peer.
	pongWait = 60 * time.Second

	// Send pings to peer with this period. Must be less than pongWait.
	pingPeriod = (pongWait * 9) / 10

	// Maximum message size allowed from peer.
	maxMessageSize = 512
)

// Receive event on a ws connection
func New() processors.Processor {
	return &processor{opt: &options{}}
}

type options struct {
	processors.CommonOptions `mapstructure:",squash"`

	// The codec used for outputed data.
	// @Default "json"
	// @Type codec
	Codec codecs.CodecCollection

	// URI path
	// @Default "wsin"
	Uri string

	// Maximum message size allowed from peer.
	MaxMessageSize int `mapstructure:"max_message_size"`
}

// Reads events from standard input
type processor struct {
	processors.Base

	opt  *options
	q    chan bool
	host string

	wsupgrader websocket.Upgrader

	Hub *Hub
}

func (p *processor) Configure(ctx processors.ProcessorContext, conf map[string]interface{}) error {
	defaults := options{
		Codec: codecs.CodecCollection{
			Dec: codecs.New("json", nil, ctx.Log(), ctx.ConfigWorkingLocation()),
		},
		Uri: "wsin",
	}
	p.opt = &defaults
	err := p.ConfigureAndValidate(ctx, conf, p.opt)
	if err != nil {
		return err
	}

	if p.host, err = os.Hostname(); err != nil {
		p.Logger.Warnf("can not get hostname : %v", err)
	}

	p.wsupgrader = websocket.Upgrader{
		ReadBufferSize:  1024,
		WriteBufferSize: 1024,
		CheckOrigin: func(r *http.Request) bool {
			return true
		},
	}

	return err
}

func (p *processor) Start(e processors.IPacket) error {

	p.Hub = newHub(p.wellcome)
	go p.Hub.run()
	p.WebHook.Add(p.opt.Uri, p.HttpHandler)
	return nil
}

func (p *processor) Stop(e processors.IPacket) error {
	p.Hub.stop()
	return nil
}

func (p *processor) processMessage(m []byte) {
	r := bytes.NewReader(m)

	var dec codecs.Decoder
	var err error
	if dec, err = p.opt.Codec.NewDecoder(r); err != nil {
		p.Logger.Errorln("decoder error : ", err.Error())
		return
	}

	for dec.More() {
		var record interface{}
		if err = dec.Decode(&record); err != nil {
			if err == io.EOF {
				p.Logger.Debugln("error while decoding : ", err)
			} else {
				p.Logger.Errorln("error while decoding : ", err)
				break
			}
		}

		var e processors.IPacket
		switch v := record.(type) {
		case nil:
			continue
		case string:
			e = p.NewPacket(v, map[string]interface{}{})
		case map[string]interface{}:
			e = p.NewPacket("", v)
		case []interface{}:
			e = p.NewPacket("", map[string]interface{}{
				"request": v,
			})
		default:
			p.Logger.Errorf("Unknow structure %#v", v)
			continue
		}

		p.opt.ProcessCommonOptions(e.Fields())
		p.Send(e)
	}
}

// Handle Request received by bitfan for this agent (url hook should be registered during p.Start)
func (p *processor) HttpHandler(w http.ResponseWriter, r *http.Request) {
	conn, err := p.wsupgrader.Upgrade(w, r, nil)

	if err != nil {
		p.Logger.Errorf("websocket upgrade - %s", err.Error())
		return
	}

	client := &Client{
		hub:       p.Hub,
		conn:      conn,
		onMessage: p.processMessage,
	}
	p.Hub.register <- client

	go client.writePump()
	go client.readPump(int64(p.opt.MaxMessageSize))
}

func (p *processor) wellcome() [][]byte {
	return [][]byte{}
}

// Client is a middleman between the websocket connection and the hub.
type Client struct {
	hub *Hub

	// The websocket connection.
	conn *websocket.Conn

	onMessage func([]byte)

	done chan struct{}
}

// readPump pumps messages from the websocket connection to the hub.

// The application runs readPump in a per-connection goroutine. The application
// ensures that there is at most one reader on a connection by executing all
// reads from this goroutine.
func (c *Client) readPump(maxMessageSize int64) {
	defer func() {
		c.conn.Close()
		close(c.done)
		c.hub.unregister <- c
	}()
	if maxMessageSize > 0 {
		c.conn.SetReadLimit(maxMessageSize)
	}

	c.conn.SetReadDeadline(time.Now().Add(pongWait))
	c.conn.SetPongHandler(func(string) error { c.conn.SetReadDeadline(time.Now().Add(pongWait)); return nil })
	for {
		_, message, err := c.conn.ReadMessage()
		if err != nil {
			if websocket.IsUnexpectedCloseError(err, websocket.CloseGoingAway) {
				log.Printf("IsUnexpectedCloseError error: %v", err)
			}
			break
		}

		// Build PACKET
		c.onMessage(message)
	}

}

// writePump pumps messages from the hub to the websocket connection.
//
// A goroutine running writePump is started for each connection. The
// application ensures that there is at most one writer to a connection by
// executing all writes from this goroutine.
func (c *Client) writePump() {
	ticker := time.NewTicker(pingPeriod)
	defer func() {
		ticker.Stop()
		c.conn.Close()
	}()
	for {
		select {
		case <-ticker.C:
			c.conn.SetWriteDeadline(time.Now().Add(writeWait))
			if err := c.conn.WriteMessage(websocket.PingMessage, []byte{}); err != nil {
				return
			}
		case <-c.done:
			return
		}
	}
}

// hub maintains the set of active clients and Broadcasts messages to the
// clients.
type Hub struct {
	// Registered clients.
	clients map[*Client]bool

	// Register requests from the clients.
	register chan *Client

	// Unregister requests from clients.
	unregister chan *Client

	done chan struct{}
}

func newHub(wellcomeMessage func() [][]byte) *Hub {
	return &Hub{
		register:   make(chan *Client),
		unregister: make(chan *Client),
		clients:    make(map[*Client]bool),
		done:       make(chan struct{}),
	}
}

func (h *Hub) stop() {
	for c, _ := range h.clients {
		h.unregister <- c
	}
	close(h.done)
}

func (h *Hub) run() {
	for {
		select {
		case <-h.done:
			return
		case client := <-h.register:
			client.done = make(chan struct{}, 1)
			h.clients[client] = true

		case client := <-h.unregister:
			if _, ok := h.clients[client]; ok {
				client.conn.Close()
				delete(h.clients, client)
			}
		}
	}
}
<|endoftext|>


<commit_before>package image

import (
	"io"
	"os"

	"github.com/dnephin/dobi/tasks/context"
	docker "github.com/fsouza/go-dockerclient"
)

// RunPush pushes an image to the registry
func RunPush(ctx *context.ExecuteContext, t *Task, _ bool) (bool, error) {
	pushTag := func(tag string) error {
		return pushImage(ctx, t, tag)
	}
	if err := t.ForEachTag(ctx, pushTag); err != nil {
		return false, err
	}
	t.logger().Info("Pushed")
	return true, nil
}

func pushImage(ctx *context.ExecuteContext, t *Task, tag string) error {
	repo, err := parseAuthRepo(tag)
	if err != nil {
		return err
	}

	return Stream(os.Stdout, func(out io.Writer) error {
		return ctx.Client.PushImage(docker.PushImageOptions{
			Name:          tag,
			OutputStream:  out,
			RawJSONStream: true,
			// TODO: timeout
		}, ctx.GetAuthConfig(repo))
	})
}
<commit_msg>Fix lint error, unused param<commit_after>package image

import (
	"io"
	"os"

	"github.com/dnephin/dobi/tasks/context"
	docker "github.com/fsouza/go-dockerclient"
)

// RunPush pushes an image to the registry
func RunPush(ctx *context.ExecuteContext, t *Task, _ bool) (bool, error) {
	pushTag := func(tag string) error {
		return pushImage(ctx, tag)
	}
	if err := t.ForEachTag(ctx, pushTag); err != nil {
		return false, err
	}
	t.logger().Info("Pushed")
	return true, nil
}

func pushImage(ctx *context.ExecuteContext, tag string) error {
	repo, err := parseAuthRepo(tag)
	if err != nil {
		return err
	}

	return Stream(os.Stdout, func(out io.Writer) error {
		return ctx.Client.PushImage(docker.PushImageOptions{
			Name:          tag,
			OutputStream:  out,
			RawJSONStream: true,
			// TODO: timeout
		}, ctx.GetAuthConfig(repo))
	})
}
<|endoftext|>


<commit_before>/*
 * Copyright (C) 2016, 2017 Sam Kumar, Michael Andersen, and the University
 * of California, Berkeley.
 *
 * This file is part of Mr. Plotter (the Multi-Resolution Plotter).
 *
 * Mr. Plotter is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published
 * by the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Mr. Plotter is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with Mr. Plotter.  If not, see <http://www.gnu.org/licenses/>.
 */

package main

import (
	"context"
	"fmt"
	"io"
	"log"
	"sync"
	"sync/atomic"

	"gopkg.in/BTrDB/btrdb.v4"

	ws "github.com/gorilla/websocket"
	uuid "github.com/pborman/uuid"
)

const (
	QUASAR_LOW   int64 = 1 - (16 << 56)
	QUASAR_HIGH  int64 = (48 << 56) - 1
	INVALID_TIME int64 = -0x8000000000000000
)

func splitTime(time int64) (millis int64, nanos int32) {
	millis = time / 1000000
	nanos = int32(time % 1000000)
	if nanos < 0 {
		nanos += 1000000
		millis++
	}
	return
}

type Writable interface {
	GetWriter() io.Writer
}

type ConnWrapper struct {
	Writing    *sync.Mutex
	Conn       *ws.Conn
	CurrWriter io.WriteCloser
}

func (cw *ConnWrapper) GetWriter() io.Writer {
	cw.Writing.Lock()
	w, err := cw.Conn.NextWriter(ws.TextMessage)
	if err == nil {
		cw.CurrWriter = w
		return w
	} else {
		log.Printf("Could not get writer on WebSocket: %v", err)
		return nil
	}
}

// DataRequester encapsulates a series of connections used for obtaining data
// from QUASAR.
type DataRequester struct {
	totalWaiting   uint64
	pending        uint32
	maxPending     uint32
	pendingLock    *sync.Mutex
	pendingCondVar *sync.Cond
	stateLock      *sync.RWMutex
	alive          bool

	btrdb *btrdb.BTrDB
}

// NewDataRequester creates a new DataRequester object.
// btrdbConn - established connection to a BTrDB cluster.
// maxPending - a limit on the maximum number of pending requests. */
func NewDataRequester(btrdbConn *btrdb.BTrDB, maxPending uint32) *DataRequester {
	pendingLock := &sync.Mutex{}
	var dr *DataRequester = &DataRequester{
		totalWaiting:   0,
		pending:        0,
		maxPending:     maxPending,
		pendingLock:    pendingLock,
		pendingCondVar: sync.NewCond(pendingLock),
		btrdb:          btrdbConn,
	}

	return dr
}

/* Makes a request for data and writes the result to the specified Writer. */
func (dr *DataRequester) MakeDataRequest(ctx context.Context, uuidBytes uuid.UUID, startTime int64, endTime int64, pw uint8, writ Writable) {
	atomic.AddUint64(&dr.totalWaiting, 1)
	defer atomic.AddUint64(&dr.totalWaiting, 0xFFFFFFFFFFFFFFFF)

	dr.pendingLock.Lock()
	for dr.pending == dr.maxPending {
		dr.pendingCondVar.Wait()
	}
	dr.pending += 1
	dr.pendingLock.Unlock()

	defer func() {
		dr.pendingLock.Lock()
		dr.pending -= 1
		dr.pendingCondVar.Signal()
		dr.pendingLock.Unlock()
	}()

	var w io.Writer

	var stream = dr.btrdb.StreamFromUUID(uuidBytes)

	var exists bool
	var err error
	exists, err = stream.Exists(ctx)
	if err != nil || !exists {
		w = writ.GetWriter()
		w.Write([]byte("[]"))
		return
	}

	var results chan btrdb.StatPoint
	var errors chan error
	results, _, errors = stream.AlignedWindows(ctx, startTime, endTime, pw, 0)

	w = writ.GetWriter()
	w.Write([]byte("["))

	var firstpt bool = true
	for statpt := range results {
		millis, nanos := splitTime(statpt.Time)
		if firstpt {
			w.Write([]byte(fmt.Sprintf("[%v,%v,%v,%v,%v,%v]", millis, nanos, statpt.Min, statpt.Mean, statpt.Max, statpt.Count)))
			firstpt = false
		} else {
			w.Write([]byte(fmt.Sprintf(",[%v,%v,%v,%v,%v,%v]", millis, nanos, statpt.Min, statpt.Mean, statpt.Max, statpt.Count)))
		}
	}

	var waserror bool = false
	for err = range errors {
		w.Write([]byte("\nError: "))
		w.Write([]byte(err.Error()))
		waserror = true
	}

	if !waserror {
		w.Write([]byte("]"))
	}
}

func (dr *DataRequester) MakeBracketRequest(ctx context.Context, uuids []uuid.UUID, writ Writable) {
	atomic.AddUint64(&dr.totalWaiting, 1)
	defer atomic.AddUint64(&dr.totalWaiting, 0xFFFFFFFFFFFFFFFF)

	dr.pendingLock.Lock()
	for dr.pending == dr.maxPending {
		dr.pendingCondVar.Wait()
	}
	dr.pending += 1
	dr.pendingLock.Unlock()

	defer func() {
		dr.pendingLock.Lock()
		dr.pending -= 1
		dr.pendingCondVar.Signal()
		dr.pendingLock.Unlock()
	}()

	var numResponses int = len(uuids) << 1
	var boundarySlice []int64 = make([]int64, numResponses)

	var wg sync.WaitGroup

	var stream *btrdb.Stream

	var i int
	for i = 0; i != numResponses; i++ {
		var seconditer bool = ((i & 1) != 0)

		if !seconditer {
			stream = dr.btrdb.StreamFromUUID(uuids[i>>1])

			var exists bool
			var err error
			exists, err = stream.Exists(ctx)
			if err != nil || !exists {
				boundarySlice[i] = INVALID_TIME
				i++
				boundarySlice[i] = INVALID_TIME
				continue
			} else {
				wg.Add(2)
			}
		}

		go func(stream *btrdb.Stream, loc *int64, high bool) {
			var rawpoint btrdb.RawPoint
			var e error
			var ref int64
			if high {
				ref = QUASAR_HIGH
			} else {
				ref = QUASAR_LOW
			}
			rawpoint, _, e = stream.Nearest(ctx, ref, 0, high)

			if e == nil {
				*loc = rawpoint.Time
			} else {
				*loc = INVALID_TIME
			}

			wg.Done()
		}(stream, &boundarySlice[i], seconditer)
	}

	wg.Wait()

	// For final processing once all responses are received
	var (
		boundary  int64
		lNanos    int32
		lMillis   int64
		rNanos    int32
		rMillis   int64
		lowest    int64     = QUASAR_HIGH
		highest   int64     = QUASAR_LOW
		trailchar rune      = ','
		w         io.Writer = writ.GetWriter()
	)

	w.Write([]byte("{\"Brackets\": ["))

	for i = 0; i < len(uuids); i++ {
		boundary = boundarySlice[i<<1]
		if boundary != INVALID_TIME && boundary < lowest {
			lowest = boundary
		}
		lMillis, lNanos = splitTime(boundary)
		boundary = boundarySlice[(i<<1)+1]
		if boundary != INVALID_TIME && boundary > highest {
			highest = boundary
		}
		rMillis, rNanos = splitTime(boundary)
		if i == len(uuids)-1 {
			trailchar = ']'
		}
		w.Write([]byte(fmt.Sprintf("[[%v,%v],[%v,%v]]%c", lMillis, lNanos, rMillis, rNanos, trailchar)))
	}
	if len(uuids) == 0 {
		w.Write([]byte("]"))
	}
	lMillis, lNanos = splitTime(lowest)
	rMillis, rNanos = splitTime(highest)
	w.Write([]byte(fmt.Sprintf(",\"Merged\":[[%v,%v],[%v,%v]]}", lMillis, lNanos, rMillis, rNanos)))
}
<commit_msg>Fix super minor backend bug<commit_after>/*
 * Copyright (C) 2016, 2017 Sam Kumar, Michael Andersen, and the University
 * of California, Berkeley.
 *
 * This file is part of Mr. Plotter (the Multi-Resolution Plotter).
 *
 * Mr. Plotter is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published
 * by the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Mr. Plotter is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with Mr. Plotter.  If not, see <http://www.gnu.org/licenses/>.
 */

package main

import (
	"context"
	"fmt"
	"io"
	"log"
	"sync"
	"sync/atomic"

	"gopkg.in/BTrDB/btrdb.v4"

	ws "github.com/gorilla/websocket"
	uuid "github.com/pborman/uuid"
)

const (
	QUASAR_LOW   int64 = 1 - (16 << 56)
	QUASAR_HIGH  int64 = (48 << 56) - 1
	INVALID_TIME int64 = -0x8000000000000000
)

func splitTime(time int64) (millis int64, nanos int32) {
	millis = time / 1000000
	nanos = int32(time % 1000000)
	if nanos < 0 {
		nanos += 1000000
		millis--
	}
	return
}

type Writable interface {
	GetWriter() io.Writer
}

type ConnWrapper struct {
	Writing    *sync.Mutex
	Conn       *ws.Conn
	CurrWriter io.WriteCloser
}

func (cw *ConnWrapper) GetWriter() io.Writer {
	cw.Writing.Lock()
	w, err := cw.Conn.NextWriter(ws.TextMessage)
	if err == nil {
		cw.CurrWriter = w
		return w
	} else {
		log.Printf("Could not get writer on WebSocket: %v", err)
		return nil
	}
}

// DataRequester encapsulates a series of connections used for obtaining data
// from QUASAR.
type DataRequester struct {
	totalWaiting   uint64
	pending        uint32
	maxPending     uint32
	pendingLock    *sync.Mutex
	pendingCondVar *sync.Cond
	stateLock      *sync.RWMutex
	alive          bool

	btrdb *btrdb.BTrDB
}

// NewDataRequester creates a new DataRequester object.
// btrdbConn - established connection to a BTrDB cluster.
// maxPending - a limit on the maximum number of pending requests. */
func NewDataRequester(btrdbConn *btrdb.BTrDB, maxPending uint32) *DataRequester {
	pendingLock := &sync.Mutex{}
	var dr *DataRequester = &DataRequester{
		totalWaiting:   0,
		pending:        0,
		maxPending:     maxPending,
		pendingLock:    pendingLock,
		pendingCondVar: sync.NewCond(pendingLock),
		btrdb:          btrdbConn,
	}

	return dr
}

/* Makes a request for data and writes the result to the specified Writer. */
func (dr *DataRequester) MakeDataRequest(ctx context.Context, uuidBytes uuid.UUID, startTime int64, endTime int64, pw uint8, writ Writable) {
	atomic.AddUint64(&dr.totalWaiting, 1)
	defer atomic.AddUint64(&dr.totalWaiting, 0xFFFFFFFFFFFFFFFF)

	dr.pendingLock.Lock()
	for dr.pending == dr.maxPending {
		dr.pendingCondVar.Wait()
	}
	dr.pending += 1
	dr.pendingLock.Unlock()

	defer func() {
		dr.pendingLock.Lock()
		dr.pending -= 1
		dr.pendingCondVar.Signal()
		dr.pendingLock.Unlock()
	}()

	var w io.Writer

	var stream = dr.btrdb.StreamFromUUID(uuidBytes)

	var exists bool
	var err error
	exists, err = stream.Exists(ctx)
	if err != nil || !exists {
		w = writ.GetWriter()
		w.Write([]byte("[]"))
		return
	}

	var results chan btrdb.StatPoint
	var errors chan error
	results, _, errors = stream.AlignedWindows(ctx, startTime, endTime, pw, 0)

	w = writ.GetWriter()
	w.Write([]byte("["))

	var firstpt bool = true
	for statpt := range results {
		millis, nanos := splitTime(statpt.Time)
		if firstpt {
			w.Write([]byte(fmt.Sprintf("[%v,%v,%v,%v,%v,%v]", millis, nanos, statpt.Min, statpt.Mean, statpt.Max, statpt.Count)))
			firstpt = false
		} else {
			w.Write([]byte(fmt.Sprintf(",[%v,%v,%v,%v,%v,%v]", millis, nanos, statpt.Min, statpt.Mean, statpt.Max, statpt.Count)))
		}
	}

	var waserror bool = false
	for err = range errors {
		w.Write([]byte("\nError: "))
		w.Write([]byte(err.Error()))
		waserror = true
	}

	if !waserror {
		w.Write([]byte("]"))
	}
}

func (dr *DataRequester) MakeBracketRequest(ctx context.Context, uuids []uuid.UUID, writ Writable) {
	atomic.AddUint64(&dr.totalWaiting, 1)
	defer atomic.AddUint64(&dr.totalWaiting, 0xFFFFFFFFFFFFFFFF)

	dr.pendingLock.Lock()
	for dr.pending == dr.maxPending {
		dr.pendingCondVar.Wait()
	}
	dr.pending += 1
	dr.pendingLock.Unlock()

	defer func() {
		dr.pendingLock.Lock()
		dr.pending -= 1
		dr.pendingCondVar.Signal()
		dr.pendingLock.Unlock()
	}()

	var numResponses int = len(uuids) << 1
	var boundarySlice []int64 = make([]int64, numResponses)

	var wg sync.WaitGroup

	var stream *btrdb.Stream

	var i int
	for i = 0; i != numResponses; i++ {
		var seconditer bool = ((i & 1) != 0)

		if !seconditer {
			stream = dr.btrdb.StreamFromUUID(uuids[i>>1])

			var exists bool
			var err error
			exists, err = stream.Exists(ctx)
			if err != nil || !exists {
				boundarySlice[i] = INVALID_TIME
				i++
				boundarySlice[i] = INVALID_TIME
				continue
			} else {
				wg.Add(2)
			}
		}

		go func(stream *btrdb.Stream, loc *int64, high bool) {
			var rawpoint btrdb.RawPoint
			var e error
			var ref int64
			if high {
				ref = QUASAR_HIGH
			} else {
				ref = QUASAR_LOW
			}
			rawpoint, _, e = stream.Nearest(ctx, ref, 0, high)

			if e == nil {
				*loc = rawpoint.Time
			} else {
				*loc = INVALID_TIME
			}

			wg.Done()
		}(stream, &boundarySlice[i], seconditer)
	}

	wg.Wait()

	// For final processing once all responses are received
	var (
		boundary  int64
		lNanos    int32
		lMillis   int64
		rNanos    int32
		rMillis   int64
		lowest    int64     = QUASAR_HIGH
		highest   int64     = QUASAR_LOW
		trailchar rune      = ','
		w         io.Writer = writ.GetWriter()
	)

	w.Write([]byte("{\"Brackets\": ["))

	for i = 0; i < len(uuids); i++ {
		boundary = boundarySlice[i<<1]
		if boundary != INVALID_TIME && boundary < lowest {
			lowest = boundary
		}
		lMillis, lNanos = splitTime(boundary)
		boundary = boundarySlice[(i<<1)+1]
		if boundary != INVALID_TIME && boundary > highest {
			highest = boundary
		}
		rMillis, rNanos = splitTime(boundary)
		if i == len(uuids)-1 {
			trailchar = ']'
		}
		w.Write([]byte(fmt.Sprintf("[[%v,%v],[%v,%v]]%c", lMillis, lNanos, rMillis, rNanos, trailchar)))
	}
	if len(uuids) == 0 {
		w.Write([]byte("]"))
	}
	lMillis, lNanos = splitTime(lowest)
	rMillis, rNanos = splitTime(highest)
	w.Write([]byte(fmt.Sprintf(",\"Merged\":[[%v,%v],[%v,%v]]}", lMillis, lNanos, rMillis, rNanos)))
}
<|endoftext|>


<commit_before>package exprel // import "layeh.com/exprel"

import (
	"strconv"
)

// Func is a function that can be executed from an Expression.
type Func func(call *Call) (interface{}, error)

// Call contains information about an expression function call.
type Call struct {
	// The name used to invoke the function.
	Name string
	// The arguments passed to the function.
	Values []interface{}
}

// String returns the ith argument, iff it is a string. Otherwise, the function
// panics with a *RuntimeError.
func (c *Call) String(i int) string {
	if len(c.Values) <= i {
		re(c.Name + " expects argument " + strconv.Itoa(i) + " to be string")
	}
	value, ok := c.Values[i].(string)
	if !ok {
		re(c.Name + " expects argument " + strconv.Itoa(i) + " to be string")
	}
	return value
}

// OptString returns the ith argument, iff it is a string. If the ith argument
// does not exist, def is returned. If the ith argument is not a string, the
// function panics with a *RuntimeError.
func (c *Call) OptString(i int, def string) string {
	if len(c.Values) <= i {
		return def
	}
	value, ok := c.Values[i].(string)
	if !ok {
		re(c.Name + " expects argument " + strconv.Itoa(i) + " to be string")
	}
	return value
}

// Number returns the ith argument, iff it is a float64. Otherwise, the
// function panics with a *RuntimeError.
func (c *Call) Number(i int) float64 {
	if len(c.Values) <= i {
		re(c.Name + " expects argument " + strconv.Itoa(i) + " to be float64")
	}
	value, ok := c.Values[i].(float64)
	if !ok {
		re(c.Name + " expects argument " + strconv.Itoa(i) + " to be float64")
	}
	return value
}

// OptNumber returns the ith argument, iff it is a float64. If the ith argument
// does not exist, def is returned. If the ith argument is not a number, the
// function panics with a *RuntimeError.
func (c *Call) OptNumber(i int, def float64) float64 {
	if len(c.Values) <= i {
		return def
	}
	value, ok := c.Values[i].(float64)
	if !ok {
		re(c.Name + " expects argument " + strconv.Itoa(i) + " to be float64")
	}
	return value
}

// Boolean returns the ith argument, iff it is a bool. Otherwise, the function
// panics with a *RuntimeError.
func (c *Call) Boolean(i int) bool {
	if len(c.Values) <= i {
		re(c.Name + " expects argument " + strconv.Itoa(i) + " to be bool")
	}
	value, ok := c.Values[i].(bool)
	if !ok {
		re(c.Name + " expects argument " + strconv.Itoa(i) + " to be bool")
	}
	return value
}

// OptBoolean returns the ith argument, iff it is a bool. If the ith argument
// does not exist, def is returned. If the ith argument is not a bool, the
// function panics with a *RuntimeError.
func (c *Call) OptBoolean(i int, def bool) bool {
	if len(c.Values) <= i {
		return def
	}
	value, ok := c.Values[i].(bool)
	if !ok {
		re(c.Name + " expects argument " + strconv.Itoa(i) + " to be bool")
	}
	return value
}

// Source is a source of data for an expression. Get is called when an
// identifier needs to be evaluated.
type Source interface {
	Get(name string) (value interface{}, ok bool)
}

// EmptySource is a Source that contains no values.
var EmptySource Source

type emptySource struct{}

func (emptySource) Get(name string) (interface{}, bool) {
	return nil, false
}

func init() {
	EmptySource = emptySource{}
}

// SourceFunc is a Source that looks up an identifier via a function.
type SourceFunc func(name string) (value interface{}, ok bool)

// Get implements Source.
func (fn SourceFunc) Get(name string) (interface{}, bool) {
	return fn(name)
}

// SourceMap is a Source that looks up an identifier in a map.
type SourceMap map[string]interface{}

// Get implements Source.
func (m SourceMap) Get(name string) (interface{}, bool) {
	value, ok := m[name]
	if !ok {
		return nil, false
	}
	switch value.(type) {
	case bool, string, float64, Func, func(*Call) (interface{}, error):
		return value, true
	default:
		return nil, false
	}
}

// Sources is a slice of sources. The first Source, in order, to return ok,
// will have its value returned.
type Sources []Source

// Get implements Source.
func (so Sources) Get(name string) (interface{}, bool) {
	for _, s := range so {
		value, ok := s.Get(name)
		if ok {
			return value, true
		}
	}
	return nil, false
}
<commit_msg>use single letter variable name in receiver functions<commit_after>package exprel // import "layeh.com/exprel"

import (
	"strconv"
)

// Func is a function that can be executed from an Expression.
type Func func(call *Call) (interface{}, error)

// Call contains information about an expression function call.
type Call struct {
	// The name used to invoke the function.
	Name string
	// The arguments passed to the function.
	Values []interface{}
}

// String returns the ith argument, iff it is a string. Otherwise, the function
// panics with a *RuntimeError.
func (c *Call) String(i int) string {
	if len(c.Values) <= i {
		re(c.Name + " expects argument " + strconv.Itoa(i) + " to be string")
	}
	value, ok := c.Values[i].(string)
	if !ok {
		re(c.Name + " expects argument " + strconv.Itoa(i) + " to be string")
	}
	return value
}

// OptString returns the ith argument, iff it is a string. If the ith argument
// does not exist, def is returned. If the ith argument is not a string, the
// function panics with a *RuntimeError.
func (c *Call) OptString(i int, def string) string {
	if len(c.Values) <= i {
		return def
	}
	value, ok := c.Values[i].(string)
	if !ok {
		re(c.Name + " expects argument " + strconv.Itoa(i) + " to be string")
	}
	return value
}

// Number returns the ith argument, iff it is a float64. Otherwise, the
// function panics with a *RuntimeError.
func (c *Call) Number(i int) float64 {
	if len(c.Values) <= i {
		re(c.Name + " expects argument " + strconv.Itoa(i) + " to be float64")
	}
	value, ok := c.Values[i].(float64)
	if !ok {
		re(c.Name + " expects argument " + strconv.Itoa(i) + " to be float64")
	}
	return value
}

// OptNumber returns the ith argument, iff it is a float64. If the ith argument
// does not exist, def is returned. If the ith argument is not a number, the
// function panics with a *RuntimeError.
func (c *Call) OptNumber(i int, def float64) float64 {
	if len(c.Values) <= i {
		return def
	}
	value, ok := c.Values[i].(float64)
	if !ok {
		re(c.Name + " expects argument " + strconv.Itoa(i) + " to be float64")
	}
	return value
}

// Boolean returns the ith argument, iff it is a bool. Otherwise, the function
// panics with a *RuntimeError.
func (c *Call) Boolean(i int) bool {
	if len(c.Values) <= i {
		re(c.Name + " expects argument " + strconv.Itoa(i) + " to be bool")
	}
	value, ok := c.Values[i].(bool)
	if !ok {
		re(c.Name + " expects argument " + strconv.Itoa(i) + " to be bool")
	}
	return value
}

// OptBoolean returns the ith argument, iff it is a bool. If the ith argument
// does not exist, def is returned. If the ith argument is not a bool, the
// function panics with a *RuntimeError.
func (c *Call) OptBoolean(i int, def bool) bool {
	if len(c.Values) <= i {
		return def
	}
	value, ok := c.Values[i].(bool)
	if !ok {
		re(c.Name + " expects argument " + strconv.Itoa(i) + " to be bool")
	}
	return value
}

// Source is a source of data for an expression. Get is called when an
// identifier needs to be evaluated.
type Source interface {
	Get(name string) (value interface{}, ok bool)
}

// EmptySource is a Source that contains no values.
var EmptySource Source

type emptySource struct{}

func (emptySource) Get(name string) (interface{}, bool) {
	return nil, false
}

func init() {
	EmptySource = emptySource{}
}

// SourceFunc is a Source that looks up an identifier via a function.
type SourceFunc func(name string) (value interface{}, ok bool)

// Get implements Source.
func (f SourceFunc) Get(name string) (interface{}, bool) {
	return f(name)
}

// SourceMap is a Source that looks up an identifier in a map.
type SourceMap map[string]interface{}

// Get implements Source.
func (m SourceMap) Get(name string) (interface{}, bool) {
	value, ok := m[name]
	if !ok {
		return nil, false
	}
	switch value.(type) {
	case bool, string, float64, Func, func(*Call) (interface{}, error):
		return value, true
	default:
		return nil, false
	}
}

// Sources is a slice of sources. The first Source, in order, to return ok,
// will have its value returned.
type Sources []Source

// Get implements Source.
func (s Sources) Get(name string) (interface{}, bool) {
	for _, s := range s {
		value, ok := s.Get(name)
		if ok {
			return value, true
		}
	}
	return nil, false
}
<|endoftext|>


<commit_before>/*******************************************************************************
The MIT License (MIT)

Copyright (c) 2013 Hajime Nakagami

Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*******************************************************************************/

package firebirdsql

import (
    "errors"
    "database/sql/driver"
    )

type _result struct {
}
func (r *_result) LastInsertId() (int64, error) {
    return 0, nil
}

func (r *_result) RowsAffected() (int64, error) {
    return 0, nil
}

type firebirdsqlStmt struct {
    wp *wireProtocol
    stmtHandle int32
    tx *firebirdsqlTx
}

func (stmt *firebirdsqlStmt) Close() (err error) {
    stmt.wp.opFreeStatement(stmt.stmtHandle, 2)     // DSQL_drop
    return
}

func (stmt *firebirdsqlStmt) NumInput() int {
    return -1
}

func (stmt *firebirdsqlStmt) Exec(args []driver.Value) (result driver.Result, err error) {
    stmt.wp.opExecute(stmt.stmtHandle, stmt.tx.transHandle, args)
    _, _, _, err = stmt.wp.opResponse()

    result = new(_result)
    return
}

func (stmt *firebirdsqlStmt) Query(args []driver.Value) (driver.Rows, error) {
    var err error
    err = errors.New("Not implement")
    return nil, err
}

func newFirebirdsqlStmt(fc *firebirdsqlConn, query string) (*firebirdsqlStmt, error) {
    var err error
    stmt := new(firebirdsqlStmt)
    stmt.wp = fc.wp
    stmt.tx = fc.tx
    fc.wp.opAllocateStatement()

    stmt.stmtHandle, _, _, err = fc.wp.opResponse()
    return stmt, err
}
<commit_msg>opPrepareStatement()<commit_after>/*******************************************************************************
The MIT License (MIT)

Copyright (c) 2013 Hajime Nakagami

Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*******************************************************************************/

package firebirdsql

import (
    "errors"
    "database/sql/driver"
    )

type _result struct {
}
func (r *_result) LastInsertId() (int64, error) {
    return 0, nil
}

func (r *_result) RowsAffected() (int64, error) {
    return 0, nil
}

type firebirdsqlStmt struct {
    wp *wireProtocol
    stmtHandle int32
    tx *firebirdsqlTx
}

func (stmt *firebirdsqlStmt) Close() (err error) {
    stmt.wp.opFreeStatement(stmt.stmtHandle, 2)     // DSQL_drop
    return
}

func (stmt *firebirdsqlStmt) NumInput() int {
    return -1
}

func (stmt *firebirdsqlStmt) Exec(args []driver.Value) (result driver.Result, err error) {
    stmt.wp.opExecute(stmt.stmtHandle, stmt.tx.transHandle, args)
    _, _, _, err = stmt.wp.opResponse()

    result = new(_result)
    return
}

func (stmt *firebirdsqlStmt) Query(args []driver.Value) (driver.Rows, error) {
    var err error
    err = errors.New("Not implement")
    return nil, err
}

func newFirebirdsqlStmt(fc *firebirdsqlConn, query string) (stmt *firebirdsqlStmt, err error) {
    stmt = new(firebirdsqlStmt)
    stmt.wp = fc.wp
    stmt.tx = fc.tx
    fc.wp.opAllocateStatement()
    stmt.stmtHandle, _, _, err = fc.wp.opResponse()
    if err != nil {
        return
    }
    fc.wp.opPrepareStatement(stmt.stmtHandle, stmt.tx.transHandle, query)
    _, _, _, err = fc.wp.opResponse()
    return
}
<|endoftext|>