<commit_before>// syncwatcher.go
package main

import (
	"code.google.com/p/go.exp/fsnotify"
	"os"
	"bufio"
	"io/ioutil"
	"net/http"
	"net/url"
	"crypto/tls"
	"encoding/json"
	"log"
	"fmt"
	"time"
	"flag"
	"runtime"
	"path/filepath"
	"strings"
	"sort"
)

type Configuration struct {
	Version			int
	Folders			[]FolderConfiguration
}

type FolderConfiguration struct {
	ID			string
	Path			string
	ReadOnly		bool
	RescanIntervalS		int
}

// HTTP Authentication
var (
	target		string
	authUser	string
	authPass	string
	csrfToken	string
	csrfFile	string
	apiKey		string
)

// HTTP Debounce
var (
	debounceTimeout = 300*time.Millisecond
	dirVsFiles = 10
)

// Main
var (
	stop = make(chan int)
)

func init() {
	flag.StringVar(&target, "target", "localhost:8080", "Target")
	flag.StringVar(&authUser, "user", "", "Username")
	flag.StringVar(&authPass, "pass", "", "Password")
	flag.StringVar(&csrfFile, "csrf", "", "CSRF token file")
	flag.StringVar(&apiKey, "api", "", "API key")
	flag.Parse()
	if !strings.Contains(target, "://") { target = "http://" + target }
	if len(csrfFile) > 0 {
		fd, err := os.Open(csrfFile)
		if err != nil {
			log.Fatal(err)
		}
		s := bufio.NewScanner(fd)
		for s.Scan() {
			csrfToken = s.Text()
		}
		fd.Close()
	}
}

func main() {

	testWebGuiPost()

	folders := getFolders()
	if (len(folders) == 0) {
		log.Fatal("No folders found");
	}
	for i := range folders {
		folder := folders[i]
		folderdir := folder.Path
		folderdir = expandTilde(folder.Path)
		go watchRepo(folder.ID, folderdir)
	}

	code := <-stop
	println("Exiting")
	os.Exit(code)

}

func getFolders() []FolderConfiguration {
	r, err := http.NewRequest("GET", target+"/rest/config", nil)
	if err != nil {
		log.Fatal(err)
	}
	if len(csrfToken) > 0 {
		r.Header.Set("X-CSRF-Token", csrfToken)
	}
	if len(authUser) > 0 {
		r.SetBasicAuth(authUser, authPass)
	}
	if len(apiKey) > 0 {
		r.Header.Set("X-API-Key", apiKey)
	}
	tr := &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify : true} }
	client := &http.Client{Transport: tr, Timeout: 5*time.Second}
	res, err := client.Do(r)
	if err != nil {
		log.Fatal(err)
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		log.Fatalf("Status %d != 200 for GET", res.StatusCode)
	}
	bs, err := ioutil.ReadAll(res.Body)
	if err != nil {
		log.Fatal(err)
	}
	var cfg Configuration
	err = json.Unmarshal(bs, &cfg)
	if err != nil {
		log.Fatal(err)
	}
	return cfg.Folders
}

func watchRepo(folder string, path string) {
	expandedPath := expandTilde(path)
	sw, err := NewSyncWatcher()
	if sw == nil || err != nil {
		log.Println(err)
		return
	}
	defer sw.Close()
	err = sw.Watch(expandedPath)
	if err != nil {
		log.Println(err)
		return
	}
	informChangeDebounced := informChangeDebounce(debounceTimeout, folder, path, dirVsFiles, informChange)
	log.Println("Watching " + folder + ": " + path)
	for {
		ev := waitForEvent(sw)
		if ev == nil {
			log.Println("Error: fsnotify event is nil")
			continue
		}
		log.Println("Change detected in " + ev.Name)
		informChangeDebounced(ev.Name)
	}
}

func waitForEvent(sw *SyncWatcher) (ev *fsnotify.FileEvent) {
	var ok bool
	select {
		case ev, ok = <-sw.Event:
			if !ok {
				log.Println("Error: channel closed")
			}
		case err, eok := <-sw.Error:
			log.Println(err, eok)
	}
	return
}

func testWebGuiPost() {
	r, err := http.NewRequest("POST", target+"/rest/404", nil)
	if err != nil {
		log.Fatal(err)
	}
	if len(csrfToken) > 0 {
		r.Header.Set("X-CSRF-Token", csrfToken)
	}
	if len(authUser) > 0 {
		r.SetBasicAuth(authUser, authPass)
	}
	if len(apiKey) > 0 {
		r.Header.Set("X-API-Key", apiKey)
	}
	tr := &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify : true} }
	client := &http.Client{Transport: tr, Timeout: 5*time.Second}
	res, err := client.Do(r)
	if err != nil {
		log.Fatal(err)
	}
	defer res.Body.Close()
	if res.StatusCode != 404 {
		log.Fatalf("Status %d != 404 for POST\n", res.StatusCode)
	}
}

func informChange(folder string, sub string) {
	data := url.Values {}
	data.Set("folder", folder)
	data.Set("sub", sub)
	r, err := http.NewRequest("POST", target+"/rest/scan?"+data.Encode(), nil)
	if err != nil {
		log.Println(err)
		return
	}
	if len(csrfToken) > 0 {
		r.Header.Set("X-CSRF-Token", csrfToken)
	}
	if len(authUser) > 0 {
		r.SetBasicAuth(authUser, authPass)
	}
	if len(apiKey) > 0 {
		r.Header.Set("X-API-Key", apiKey)
	}
	tr := &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify : true} }
	client := &http.Client{Transport: tr, Timeout: 5*time.Second}
	res, err := client.Do(r)
	if err != nil {
		log.Println(err)
		return
	}
	if res.StatusCode != 200 {
		log.Printf("Error: Status %d != 200 for POST.\n" + folder + ": " + sub, res.StatusCode)
		return
	} else {
		log.Println("Syncthing is indexing change in " + folder + ": " + sub)
	}
}

func informChangeDebounce(interval time.Duration, folder string, folderPath string, dirVsFiles int, callback func(folder string, sub string)) func(string) {
	debounce := func(f func(paths []string)) func(string) {
		timer := &time.Timer{}
		subs := make([]string, 0)
		return func(sub string) {
			timer.Stop()
			subs = append(subs, sub)
			timer = time.AfterFunc(interval, func() {
				f(subs)
				subs = make([]string, 0)
			})
		}
	}

	return debounce(func(paths []string) {
		// Do not inform Syncthing immediately but wait for debounce
		// Therefore, we need to keep track of the paths that were changed
		// This function optimises tracking in two ways:
		//	- If there are more than `dirVsFiles` changes in a directory, we inform Syncthing to scan the entire directory
		//	- Directories with parent directory changes are aggregated. If A/B has 3 changes and A/C has 8, A will have 11 changes and if this is bigger than dirVsFiles we will scan A.
		if (len(paths) == 0) { return }
		trackedPaths := make(map[string]int) // Map directories to scores; if score == -1 the path is a filename
		sort.Strings(paths) // Make sure parent paths are processed first
		for i := range paths {
			path := paths[i]
			dir := filepath.Dir(path)
			score := 1 // File change counts for 1 per directory
			if dir == filepath.Clean(path) {
				score = dirVsFiles // Is directory itself, should definitely inform
			}
			// Search for existing parent directory relations in the map
			for trackedPath, _ := range trackedPaths {
				if strings.Contains(dir, trackedPath) {
					// Increment score of tracked current/parent directory
					trackedPaths[trackedPath] += score
				}
			}
			_, exists := trackedPaths[dir]
			if !exists {
				trackedPaths[dir] = score
			}
			trackedPaths[path] = -1
		}
		var keys []string
		for k := range trackedPaths {
			keys = append(keys, k)
		}
		sort.Strings(keys) // Sort directories before their own files
		previousDone, previousPath := false, ""
		for i := range keys {
			trackedPath := keys[i]
			trackedPathScore, _ := trackedPaths[trackedPath]
			if previousDone && strings.Contains(trackedPath, previousPath) { continue } // Already informed parent directory change
			if trackedPathScore < dirVsFiles && trackedPathScore != -1 { continue } // Not enough files for this directory or it is a file
			previousDone = trackedPathScore != -1
			previousPath = trackedPath
			sub := strings.TrimPrefix(trackedPath, folderPath)
			sub = strings.TrimPrefix(sub, string(os.PathSeparator))
			callback(folder, sub)
		}
	})
}

func getHomeDir() string {
	var home string
	switch runtime.GOOS {
		case "windows":
		home = filepath.Join(os.Getenv("HomeDrive"), os.Getenv("HomePath"))
		if home == "" {
			home = os.Getenv("UserProfile")
		}
		default:
			home = os.Getenv("HOME")
	}
	if home == "" {
		log.Fatal("No home path found - set $HOME (or the platform equivalent).")
	}
	return home
}

func expandTilde(p string) string {
	if p == "~" {
		return getHomeDir()
	}
	p = filepath.FromSlash(p)
	if !strings.HasPrefix(p, fmt.Sprintf("~%c", os.PathSeparator)) {
		return p
	}
	return filepath.Join(getHomeDir(), p[2:])
}
<commit_msg>Filter duplicate inotify events<commit_after>// syncwatcher.go
package main

import (
	"code.google.com/p/go.exp/fsnotify"
	"os"
	"bufio"
	"io/ioutil"
	"net/http"
	"net/url"
	"crypto/tls"
	"encoding/json"
	"log"
	"fmt"
	"time"
	"flag"
	"runtime"
	"path/filepath"
	"strings"
	"sort"
)

type Configuration struct {
	Version			int
	Folders			[]FolderConfiguration
}

type FolderConfiguration struct {
	ID			string
	Path			string
	ReadOnly		bool
	RescanIntervalS		int
}

// HTTP Authentication
var (
	target		string
	authUser	string
	authPass	string
	csrfToken	string
	csrfFile	string
	apiKey		string
)

// HTTP Debounce
var (
	debounceTimeout = 300*time.Millisecond
	dirVsFiles = 10
)

// Main
var (
	stop = make(chan int)
)

func init() {
	flag.StringVar(&target, "target", "localhost:8080", "Target")
	flag.StringVar(&authUser, "user", "", "Username")
	flag.StringVar(&authPass, "pass", "", "Password")
	flag.StringVar(&csrfFile, "csrf", "", "CSRF token file")
	flag.StringVar(&apiKey, "api", "", "API key")
	flag.Parse()
	if !strings.Contains(target, "://") { target = "http://" + target }
	if len(csrfFile) > 0 {
		fd, err := os.Open(csrfFile)
		if err != nil {
			log.Fatal(err)
		}
		s := bufio.NewScanner(fd)
		for s.Scan() {
			csrfToken = s.Text()
		}
		fd.Close()
	}
}

func main() {

	testWebGuiPost()

	folders := getFolders()
	if (len(folders) == 0) {
		log.Fatal("No folders found");
	}
	for i := range folders {
		folder := folders[i]
		folderdir := folder.Path
		folderdir = expandTilde(folder.Path)
		go watchRepo(folder.ID, folderdir)
	}

	code := <-stop
	println("Exiting")
	os.Exit(code)

}

func getFolders() []FolderConfiguration {
	r, err := http.NewRequest("GET", target+"/rest/config", nil)
	if err != nil {
		log.Fatal(err)
	}
	if len(csrfToken) > 0 {
		r.Header.Set("X-CSRF-Token", csrfToken)
	}
	if len(authUser) > 0 {
		r.SetBasicAuth(authUser, authPass)
	}
	if len(apiKey) > 0 {
		r.Header.Set("X-API-Key", apiKey)
	}
	tr := &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify : true} }
	client := &http.Client{Transport: tr, Timeout: 5*time.Second}
	res, err := client.Do(r)
	if err != nil {
		log.Fatal(err)
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		log.Fatalf("Status %d != 200 for GET", res.StatusCode)
	}
	bs, err := ioutil.ReadAll(res.Body)
	if err != nil {
		log.Fatal(err)
	}
	var cfg Configuration
	err = json.Unmarshal(bs, &cfg)
	if err != nil {
		log.Fatal(err)
	}
	return cfg.Folders
}

func watchRepo(folder string, path string) {
	expandedPath := expandTilde(path)
	sw, err := NewSyncWatcher()
	if sw == nil || err != nil {
		log.Println(err)
		return
	}
	defer sw.Close()
	err = sw.Watch(expandedPath)
	if err != nil {
		log.Println(err)
		return
	}
	informChangeDebounced := informChangeDebounce(debounceTimeout, folder, path, dirVsFiles, informChange)
	log.Println("Watching " + folder + ": " + path)
	for {
		ev := waitForEvent(sw)
		if ev == nil {
			log.Println("Error: fsnotify event is nil")
			continue
		}
		log.Println("Change detected in " + ev.Name)
		informChangeDebounced(ev.Name)
	}
}

func waitForEvent(sw *SyncWatcher) (ev *fsnotify.FileEvent) {
	var ok bool
	select {
		case ev, ok = <-sw.Event:
			if !ok {
				log.Println("Error: channel closed")
			}
		case err, eok := <-sw.Error:
			log.Println(err, eok)
	}
	return
}

func testWebGuiPost() {
	r, err := http.NewRequest("POST", target+"/rest/404", nil)
	if err != nil {
		log.Fatal(err)
	}
	if len(csrfToken) > 0 {
		r.Header.Set("X-CSRF-Token", csrfToken)
	}
	if len(authUser) > 0 {
		r.SetBasicAuth(authUser, authPass)
	}
	if len(apiKey) > 0 {
		r.Header.Set("X-API-Key", apiKey)
	}
	tr := &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify : true} }
	client := &http.Client{Transport: tr, Timeout: 5*time.Second}
	res, err := client.Do(r)
	if err != nil {
		log.Fatal(err)
	}
	defer res.Body.Close()
	if res.StatusCode != 404 {
		log.Fatalf("Status %d != 404 for POST\n", res.StatusCode)
	}
}

func informChange(folder string, sub string) {
	data := url.Values {}
	data.Set("folder", folder)
	data.Set("sub", sub)
	r, err := http.NewRequest("POST", target+"/rest/scan?"+data.Encode(), nil)
	if err != nil {
		log.Println(err)
		return
	}
	if len(csrfToken) > 0 {
		r.Header.Set("X-CSRF-Token", csrfToken)
	}
	if len(authUser) > 0 {
		r.SetBasicAuth(authUser, authPass)
	}
	if len(apiKey) > 0 {
		r.Header.Set("X-API-Key", apiKey)
	}
	tr := &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify : true} }
	client := &http.Client{Transport: tr, Timeout: 5*time.Second}
	res, err := client.Do(r)
	if err != nil {
		log.Println(err)
		return
	}
	if res.StatusCode != 200 {
		log.Printf("Error: Status %d != 200 for POST.\n" + folder + ": " + sub, res.StatusCode)
		return
	} else {
		log.Println("Syncthing is indexing change in " + folder + ": " + sub)
	}
}

func informChangeDebounce(interval time.Duration, folder string, folderPath string, dirVsFiles int, callback func(folder string, sub string)) func(string) {
	debounce := func(f func(paths []string)) func(string) {
		timer := &time.Timer{}
		subs := make([]string, 0)
		return func(sub string) {
			timer.Stop()
			subs = append(subs, sub)
			timer = time.AfterFunc(interval, func() {
				f(subs)
				subs = make([]string, 0)
			})
		}
	}

	return debounce(func(paths []string) {
		// Do not inform Syncthing immediately but wait for debounce
		// Therefore, we need to keep track of the paths that were changed
		// This function optimises tracking in two ways:
		//	- If there are more than `dirVsFiles` changes in a directory, we inform Syncthing to scan the entire directory
		//	- Directories with parent directory changes are aggregated. If A/B has 3 changes and A/C has 8, A will have 11 changes and if this is bigger than dirVsFiles we will scan A.
		if (len(paths) == 0) { return }
		trackedPaths := make(map[string]int) // Map directories to scores; if score == -1 the path is a filename
		sort.Strings(paths) // Make sure parent paths are processed first
		previousPath := "" // Filter duplicates
		for i := range paths {
			path := paths[i]
			if (path == previousPath) {
				continue
			}
			previousPath = path
			dir := filepath.Dir(path)
			score := 1 // File change counts for 1 per directory
			if dir == filepath.Clean(path) {
				score = dirVsFiles // Is directory itself, should definitely inform
			}
			// Search for existing parent directory relations in the map
			for trackedPath, _ := range trackedPaths {
				if strings.Contains(dir, trackedPath) {
					// Increment score of tracked current/parent directory
					trackedPaths[trackedPath] += score
				}
			}
			_, exists := trackedPaths[dir]
			if !exists {
				trackedPaths[dir] = score
			}
			trackedPaths[path] = -1
		}
		var keys []string
		for k := range trackedPaths {
			keys = append(keys, k)
		}
		sort.Strings(keys) // Sort directories before their own files
		previousDone, previousPath := false, ""
		for i := range keys {
			trackedPath := keys[i]
			trackedPathScore, _ := trackedPaths[trackedPath]
			if previousDone && strings.Contains(trackedPath, previousPath) { continue } // Already informed parent directory change
			if trackedPathScore < dirVsFiles && trackedPathScore != -1 { continue } // Not enough files for this directory or it is a file
			previousDone = trackedPathScore != -1
			previousPath = trackedPath
			sub := strings.TrimPrefix(trackedPath, folderPath)
			sub = strings.TrimPrefix(sub, string(os.PathSeparator))
			callback(folder, sub)
		}
	})
}

func getHomeDir() string {
	var home string
	switch runtime.GOOS {
		case "windows":
		home = filepath.Join(os.Getenv("HomeDrive"), os.Getenv("HomePath"))
		if home == "" {
			home = os.Getenv("UserProfile")
		}
		default:
			home = os.Getenv("HOME")
	}
	if home == "" {
		log.Fatal("No home path found - set $HOME (or the platform equivalent).")
	}
	return home
}

func expandTilde(p string) string {
	if p == "~" {
		return getHomeDir()
	}
	p = filepath.FromSlash(p)
	if !strings.HasPrefix(p, fmt.Sprintf("~%c", os.PathSeparator)) {
		return p
	}
	return filepath.Join(getHomeDir(), p[2:])
}
<|endoftext|>


<commit_before>package main

//import needed libraries
import (
	"bytes"
	"flag"
	"fmt"
	"net"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
	"time"
)

//EXAMPLE:
//go run telnet2http.go "destination url" [port [method [timeout]]]
//go run telnet2http.go "http://de.wikipedia.org:80/" 3000 GET 5
//telnet localhost 3000
//nc localhost 3000 < LICENSE

var timeout int64 = 0

//handle tcp/ip connections
func handleConnection(c net.Conn, msgchan chan<- string) {
	defer c.Close()
	fmt.Printf("Connection from %v established.\n", c.RemoteAddr())
	if timeout != 0 {
		c.SetReadDeadline(time.Now().Add(time.Second * time.Duration(timeout)))
	}
	buf := make([]byte, 4096)
	for {
		n, err := c.Read(buf)
		if (err != nil) || (n == 0) {
			c.Close()
			break
		}
		msgchan <- string(buf[0:n])
	}
	fmt.Printf("Connection from %v closed.\n", c.RemoteAddr())
}

//send message via http
func printMessages(msgchan <-chan string, urlstr string, method string) {
	for {
		msg := strings.TrimSpace(<-msgchan)
		fmt.Printf("data: %s\n", msg)
		data := url.Values{}
		data.Add("value", strings.TrimSpace(msg))
		client := &http.Client{}
		if method == "POST" {
			r, _ := http.NewRequest("POST", urlstr, bytes.NewBufferString(data.Encode()))
			r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
			r.Header.Add("Content-Length", strconv.Itoa(len(data.Encode())))
			rsp, _ := client.Do(r)
			defer rsp.Body.Close()
		} else {
			r, _ := http.NewRequest("GET", urlstr+"?"+data.Encode(), nil)
			r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
			r.Header.Add("Content-Length", strconv.Itoa(len(data.Encode())))
			rsp, _ := client.Do(r)
			defer rsp.Body.Close()
		}
	}
}

//main function to start listener
func main() {
	flag.Parse()
	url := flag.Arg(0)
	port := ":" + flag.Arg(1)
	method := flag.Arg(2)
	timeout, _ = strconv.ParseInt(flag.Arg(3), 10, 64)
	if url == "" {
		url = "localhost"
	}
	if port == ":" {
		port = ":23"
	}
	if method == "" {
		method = "POST"
	}
	fmt.Printf("Listening on port %v and sending via %v to %v\n", port, method, url)
	if timeout != 0 {
		fmt.Printf("Abort telnet connection after %v seconds\n", timeout)
	}
	ln, err := net.Listen("tcp", port)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	msgchan := make(chan string)
	go printMessages(msgchan, url, method)
	for {
		conn, err := ln.Accept()
		if err != nil {
			fmt.Println(err)
			continue
		}
		go handleConnection(conn, msgchan)
	}
}
<commit_msg>better code for GET/POST select<commit_after>package main

//import needed libraries
import (
	"bytes"
	"flag"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	uri "net/url"
	"os"
	"strconv"
	"strings"
	"time"
)

//EXAMPLE:
//go run telnet2http.go "destination url" [port [method [timeout]]]
//go run telnet2http.go "http://de.wikipedia.org:80/" 3000 GET 5
//telnet localhost 3000
//nc localhost 3000 < LICENSE

var timeout int64 = 0

//handle tcp/ip connections
func handleConnection(c *net.TCPConn, msgchan chan<- string) {
	defer c.Close()
	fmt.Printf("Connection from %v established.\n", c.RemoteAddr())
	if timeout != 0 {
		c.SetReadDeadline(time.Now().Add(time.Second * time.Duration(timeout)))
	}
	buf := make([]byte, 4096)
	for {
		n, err := c.Read(buf)
		if (err != nil) || (n == 0) {
			c.Close()
			break
		}
		msgchan <- string(buf[0:n])
	}
	fmt.Printf("Connection from %v closed.\n", c.RemoteAddr())
}

//send message via http
func printMessages(msgchan <-chan string, urlstr string, method string) {
	var url string
	var body *bytes.Buffer
	var rtype string
	var clength string
	var bodyBytes []byte
	for {
		msg := strings.TrimSpace(<-msgchan)
		fmt.Printf("data: %s\n", msg)
		data := uri.Values{}
		data.Add("value", strings.TrimSpace(msg))
		client := &http.Client{}
		if method == "POST" {
			rtype = "POST"
			url = urlstr
			body = bytes.NewBufferString(data.Encode())
			clength = strconv.Itoa(len(data.Encode()))
		} else {
			rtype = "GET"
			url = urlstr + "?" + data.Encode()
			body = bytes.NewBufferString("")
			clength = "0"
		}
		fmt.Println(rtype + "\n" + url + "\n" + clength)
		r, _ := http.NewRequest(rtype, url, body)
		r.Header.Add("User-Agent", "telnet2http")
		r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
		r.Header.Add("Content-Length", clength)
		rsp, err := client.Do(r)
		if err != nil {
			fmt.Println(err)
		} else {
			if rsp.StatusCode == 200 {
				bodyBytes, _ = ioutil.ReadAll(rsp.Body)
				fmt.Println(string(bodyBytes))
			} else if err != nil {
				fmt.Println(err)
			} else {
				fmt.Println("The remote end did not return a HTTP 200 (OK) response.")
			}
			rsp.Body.Close()
		}
	}
}

//main function to start listener
func main() {
	flag.Parse()
	url := flag.Arg(0)
	port := ":" + flag.Arg(1)
	method := flag.Arg(2)
	timeout, _ = strconv.ParseInt(flag.Arg(3), 10, 64)
	if url == "" {
		url = "localhost"
	}
	if port == ":" {
		port = ":23"
	}
	if method == "" {
		method = "POST"
	}
	fmt.Printf("Listening on port %v and sending via %v to %v\n", port, method, url)
	if timeout != 0 {
		fmt.Printf("Abort telnet connection after %v seconds\n", timeout)
	}
	l, err := net.Listen("tcp", port)
	ln := l.(*net.TCPListener)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	msgchan := make(chan string)
	go printMessages(msgchan, url, method)
	for {
		conn, err := ln.AcceptTCP()
		if err != nil {
			fmt.Println(err)
			continue
		}
		go handleConnection(conn, msgchan)
	}
}
<|endoftext|>


<commit_before>package genetics

import (
	"math/rand"
	"testing"
	"github.com/yaricom/goNEAT/neat"
	"sort"
	"bytes"
	"math"
	"fmt"
)

func buildSpeciesWithOrganisms(id int) *Species {
	gen := &Genome{
		Id:1,
	}

	sp := NewSpecies(id)
	sp.addOrganism(NewOrganism(5.0 * float64(id), gen, id))
	sp.addOrganism(NewOrganism(15.0 * float64(id), gen, id))
	sp.addOrganism(NewOrganism(10.0 * float64(id), gen, id))

	return sp
}

func TestSpecies_GaussianMate(t *testing.T) {
	s, rand_species := 1, 1
	sorted_species := []int {6, 5, 4, 3, 2, 1, 0}
	giveup := 0
	for ; rand_species == s && giveup < 5; {
		// Choose a random species tending towards better species
		rand_mult := (gaussian.StdGaussian() / 2.0 + 1.0) / 4.0
		// This tends to select better species
		rand_species_num := int(math.Floor(rand_mult * float64(len(sorted_species) - 1) + 0.5))
		fmt.Printf("rand_species_num: %d, rand_mult: %f\n", rand_species_num, rand_mult)
		rand_species = sorted_species[rand_species_num]

		giveup++
	}
}

func TestSpecies_Write(t *testing.T) {
	sp := buildSpeciesWithOrganisms(1)

	out_buf := bytes.NewBufferString("")
	sp.Write(out_buf)

	t.Log(out_buf)
}

// Tests Species adjustFitness
func TestSpecies_adjustFitness(t *testing.T)  {
	sp := buildSpeciesWithOrganisms(1)

	// Configuration
	conf := neat.NeatContext{
		DropOffAge:5,
		SurvivalThresh:0.5,
		AgeSignificance:0.5,
	}
	sp.adjustFitness(&conf)

	// test results
	if sp.Organisms[0].IsChampion != true {
		t.Error("sp.Organisms[0].IsChampion", true, sp.Organisms[0].IsChampion)
	}
	if sp.AgeOfLastImprovement != 1 {
		t.Error("sp.AgeOfLastImprovement", 1, sp.AgeOfLastImprovement)
	}
	if sp.MaxFitnessEver != 15.0 {
		t.Error("sp.MaxFitnessEver", 15.0, sp.MaxFitnessEver)
	}
	if sp.Organisms[2].ToEliminate != true {
		t.Error("sp.Organisms[2].ToEliminate", true, sp.Organisms[2].ToEliminate)
	}
}

// Tests Species countOffspring
func TestSpecies_countOffspring(t *testing.T) {
	sp := buildSpeciesWithOrganisms(1)
	for i, o := range sp.Organisms {
		o.ExpectedOffspring = float64(i) * 1.5
	}

	skim := sp.countOffspring(0.5)

	if sp.ExpectedOffspring != 5 {
		t.Error("sp.ExpectedOffspring", 5, sp.ExpectedOffspring)
	}
	if skim != 0 {
		t.Error("skim", 0, skim)
	}

	sp = buildSpeciesWithOrganisms(2)
	for i, o := range sp.Organisms {
		o.ExpectedOffspring = float64(i) * 1.5
	}
	skim = sp.countOffspring(0.4)
	if sp.ExpectedOffspring != 4 {
		t.Error("sp.ExpectedOffspring", 5, sp.ExpectedOffspring)
	}
	if skim != 0.9 {
		t.Error("skim", 0.9, skim)
	}
}

// Tests Species computeAvgFitness
func TestSpecies_computeAvgFitness(t *testing.T) {
	sp := buildSpeciesWithOrganisms(1)

	avg_check := 0.0
	for _, o := range sp.Organisms{
		avg_check += o.Fitness
	}
	avg_check /= float64(len(sp.Organisms))

	sp.ComputeAvgFitness()
	if sp.AvgFitness != avg_check {
		t.Error("sp.AvgFitness != avg_check", sp.AvgFitness, avg_check)
	}
}

func TestSpecies_computeMaxFitness(t *testing.T) {
	sp := buildSpeciesWithOrganisms(1)

	sp.ComputeMaxFitness()
	if sp.MaxFitness != 15.0 {
		t.Error("sp.MaxFitness != 15.0", 15.0, sp.MaxFitness)
	}
}

func TestSpecies_findChampion(t *testing.T) {
	sp := buildSpeciesWithOrganisms(1)

	champ := sp.findChampion()
	if champ.Fitness != 15.0 {
		t.Error("champ.Fitness != 15.0", champ.Fitness)
	}

}

func TestSpecies_removeOrganism(t *testing.T) {
	sp := buildSpeciesWithOrganisms(1)

	// test remove
	size := len(sp.Organisms)
	res, err := sp.removeOrganism(sp.Organisms[0])
	if res != true {
		t.Error("res != true", res, err)
	}
	if err != nil {
		t.Error("err != nil", err)
	}
	if size - 1 != len(sp.Organisms) {
		t.Error("size - 1 != len(sp.Organisms)", size - 1, len(sp.Organisms))
	}

	// test fail to remove
	size = len(sp.Organisms)
	gen := &Genome{
		Id:1,
	}
	org := NewOrganism(6.0, gen, 1)
	res, err = sp.removeOrganism(org)
	if res == true {
		t.Error("res == true", res, err)
	}
	if err == nil {
		t.Error("err == nil", res, err)
	}
	if size != len(sp.Organisms) {
		t.Error("size != len(sp.Organisms)", size, len(sp.Organisms))
	}
}

// Tests Species reproduce failure
func TestSpecies_reproduce_fail(t *testing.T) {
	sp := NewSpecies(1)

	sp.ExpectedOffspring = 1

	res, err := sp.reproduce(1, nil, nil, nil)
	if res != false {
		t.Error("res != false")
	}
	if err == nil {
		t.Error("err == nil")
	}
}

// Tests Species reproduce success
func TestSpecies_reproduce(t *testing.T) {
	rand.Seed(42)
	in, out, nmax, n := 3, 2, 15, 3
	recurrent := false
	link_prob := 0.8

	// Configuration
	conf := neat.NeatContext {
		DropOffAge:5,
		SurvivalThresh:0.5,
		AgeSignificance:0.5,
		PopSize:30,
		CompatThreshold:0.6,
	}

	gen := NewGenomeRand(1, in, out, n, nmax, recurrent, link_prob)
	pop, err := NewPopulation(gen, &conf)
	if err != nil {
		t.Error(err)
	}
	if pop == nil {
		t.Error("pop == nil")
	}

	// Stick the Species pointers into a new Species list for sorting
	sorted_species := make([]*Species, len(pop.Species))
	copy(sorted_species, pop.Species)

	// Sort the Species by max original fitness of its first organism
	sort.Sort(ByOrganismOrigFitness(sorted_species))

	pop.Species[0].ExpectedOffspring = 11

	res, err := pop.Species[0].reproduce(1, pop, sorted_species, &conf)
	if !res {
		t.Error("No reproduction", err)
	}
	if err != nil {
		t.Error("err != nil", err)
	}
	after := 0
	for _, sp := range pop.Species {
		after += len(sp.Organisms)
	}

	if after != conf.PopSize + pop.Species[0].ExpectedOffspring {
		t.Error("No new baby was created", after)
	}
}
<commit_msg>Removed obsolete test case<commit_after>package genetics

import (
	"math/rand"
	"testing"
	"github.com/yaricom/goNEAT/neat"
	"sort"
	"bytes"
)

func buildSpeciesWithOrganisms(id int) *Species {
	gen := &Genome{
		Id:1,
	}

	sp := NewSpecies(id)
	sp.addOrganism(NewOrganism(5.0 * float64(id), gen, id))
	sp.addOrganism(NewOrganism(15.0 * float64(id), gen, id))
	sp.addOrganism(NewOrganism(10.0 * float64(id), gen, id))

	return sp
}

func TestSpecies_Write(t *testing.T) {
	sp := buildSpeciesWithOrganisms(1)

	out_buf := bytes.NewBufferString("")
	sp.Write(out_buf)

	t.Log(out_buf)
}

// Tests Species adjustFitness
func TestSpecies_adjustFitness(t *testing.T)  {
	sp := buildSpeciesWithOrganisms(1)

	// Configuration
	conf := neat.NeatContext{
		DropOffAge:5,
		SurvivalThresh:0.5,
		AgeSignificance:0.5,
	}
	sp.adjustFitness(&conf)

	// test results
	if sp.Organisms[0].IsChampion != true {
		t.Error("sp.Organisms[0].IsChampion", true, sp.Organisms[0].IsChampion)
	}
	if sp.AgeOfLastImprovement != 1 {
		t.Error("sp.AgeOfLastImprovement", 1, sp.AgeOfLastImprovement)
	}
	if sp.MaxFitnessEver != 15.0 {
		t.Error("sp.MaxFitnessEver", 15.0, sp.MaxFitnessEver)
	}
	if sp.Organisms[2].ToEliminate != true {
		t.Error("sp.Organisms[2].ToEliminate", true, sp.Organisms[2].ToEliminate)
	}
}

// Tests Species countOffspring
func TestSpecies_countOffspring(t *testing.T) {
	sp := buildSpeciesWithOrganisms(1)
	for i, o := range sp.Organisms {
		o.ExpectedOffspring = float64(i) * 1.5
	}

	skim := sp.countOffspring(0.5)

	if sp.ExpectedOffspring != 5 {
		t.Error("sp.ExpectedOffspring", 5, sp.ExpectedOffspring)
	}
	if skim != 0 {
		t.Error("skim", 0, skim)
	}

	sp = buildSpeciesWithOrganisms(2)
	for i, o := range sp.Organisms {
		o.ExpectedOffspring = float64(i) * 1.5
	}
	skim = sp.countOffspring(0.4)
	if sp.ExpectedOffspring != 4 {
		t.Error("sp.ExpectedOffspring", 5, sp.ExpectedOffspring)
	}
	if skim != 0.9 {
		t.Error("skim", 0.9, skim)
	}
}

// Tests Species computeAvgFitness
func TestSpecies_computeAvgFitness(t *testing.T) {
	sp := buildSpeciesWithOrganisms(1)

	avg_check := 0.0
	for _, o := range sp.Organisms{
		avg_check += o.Fitness
	}
	avg_check /= float64(len(sp.Organisms))

	sp.ComputeAvgFitness()
	if sp.AvgFitness != avg_check {
		t.Error("sp.AvgFitness != avg_check", sp.AvgFitness, avg_check)
	}
}

func TestSpecies_computeMaxFitness(t *testing.T) {
	sp := buildSpeciesWithOrganisms(1)

	sp.ComputeMaxFitness()
	if sp.MaxFitness != 15.0 {
		t.Error("sp.MaxFitness != 15.0", 15.0, sp.MaxFitness)
	}
}

func TestSpecies_findChampion(t *testing.T) {
	sp := buildSpeciesWithOrganisms(1)

	champ := sp.findChampion()
	if champ.Fitness != 15.0 {
		t.Error("champ.Fitness != 15.0", champ.Fitness)
	}

}

func TestSpecies_removeOrganism(t *testing.T) {
	sp := buildSpeciesWithOrganisms(1)

	// test remove
	size := len(sp.Organisms)
	res, err := sp.removeOrganism(sp.Organisms[0])
	if res != true {
		t.Error("res != true", res, err)
	}
	if err != nil {
		t.Error("err != nil", err)
	}
	if size - 1 != len(sp.Organisms) {
		t.Error("size - 1 != len(sp.Organisms)", size - 1, len(sp.Organisms))
	}

	// test fail to remove
	size = len(sp.Organisms)
	gen := &Genome{
		Id:1,
	}
	org := NewOrganism(6.0, gen, 1)
	res, err = sp.removeOrganism(org)
	if res == true {
		t.Error("res == true", res, err)
	}
	if err == nil {
		t.Error("err == nil", res, err)
	}
	if size != len(sp.Organisms) {
		t.Error("size != len(sp.Organisms)", size, len(sp.Organisms))
	}
}

// Tests Species reproduce failure
func TestSpecies_reproduce_fail(t *testing.T) {
	sp := NewSpecies(1)

	sp.ExpectedOffspring = 1

	res, err := sp.reproduce(1, nil, nil, nil)
	if res != false {
		t.Error("res != false")
	}
	if err == nil {
		t.Error("err == nil")
	}
}

// Tests Species reproduce success
func TestSpecies_reproduce(t *testing.T) {
	rand.Seed(42)
	in, out, nmax, n := 3, 2, 15, 3
	recurrent := false
	link_prob := 0.8

	// Configuration
	conf := neat.NeatContext {
		DropOffAge:5,
		SurvivalThresh:0.5,
		AgeSignificance:0.5,
		PopSize:30,
		CompatThreshold:0.6,
	}

	gen := NewGenomeRand(1, in, out, n, nmax, recurrent, link_prob)
	pop, err := NewPopulation(gen, &conf)
	if err != nil {
		t.Error(err)
	}
	if pop == nil {
		t.Error("pop == nil")
	}

	// Stick the Species pointers into a new Species list for sorting
	sorted_species := make([]*Species, len(pop.Species))
	copy(sorted_species, pop.Species)

	// Sort the Species by max original fitness of its first organism
	sort.Sort(ByOrganismOrigFitness(sorted_species))

	pop.Species[0].ExpectedOffspring = 11

	res, err := pop.Species[0].reproduce(1, pop, sorted_species, &conf)
	if !res {
		t.Error("No reproduction", err)
	}
	if err != nil {
		t.Error("err != nil", err)
	}
	after := 0
	for _, sp := range pop.Species {
		after += len(sp.Organisms)
	}

	if after != conf.PopSize + pop.Species[0].ExpectedOffspring {
		t.Error("No new baby was created", after)
	}
}
<|endoftext|>


<commit_before>package httpctx

import (
	"fmt"
	"net/http"
	"net/http/httptest"
	"net/url"
	"testing"
)

func TestShouldSendJSON(t *testing.T) {
	for i, tc := range []struct {
		Accept, Path string
		Want         bool
	}{
		{"", "", false},
		{"", "/api", false},
		{"", "/api/", true},
		{"application/json", "", true},
		{"application/json", "/api/", true},
	} {
		got := shouldSendJson(&http.Request{
			URL:    &url.URL{Path: tc.Path},
			Header: http.Header(map[string][]string{"Accept": {tc.Accept}}),
		})
		if got != tc.Want {
			t.Errorf("%d. got %t, want %t (Path=%q Accept=%q).",
				i, got, tc.Want, tc.Path, tc.Accept)
		}
	}
}

func TestSendError(t *testing.T) {
	for i, tc := range []struct {
		Code int
		Err  error
		Body string
	}{
		{http.StatusInternalServerError, fmt.Errorf("ERR"),
			"ERR\n"},
		{3, HTTPError{Err: nil, Code: 3}, "\n"},
		{3, HTTPError{Err: fmt.Errorf("bad status code"), Code: 3}, "bad status code\n"},
		{500, HTTPErrorCode{Err: nil, StatusCode: "418 I'm not a Teapot"},
			`{"error":{"code":"418 I'm not a Teapot","message":"","status":500}}`},
		{500, HTTPErrorCode{Err: fmt.Errorf("something happened"), StatusCode: "418 I'm not a Teapot"},
			`{"error":{"code":"418 I'm not a Teapot","message":"something happened","status":500}}`},
	} {
		rr := httptest.NewRecorder()
		var hdr http.Header
		if _, ok := tc.Err.(HTTPErrorCode); ok {
			hdr = http.Header(map[string][]string{"Accept": {"application/json"}})
		}
		sendError(rr, &http.Request{URL: &url.URL{}, Header: hdr}, tc.Err)
		if rr.Code != tc.Code {
			t.Errorf("%d. got %d, want %d for %#v.", i, rr.Code, tc.Code, tc.Err)
		}
		got := rr.Body.String()
		if got != tc.Body {
			t.Errorf("%d. got %q, want %q for %v.", i, got, tc.Body, tc.Err)
		}
	}
}

type HTTPErrorCode struct {
	Err        error
	StatusCode string
}

// Code returns the HTTP status code accompanies the error.
func (he HTTPErrorCode) Code() string {
	return he.StatusCode
}
func (he HTTPErrorCode) Error() string {
	if he.Err == nil {
		return ""
	}
	return he.Err.Error()
}

// ErrorWithStatusCode allows errors to provide the HTTP status code to return.
type ErrorWithStatusCode interface {
	StatusCode() int
}

var _ = ErrorWithStatusCode(HTTPError{})
var _ = error(HTTPError{})

type HTTPError struct {
	Err  error
	Code int
}

// StatusCode returns the HTTP status code accompanies the error.
func (he HTTPError) StatusCode() int {
	return he.Code
}
func (he HTTPError) Error() string {
	if he.Err == nil {
		return ""
	}
	return he.Err.Error()
}
<commit_msg>Fix failing test<commit_after>package httpctx

import (
	"fmt"
	"net/http"
	"net/http/httptest"
	"net/url"
	"testing"
)

func TestShouldSendJSON(t *testing.T) {
	for i, tc := range []struct {
		Accept, Path string
		Want         bool
	}{
		{"", "", false},
		{"", "/api", false},
		{"", "/api/", true},
		{"application/json", "", true},
		{"application/json", "/api/", true},
	} {
		got := shouldSendJSON(&http.Request{
			URL:    &url.URL{Path: tc.Path},
			Header: http.Header(map[string][]string{"Accept": {tc.Accept}}),
		})
		if got != tc.Want {
			t.Errorf("%d. got %t, want %t (Path=%q Accept=%q).",
				i, got, tc.Want, tc.Path, tc.Accept)
		}
	}
}

func TestSendError(t *testing.T) {
	for i, tc := range []struct {
		Code int
		Err  error
		Body string
	}{
		{http.StatusInternalServerError, fmt.Errorf("ERR"),
			"ERR\n"},
		{3, HTTPError{Err: nil, Code: 3}, "\n"},
		{3, HTTPError{Err: fmt.Errorf("bad status code"), Code: 3}, "bad status code\n"},
		{500, HTTPErrorCode{Err: nil, StatusCode: "418 I'm not a Teapot"},
			`{"error":{"code":"418 I'm not a Teapot","message":"","status":500}}`},
		{500, HTTPErrorCode{Err: fmt.Errorf("something happened"), StatusCode: "418 I'm not a Teapot"},
			`{"error":{"code":"418 I'm not a Teapot","message":"something happened","status":500}}`},
	} {
		rr := httptest.NewRecorder()
		var hdr http.Header
		if _, ok := tc.Err.(HTTPErrorCode); ok {
			hdr = http.Header(map[string][]string{"Accept": {"application/json"}})
		}
		sendError(rr, &http.Request{URL: &url.URL{}, Header: hdr}, tc.Err)
		if rr.Code != tc.Code {
			t.Errorf("%d. got %d, want %d for %#v.", i, rr.Code, tc.Code, tc.Err)
		}
		got := rr.Body.String()
		if got != tc.Body {
			t.Errorf("%d. got %q, want %q for %v.", i, got, tc.Body, tc.Err)
		}
	}
}

type HTTPErrorCode struct {
	Err        error
	StatusCode string
}

// Code returns the HTTP status code accompanies the error.
func (he HTTPErrorCode) Code() string {
	return he.StatusCode
}
func (he HTTPErrorCode) Error() string {
	if he.Err == nil {
		return ""
	}
	return he.Err.Error()
}

// ErrorWithStatusCode allows errors to provide the HTTP status code to return.
type ErrorWithStatusCode interface {
	StatusCode() int
}

var _ = ErrorWithStatusCode(HTTPError{})
var _ = error(HTTPError{})

type HTTPError struct {
	Err  error
	Code int
}

// StatusCode returns the HTTP status code accompanies the error.
func (he HTTPError) StatusCode() int {
	return he.Code
}
func (he HTTPError) Error() string {
	if he.Err == nil {
		return ""
	}
	return he.Err.Error()
}
<|endoftext|>


<commit_before>// Package websocket implements a websocket based transport for go-libp2p.
package websocket

import (
	"context"
	"crypto/tls"
	"net"
	"net/http"

	"github.com/libp2p/go-libp2p-core/network"
	"github.com/libp2p/go-libp2p-core/peer"
	"github.com/libp2p/go-libp2p-core/transport"

	ws "github.com/gorilla/websocket"
	ma "github.com/multiformats/go-multiaddr"
	mafmt "github.com/multiformats/go-multiaddr-fmt"
	manet "github.com/multiformats/go-multiaddr/net"
)

// WsFmt is multiaddr formatter for WsProtocol
var WsFmt = mafmt.And(mafmt.TCP, mafmt.Base(ma.P_WS))

// This is _not_ WsFmt because we want the transport to stick to dialing fully
// resolved addresses.
var dialMatcher = mafmt.And(mafmt.IP, mafmt.Base(ma.P_TCP), mafmt.Or(mafmt.Base(ma.P_WS), mafmt.Base(ma.P_WSS)))

func init() {
	manet.RegisterFromNetAddr(ParseWebsocketNetAddr, "websocket")
	manet.RegisterToNetAddr(ConvertWebsocketMultiaddrToNetAddr, "ws")
	manet.RegisterToNetAddr(ConvertWebsocketMultiaddrToNetAddr, "wss")
}

// Default gorilla upgrader
var upgrader = ws.Upgrader{
	// Allow requests from *all* origins.
	CheckOrigin: func(r *http.Request) bool {
		return true
	},
}

type Option func(*WebsocketTransport) error

// WithTLSClientConfig sets a TLS client configuration on the WebSocket Dialer. Only
// relevant for non-browser usages.
//
// Some useful use cases include setting InsecureSkipVerify to `true`, or
// setting user-defined trusted CA certificates.
func WithTLSClientConfig(c *tls.Config) Option {
	return func(t *WebsocketTransport) error {
		t.tlsClientConf = c
		return nil
	}
}

// WebsocketTransport is the actual go-libp2p transport
type WebsocketTransport struct {
	upgrader transport.Upgrader
	rcmgr    network.ResourceManager

	tlsClientConf *tls.Config
}

var _ transport.Transport = (*WebsocketTransport)(nil)

func New(u transport.Upgrader, rcmgr network.ResourceManager, opts ...Option) (*WebsocketTransport, error) {
	if rcmgr == nil {
		rcmgr = network.NullResourceManager
	}
	t := &WebsocketTransport{
		upgrader: u,
		rcmgr:    rcmgr,
	}
	for _, opt := range opts {
		if err := opt(t); err != nil {
			return nil, err
		}
	}
	return t, nil
}

func (t *WebsocketTransport) CanDial(a ma.Multiaddr) bool {
	return dialMatcher.Matches(a)
}

func (t *WebsocketTransport) Protocols() []int {
	return []int{ma.P_WS, ma.P_WSS}
}

func (t *WebsocketTransport) Proxy() bool {
	return false
}

func (t *WebsocketTransport) Dial(ctx context.Context, raddr ma.Multiaddr, p peer.ID) (transport.CapableConn, error) {
	connScope, err := t.rcmgr.OpenConnection(network.DirOutbound, true)
	if err != nil {
		return nil, err
	}
	macon, err := t.maDial(ctx, raddr)
	if err != nil {
		connScope.Done()
		return nil, err
	}
	return t.upgrader.Upgrade(ctx, t, macon, network.DirOutbound, p, connScope)
}

func (t *WebsocketTransport) maDial(ctx context.Context, raddr ma.Multiaddr) (manet.Conn, error) {
	wsurl, err := parseMultiaddr(raddr)
	if err != nil {
		return nil, err
	}

	wscon, _, err := ws.DefaultDialer.Dial(wsurl.String(), nil)
	if err != nil {
		return nil, err
	}

	mnc, err := manet.WrapNetConn(NewConn(wscon, wsurl.Scheme == "wss"))
	if err != nil {
		wscon.Close()
		return nil, err
	}
	return mnc, nil
}

func (t *WebsocketTransport) maListen(a ma.Multiaddr) (manet.Listener, error) {
	lnet, lnaddr, err := manet.DialArgs(a)
	if err != nil {
		return nil, err
	}
	nl, err := net.Listen(lnet, lnaddr)
	if err != nil {
		return nil, err
	}
	malist, err := t.wrapListener(nl)
	if err != nil {
		nl.Close()
		return nil, err
	}
	go malist.serve()

	return malist, nil
}

func (t *WebsocketTransport) Listen(a ma.Multiaddr) (transport.Listener, error) {
	malist, err := t.maListen(a)
	if err != nil {
		return nil, err
	}
	return t.upgrader.UpgradeListener(t, malist), nil
}

func (t *WebsocketTransport) wrapListener(l net.Listener) (*listener, error) {
	laddr, err := manet.FromNetAddr(l.Addr())
	if err != nil {
		return nil, err
	}
	wsma, err := ma.NewMultiaddr("/ws")
	if err != nil {
		return nil, err
	}
	laddr = laddr.Encapsulate(wsma)

	return &listener{
		laddr:    laddr,
		Listener: l,
		incoming: make(chan *Conn),
		closed:   make(chan struct{}),
	}, nil
}
<commit_msg>only initialize the ws multiaddr component once<commit_after>// Package websocket implements a websocket based transport for go-libp2p.
package websocket

import (
	"context"
	"crypto/tls"
	"net"
	"net/http"

	"github.com/libp2p/go-libp2p-core/network"
	"github.com/libp2p/go-libp2p-core/peer"
	"github.com/libp2p/go-libp2p-core/transport"

	ws "github.com/gorilla/websocket"
	ma "github.com/multiformats/go-multiaddr"
	mafmt "github.com/multiformats/go-multiaddr-fmt"
	manet "github.com/multiformats/go-multiaddr/net"
)

// WsFmt is multiaddr formatter for WsProtocol
var WsFmt = mafmt.And(mafmt.TCP, mafmt.Base(ma.P_WS))

var wsma = ma.StringCast("/ws")

// This is _not_ WsFmt because we want the transport to stick to dialing fully
// resolved addresses.
var dialMatcher = mafmt.And(mafmt.IP, mafmt.Base(ma.P_TCP), mafmt.Or(mafmt.Base(ma.P_WS), mafmt.Base(ma.P_WSS)))

func init() {
	manet.RegisterFromNetAddr(ParseWebsocketNetAddr, "websocket")
	manet.RegisterToNetAddr(ConvertWebsocketMultiaddrToNetAddr, "ws")
	manet.RegisterToNetAddr(ConvertWebsocketMultiaddrToNetAddr, "wss")
}

// Default gorilla upgrader
var upgrader = ws.Upgrader{
	// Allow requests from *all* origins.
	CheckOrigin: func(r *http.Request) bool {
		return true
	},
}

type Option func(*WebsocketTransport) error

// WithTLSClientConfig sets a TLS client configuration on the WebSocket Dialer. Only
// relevant for non-browser usages.
//
// Some useful use cases include setting InsecureSkipVerify to `true`, or
// setting user-defined trusted CA certificates.
func WithTLSClientConfig(c *tls.Config) Option {
	return func(t *WebsocketTransport) error {
		t.tlsClientConf = c
		return nil
	}
}

// WebsocketTransport is the actual go-libp2p transport
type WebsocketTransport struct {
	upgrader transport.Upgrader
	rcmgr    network.ResourceManager

	tlsClientConf *tls.Config
}

var _ transport.Transport = (*WebsocketTransport)(nil)

func New(u transport.Upgrader, rcmgr network.ResourceManager, opts ...Option) (*WebsocketTransport, error) {
	if rcmgr == nil {
		rcmgr = network.NullResourceManager
	}
	t := &WebsocketTransport{
		upgrader: u,
		rcmgr:    rcmgr,
	}
	for _, opt := range opts {
		if err := opt(t); err != nil {
			return nil, err
		}
	}
	return t, nil
}

func (t *WebsocketTransport) CanDial(a ma.Multiaddr) bool {
	return dialMatcher.Matches(a)
}

func (t *WebsocketTransport) Protocols() []int {
	return []int{ma.P_WS, ma.P_WSS}
}

func (t *WebsocketTransport) Proxy() bool {
	return false
}

func (t *WebsocketTransport) Dial(ctx context.Context, raddr ma.Multiaddr, p peer.ID) (transport.CapableConn, error) {
	connScope, err := t.rcmgr.OpenConnection(network.DirOutbound, true)
	if err != nil {
		return nil, err
	}
	macon, err := t.maDial(ctx, raddr)
	if err != nil {
		connScope.Done()
		return nil, err
	}
	return t.upgrader.Upgrade(ctx, t, macon, network.DirOutbound, p, connScope)
}

func (t *WebsocketTransport) maDial(ctx context.Context, raddr ma.Multiaddr) (manet.Conn, error) {
	wsurl, err := parseMultiaddr(raddr)
	if err != nil {
		return nil, err
	}

	wscon, _, err := ws.DefaultDialer.Dial(wsurl.String(), nil)
	if err != nil {
		return nil, err
	}

	mnc, err := manet.WrapNetConn(NewConn(wscon, wsurl.Scheme == "wss"))
	if err != nil {
		wscon.Close()
		return nil, err
	}
	return mnc, nil
}

func (t *WebsocketTransport) maListen(a ma.Multiaddr) (manet.Listener, error) {
	lnet, lnaddr, err := manet.DialArgs(a)
	if err != nil {
		return nil, err
	}
	nl, err := net.Listen(lnet, lnaddr)
	if err != nil {
		return nil, err
	}
	malist, err := t.wrapListener(nl)
	if err != nil {
		nl.Close()
		return nil, err
	}
	go malist.serve()

	return malist, nil
}

func (t *WebsocketTransport) Listen(a ma.Multiaddr) (transport.Listener, error) {
	malist, err := t.maListen(a)
	if err != nil {
		return nil, err
	}
	return t.upgrader.UpgradeListener(t, malist), nil
}

func (t *WebsocketTransport) wrapListener(l net.Listener) (*listener, error) {
	laddr, err := manet.FromNetAddr(l.Addr())
	if err != nil {
		return nil, err
	}
	return &listener{
		laddr:    laddr.Encapsulate(wsma),
		Listener: l,
		incoming: make(chan *Conn),
		closed:   make(chan struct{}),
	}, nil
}
<|endoftext|>


<commit_before>package structs

import (
	"net"
	"reflect"
	"testing"
)

func TestNetworkIndex_Overcommitted(t *testing.T) {
	idx := NewNetworkIndex()

	// Consume some network
	reserved := &NetworkResource{
		Device:        "eth0",
		IP:            "192.168.0.100",
		MBits:         505,
		ReservedPorts: []Port{{"one", 8000}, {"two", 9000}},
	}
	collide := idx.AddReserved(reserved)
	if collide {
		t.Fatalf("bad")
	}
	if !idx.Overcommitted() {
		t.Fatalf("have no resources")
	}

	// Add resources
	n := &Node{
		Resources: &Resources{
			Networks: []*NetworkResource{
				&NetworkResource{
					Device: "eth0",
					CIDR:   "192.168.0.100/32",
					MBits:  1000,
				},
			},
		},
	}
	idx.SetNode(n)
	if idx.Overcommitted() {
		t.Fatalf("have resources")
	}

	// Double up our ussage
	idx.AddReserved(reserved)
	if !idx.Overcommitted() {
		t.Fatalf("should be overcommitted")
	}
}

func TestNetworkIndex_SetNode(t *testing.T) {
	idx := NewNetworkIndex()
	n := &Node{
		Resources: &Resources{
			Networks: []*NetworkResource{
				&NetworkResource{
					Device: "eth0",
					CIDR:   "192.168.0.100/32",
					MBits:  1000,
				},
			},
		},
		Reserved: &Resources{
			Networks: []*NetworkResource{
				&NetworkResource{
					Device:        "eth0",
					IP:            "192.168.0.100",
					ReservedPorts: []Port{{"ssh", 22}},
					MBits:         1,
				},
			},
		},
	}
	collide := idx.SetNode(n)
	if collide {
		t.Fatalf("bad")
	}

	if len(idx.AvailNetworks) != 1 {
		t.Fatalf("Bad")
	}
	if idx.AvailBandwidth["eth0"] != 1000 {
		t.Fatalf("Bad")
	}
	if idx.UsedBandwidth["eth0"] != 1 {
		t.Fatalf("Bad")
	}
	if !idx.UsedPorts["192.168.0.100"].Check(22) {
		t.Fatalf("Bad")
	}
}

func TestNetworkIndex_AddAllocs(t *testing.T) {
	idx := NewNetworkIndex()
	allocs := []*Allocation{
		&Allocation{
			TaskResources: map[string]*Resources{
				"web": &Resources{
					Networks: []*NetworkResource{
						&NetworkResource{
							Device:        "eth0",
							IP:            "192.168.0.100",
							MBits:         20,
							ReservedPorts: []Port{{"one", 8000}, {"two", 9000}},
						},
					},
				},
			},
		},
		&Allocation{
			TaskResources: map[string]*Resources{
				"api": &Resources{
					Networks: []*NetworkResource{
						&NetworkResource{
							Device:        "eth0",
							IP:            "192.168.0.100",
							MBits:         50,
							ReservedPorts: []Port{{"one", 10000}},
						},
					},
				},
			},
		},
	}
	collide := idx.AddAllocs(allocs)
	if collide {
		t.Fatalf("bad")
	}

	if idx.UsedBandwidth["eth0"] != 70 {
		t.Fatalf("Bad")
	}
	if !idx.UsedPorts["192.168.0.100"].Check(8000) {
		t.Fatalf("Bad")
	}
	if !idx.UsedPorts["192.168.0.100"].Check(9000) {
		t.Fatalf("Bad")
	}
	if !idx.UsedPorts["192.168.0.100"].Check(10000) {
		t.Fatalf("Bad")
	}
}

func TestNetworkIndex_AddReserved(t *testing.T) {
	idx := NewNetworkIndex()

	reserved := &NetworkResource{
		Device:        "eth0",
		IP:            "192.168.0.100",
		MBits:         20,
		ReservedPorts: []Port{{"one", 8000}, {"two", 9000}},
	}
	collide := idx.AddReserved(reserved)
	if collide {
		t.Fatalf("bad")
	}

	if idx.UsedBandwidth["eth0"] != 20 {
		t.Fatalf("Bad")
	}
	if !idx.UsedPorts["192.168.0.100"].Check(8000) {
		t.Fatalf("Bad")
	}
	if !idx.UsedPorts["192.168.0.100"].Check(9000) {
		t.Fatalf("Bad")
	}

	// Try to reserve the same network
	collide = idx.AddReserved(reserved)
	if !collide {
		t.Fatalf("bad")
	}
}

func TestNetworkIndex_yieldIP(t *testing.T) {
	idx := NewNetworkIndex()
	n := &Node{
		Resources: &Resources{
			Networks: []*NetworkResource{
				&NetworkResource{
					Device: "eth0",
					CIDR:   "192.168.0.100/30",
					MBits:  1000,
				},
			},
		},
		Reserved: &Resources{
			Networks: []*NetworkResource{
				&NetworkResource{
					Device:        "eth0",
					IP:            "192.168.0.100",
					ReservedPorts: []Port{{"ssh", 22}},
					MBits:         1,
				},
			},
		},
	}
	idx.SetNode(n)

	var out []string
	idx.yieldIP(func(n *NetworkResource, ip net.IP) (stop bool) {
		out = append(out, ip.String())
		return
	})

	expect := []string{"192.168.0.100", "192.168.0.101",
		"192.168.0.102", "192.168.0.103"}
	if !reflect.DeepEqual(out, expect) {
		t.Fatalf("bad: %v", out)
	}
}

func TestNetworkIndex_AssignNetwork(t *testing.T) {
	idx := NewNetworkIndex()
	n := &Node{
		Resources: &Resources{
			Networks: []*NetworkResource{
				&NetworkResource{
					Device: "eth0",
					CIDR:   "192.168.0.100/30",
					MBits:  1000,
				},
			},
		},
		Reserved: &Resources{
			Networks: []*NetworkResource{
				&NetworkResource{
					Device:        "eth0",
					IP:            "192.168.0.100",
					ReservedPorts: []Port{{"ssh", 22}},
					MBits:         1,
				},
			},
		},
	}
	idx.SetNode(n)

	allocs := []*Allocation{
		&Allocation{
			TaskResources: map[string]*Resources{
				"web": &Resources{
					Networks: []*NetworkResource{
						&NetworkResource{
							Device:        "eth0",
							IP:            "192.168.0.100",
							MBits:         20,
							ReservedPorts: []Port{{"one", 8000}, {"two", 9000}},
						},
					},
				},
			},
		},
		&Allocation{
			TaskResources: map[string]*Resources{
				"api": &Resources{
					Networks: []*NetworkResource{
						&NetworkResource{
							Device:        "eth0",
							IP:            "192.168.0.100",
							MBits:         50,
							ReservedPorts: []Port{{"main", 10000}},
						},
					},
				},
			},
		},
	}
	idx.AddAllocs(allocs)

	// Ask for a reserved port
	ask := &NetworkResource{
		ReservedPorts: []Port{{"main", 8000}},
	}
	offer, err := idx.AssignNetwork(ask)
	if err != nil {
		t.Fatalf("err: %v", err)
	}
	if offer == nil {
		t.Fatalf("bad")
	}
	if offer.IP != "192.168.0.101" {
		t.Fatalf("bad: %#v", offer)
	}
	rp := Port{"main", 8000}
	if len(offer.ReservedPorts) != 1 || offer.ReservedPorts[0] != rp {
		t.Fatalf("bad: %#v", offer)
	}

	// Ask for dynamic ports
	ask = &NetworkResource{
		DynamicPorts: []Port{{"http", 0}, {"https", 0}, {"admin", 0}},
	}
	offer, err = idx.AssignNetwork(ask)
	if err != nil {
		t.Fatalf("err: %v", err)
	}
	if offer == nil {
		t.Fatalf("bad")
	}
	if offer.IP != "192.168.0.100" {
		t.Fatalf("bad: %#v", offer)
	}
	if len(offer.DynamicPorts) != 3 {
		t.Fatalf("There should be three dynamic ports")
	}
	for _, port := range offer.DynamicPorts {
		if port.Value == 0 {
			t.Fatalf("Dynamic Port: %v should have been assigned a host port", port.Label)
		}
	}

	// Ask for reserved + dynamic ports
	ask = &NetworkResource{
		ReservedPorts: []Port{{"main", 2345}},
		DynamicPorts:  []Port{{"http", 0}, {"https", 0}, {"admin", 0}},
	}
	offer, err = idx.AssignNetwork(ask)
	if err != nil {
		t.Fatalf("err: %v", err)
	}
	if offer == nil {
		t.Fatalf("bad")
	}
	if offer.IP != "192.168.0.100" {
		t.Fatalf("bad: %#v", offer)
	}

	rp = Port{"main", 2345}
	if len(offer.ReservedPorts) != 1 || offer.ReservedPorts[0] != rp {
		t.Fatalf("bad: %#v", offer)
	}

	// Ask for too much bandwidth
	ask = &NetworkResource{
		MBits: 1000,
	}
	offer, err = idx.AssignNetwork(ask)
	if err.Error() != "bandwidth exceeded" {
		t.Fatalf("err: %v", err)
	}
	if offer != nil {
		t.Fatalf("bad")
	}
}

// This test ensures that even with a small domain of available ports we are
// able to make a dynamic port allocation.
func TestNetworkIndex_AssignNetwork_Dynamic_Contention(t *testing.T) {

	// Create a node that only has one free port
	idx := NewNetworkIndex()
	n := &Node{
		Resources: &Resources{
			Networks: []*NetworkResource{
				&NetworkResource{
					Device: "eth0",
					CIDR:   "192.168.0.100/32",
					MBits:  1000,
				},
			},
		},
		Reserved: &Resources{
			Networks: []*NetworkResource{
				&NetworkResource{
					Device: "eth0",
					IP:     "192.168.0.100",
					MBits:  1,
				},
			},
		},
	}
	for i := MinDynamicPort; i < MaxDynamicPort-1; i++ {
		n.Reserved.Networks[0].ReservedPorts = append(n.Reserved.Networks[0].ReservedPorts, Port{Value: i})
	}

	idx.SetNode(n)

	// Ask for dynamic ports
	ask := &NetworkResource{
		DynamicPorts: []Port{{"http", 0}},
	}
	offer, err := idx.AssignNetwork(ask)
	if err != nil {
		t.Fatalf("err: %v", err)
	}
	if offer == nil {
		t.Fatalf("bad")
	}
	if offer.IP != "192.168.0.100" {
		t.Fatalf("bad: %#v", offer)
	}
	if len(offer.DynamicPorts) != 1 {
		t.Fatalf("There should be three dynamic ports")
	}
	if p := offer.DynamicPorts[0].Value; p == MaxDynamicPort {
		t.Fatalf("Dynamic Port: should have been assigned %d; got %d", p, MaxDynamicPort)
	}
}

func TestIntContains(t *testing.T) {
	l := []int{1, 2, 10, 20}
	if isPortReserved(l, 50) {
		t.Fatalf("bad")
	}
	if !isPortReserved(l, 20) {
		t.Fatalf("bad")
	}
	if !isPortReserved(l, 1) {
		t.Fatalf("bad")
	}
}
<commit_msg>Fix network dynamic port test<commit_after>package structs

import (
	"net"
	"reflect"
	"testing"
)

func TestNetworkIndex_Overcommitted(t *testing.T) {
	idx := NewNetworkIndex()

	// Consume some network
	reserved := &NetworkResource{
		Device:        "eth0",
		IP:            "192.168.0.100",
		MBits:         505,
		ReservedPorts: []Port{{"one", 8000}, {"two", 9000}},
	}
	collide := idx.AddReserved(reserved)
	if collide {
		t.Fatalf("bad")
	}
	if !idx.Overcommitted() {
		t.Fatalf("have no resources")
	}

	// Add resources
	n := &Node{
		Resources: &Resources{
			Networks: []*NetworkResource{
				&NetworkResource{
					Device: "eth0",
					CIDR:   "192.168.0.100/32",
					MBits:  1000,
				},
			},
		},
	}
	idx.SetNode(n)
	if idx.Overcommitted() {
		t.Fatalf("have resources")
	}

	// Double up our ussage
	idx.AddReserved(reserved)
	if !idx.Overcommitted() {
		t.Fatalf("should be overcommitted")
	}
}

func TestNetworkIndex_SetNode(t *testing.T) {
	idx := NewNetworkIndex()
	n := &Node{
		Resources: &Resources{
			Networks: []*NetworkResource{
				&NetworkResource{
					Device: "eth0",
					CIDR:   "192.168.0.100/32",
					MBits:  1000,
				},
			},
		},
		Reserved: &Resources{
			Networks: []*NetworkResource{
				&NetworkResource{
					Device:        "eth0",
					IP:            "192.168.0.100",
					ReservedPorts: []Port{{"ssh", 22}},
					MBits:         1,
				},
			},
		},
	}
	collide := idx.SetNode(n)
	if collide {
		t.Fatalf("bad")
	}

	if len(idx.AvailNetworks) != 1 {
		t.Fatalf("Bad")
	}
	if idx.AvailBandwidth["eth0"] != 1000 {
		t.Fatalf("Bad")
	}
	if idx.UsedBandwidth["eth0"] != 1 {
		t.Fatalf("Bad")
	}
	if !idx.UsedPorts["192.168.0.100"].Check(22) {
		t.Fatalf("Bad")
	}
}

func TestNetworkIndex_AddAllocs(t *testing.T) {
	idx := NewNetworkIndex()
	allocs := []*Allocation{
		&Allocation{
			TaskResources: map[string]*Resources{
				"web": &Resources{
					Networks: []*NetworkResource{
						&NetworkResource{
							Device:        "eth0",
							IP:            "192.168.0.100",
							MBits:         20,
							ReservedPorts: []Port{{"one", 8000}, {"two", 9000}},
						},
					},
				},
			},
		},
		&Allocation{
			TaskResources: map[string]*Resources{
				"api": &Resources{
					Networks: []*NetworkResource{
						&NetworkResource{
							Device:        "eth0",
							IP:            "192.168.0.100",
							MBits:         50,
							ReservedPorts: []Port{{"one", 10000}},
						},
					},
				},
			},
		},
	}
	collide := idx.AddAllocs(allocs)
	if collide {
		t.Fatalf("bad")
	}

	if idx.UsedBandwidth["eth0"] != 70 {
		t.Fatalf("Bad")
	}
	if !idx.UsedPorts["192.168.0.100"].Check(8000) {
		t.Fatalf("Bad")
	}
	if !idx.UsedPorts["192.168.0.100"].Check(9000) {
		t.Fatalf("Bad")
	}
	if !idx.UsedPorts["192.168.0.100"].Check(10000) {
		t.Fatalf("Bad")
	}
}

func TestNetworkIndex_AddReserved(t *testing.T) {
	idx := NewNetworkIndex()

	reserved := &NetworkResource{
		Device:        "eth0",
		IP:            "192.168.0.100",
		MBits:         20,
		ReservedPorts: []Port{{"one", 8000}, {"two", 9000}},
	}
	collide := idx.AddReserved(reserved)
	if collide {
		t.Fatalf("bad")
	}

	if idx.UsedBandwidth["eth0"] != 20 {
		t.Fatalf("Bad")
	}
	if !idx.UsedPorts["192.168.0.100"].Check(8000) {
		t.Fatalf("Bad")
	}
	if !idx.UsedPorts["192.168.0.100"].Check(9000) {
		t.Fatalf("Bad")
	}

	// Try to reserve the same network
	collide = idx.AddReserved(reserved)
	if !collide {
		t.Fatalf("bad")
	}
}

func TestNetworkIndex_yieldIP(t *testing.T) {
	idx := NewNetworkIndex()
	n := &Node{
		Resources: &Resources{
			Networks: []*NetworkResource{
				&NetworkResource{
					Device: "eth0",
					CIDR:   "192.168.0.100/30",
					MBits:  1000,
				},
			},
		},
		Reserved: &Resources{
			Networks: []*NetworkResource{
				&NetworkResource{
					Device:        "eth0",
					IP:            "192.168.0.100",
					ReservedPorts: []Port{{"ssh", 22}},
					MBits:         1,
				},
			},
		},
	}
	idx.SetNode(n)

	var out []string
	idx.yieldIP(func(n *NetworkResource, ip net.IP) (stop bool) {
		out = append(out, ip.String())
		return
	})

	expect := []string{"192.168.0.100", "192.168.0.101",
		"192.168.0.102", "192.168.0.103"}
	if !reflect.DeepEqual(out, expect) {
		t.Fatalf("bad: %v", out)
	}
}

func TestNetworkIndex_AssignNetwork(t *testing.T) {
	idx := NewNetworkIndex()
	n := &Node{
		Resources: &Resources{
			Networks: []*NetworkResource{
				&NetworkResource{
					Device: "eth0",
					CIDR:   "192.168.0.100/30",
					MBits:  1000,
				},
			},
		},
		Reserved: &Resources{
			Networks: []*NetworkResource{
				&NetworkResource{
					Device:        "eth0",
					IP:            "192.168.0.100",
					ReservedPorts: []Port{{"ssh", 22}},
					MBits:         1,
				},
			},
		},
	}
	idx.SetNode(n)

	allocs := []*Allocation{
		&Allocation{
			TaskResources: map[string]*Resources{
				"web": &Resources{
					Networks: []*NetworkResource{
						&NetworkResource{
							Device:        "eth0",
							IP:            "192.168.0.100",
							MBits:         20,
							ReservedPorts: []Port{{"one", 8000}, {"two", 9000}},
						},
					},
				},
			},
		},
		&Allocation{
			TaskResources: map[string]*Resources{
				"api": &Resources{
					Networks: []*NetworkResource{
						&NetworkResource{
							Device:        "eth0",
							IP:            "192.168.0.100",
							MBits:         50,
							ReservedPorts: []Port{{"main", 10000}},
						},
					},
				},
			},
		},
	}
	idx.AddAllocs(allocs)

	// Ask for a reserved port
	ask := &NetworkResource{
		ReservedPorts: []Port{{"main", 8000}},
	}
	offer, err := idx.AssignNetwork(ask)
	if err != nil {
		t.Fatalf("err: %v", err)
	}
	if offer == nil {
		t.Fatalf("bad")
	}
	if offer.IP != "192.168.0.101" {
		t.Fatalf("bad: %#v", offer)
	}
	rp := Port{"main", 8000}
	if len(offer.ReservedPorts) != 1 || offer.ReservedPorts[0] != rp {
		t.Fatalf("bad: %#v", offer)
	}

	// Ask for dynamic ports
	ask = &NetworkResource{
		DynamicPorts: []Port{{"http", 0}, {"https", 0}, {"admin", 0}},
	}
	offer, err = idx.AssignNetwork(ask)
	if err != nil {
		t.Fatalf("err: %v", err)
	}
	if offer == nil {
		t.Fatalf("bad")
	}
	if offer.IP != "192.168.0.100" {
		t.Fatalf("bad: %#v", offer)
	}
	if len(offer.DynamicPorts) != 3 {
		t.Fatalf("There should be three dynamic ports")
	}
	for _, port := range offer.DynamicPorts {
		if port.Value == 0 {
			t.Fatalf("Dynamic Port: %v should have been assigned a host port", port.Label)
		}
	}

	// Ask for reserved + dynamic ports
	ask = &NetworkResource{
		ReservedPorts: []Port{{"main", 2345}},
		DynamicPorts:  []Port{{"http", 0}, {"https", 0}, {"admin", 0}},
	}
	offer, err = idx.AssignNetwork(ask)
	if err != nil {
		t.Fatalf("err: %v", err)
	}
	if offer == nil {
		t.Fatalf("bad")
	}
	if offer.IP != "192.168.0.100" {
		t.Fatalf("bad: %#v", offer)
	}

	rp = Port{"main", 2345}
	if len(offer.ReservedPorts) != 1 || offer.ReservedPorts[0] != rp {
		t.Fatalf("bad: %#v", offer)
	}

	// Ask for too much bandwidth
	ask = &NetworkResource{
		MBits: 1000,
	}
	offer, err = idx.AssignNetwork(ask)
	if err.Error() != "bandwidth exceeded" {
		t.Fatalf("err: %v", err)
	}
	if offer != nil {
		t.Fatalf("bad")
	}
}

// This test ensures that even with a small domain of available ports we are
// able to make a dynamic port allocation.
func TestNetworkIndex_AssignNetwork_Dynamic_Contention(t *testing.T) {

	// Create a node that only has one free port
	idx := NewNetworkIndex()
	n := &Node{
		Resources: &Resources{
			Networks: []*NetworkResource{
				&NetworkResource{
					Device: "eth0",
					CIDR:   "192.168.0.100/32",
					MBits:  1000,
				},
			},
		},
		Reserved: &Resources{
			Networks: []*NetworkResource{
				&NetworkResource{
					Device: "eth0",
					IP:     "192.168.0.100",
					MBits:  1,
				},
			},
		},
	}
	for i := MinDynamicPort; i < MaxDynamicPort-1; i++ {
		n.Reserved.Networks[0].ReservedPorts = append(n.Reserved.Networks[0].ReservedPorts, Port{Value: i})
	}

	idx.SetNode(n)

	// Ask for dynamic ports
	ask := &NetworkResource{
		DynamicPorts: []Port{{"http", 0}},
	}
	offer, err := idx.AssignNetwork(ask)
	if err != nil {
		t.Fatalf("err: %v", err)
	}
	if offer == nil {
		t.Fatalf("bad")
	}
	if offer.IP != "192.168.0.100" {
		t.Fatalf("bad: %#v", offer)
	}
	if len(offer.DynamicPorts) != 1 {
		t.Fatalf("There should be three dynamic ports")
	}
	if p := offer.DynamicPorts[0].Value; p != MaxDynamicPort {
		t.Fatalf("Dynamic Port: should have been assigned %d; got %d", p, MaxDynamicPort)
	}
}

func TestIntContains(t *testing.T) {
	l := []int{1, 2, 10, 20}
	if isPortReserved(l, 50) {
		t.Fatalf("bad")
	}
	if !isPortReserved(l, 20) {
		t.Fatalf("bad")
	}
	if !isPortReserved(l, 1) {
		t.Fatalf("bad")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"sort"
	"sync"
	"time"

	"github.com/dgryski/go-tsz"
	"github.com/gocql/gocql"
)

// write aggregated data to cassandra.

const month = 60 * 60 * 24 * 28

const keyspace_schema = `CREATE KEYSPACE IF NOT EXISTS raintank WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}  AND durable_writes = true`
const table_schema = `CREATE TABLE IF NOT EXISTS raintank.metric (
    key ascii,
    ts int,
    data blob,
    PRIMARY KEY (key, ts)
) WITH COMPACT STORAGE
    AND CLUSTERING ORDER BY (ts DESC)`

/*
https://godoc.org/github.com/gocql/gocql#Session
Session is the interface used by users to interact with the database.
It's safe for concurrent use by multiple goroutines and a typical usage scenario is to have one global session
object to interact with the whole Cassandra cluster.
*/
var cSession *gocql.Session

func InitCassandra() error {
	cluster := gocql.NewCluster(cassandraAddrs...)
	cluster.Consistency = gocql.One
	var err error
	tmpSession, err := cluster.CreateSession()
	if err != nil {
		return err
	}
	// ensure the keyspace and table exist.
	err = tmpSession.Query(keyspace_schema).Exec()
	if err != nil {
		return err
	}
	err = tmpSession.Query(table_schema).Exec()
	if err != nil {
		return err
	}
	tmpSession.Close()
	cluster.Keyspace = "raintank"
	cSession, err = cluster.CreateSession()
	return err
}

// Insert metric into Cassandra.
//
// key: is the metric_id
// ts: is the start of the aggregated time range.
// data: is the payload as bytes.
func InsertMetric(key string, t0 uint32, data []byte, ttl int) error {
	// for unit tests
	if cSession == nil {
		return nil
	}
	query := fmt.Sprintf("INSERT INTO metric (key, ts, data) values(?,?,?) USING TTL %d", ttl)
	row_key := fmt.Sprintf("%s_%d", key, t0/month) // "month number" based on unix timestamp (rounded down)
	pre := time.Now()
	ret := cSession.Query(query, row_key, t0, data).Exec()
	cassandraPutDuration.Value(time.Now().Sub(pre))
	return ret
}

type outcome struct {
	mark uint32
	i    *gocql.Iter
}
type asc []outcome

func (o asc) Len() int           { return len(o) }
func (o asc) Swap(i, j int)      { o[i], o[j] = o[j], o[i] }
func (o asc) Less(i, j int) bool { return o[i].mark < o[j].mark }

// Basic search of cassandra.
// start inclusive, end exclusive
func searchCassandra(key string, start, end uint32) ([]*tsz.Iter, error) {
	if start > end {
		panic(fmt.Sprintf("searchCassandra start %d > end %d", start, end))
	}

	results := make(chan outcome)
	wg := &sync.WaitGroup{}

	query := func(mark uint32, q string, p ...interface{}) {
		wg.Add(1)
		go func() {
			//		if len(p) == 3 {
			//		log.Println("querying cassandra for", q, p[0], TS(p[1]), TS(p[2]))
			//		} else if len(p) == 2 {
			//		log.Println("querying cassandra for", q, p[0], TS(p[1]))
			//		} else {
			//		log.Println("querying cassandra for", q, p)
			//		}
			results <- outcome{mark, cSession.Query(q, p...).Iter()}
			wg.Done()
		}()
	}

	start_month := start - (start % month)       // starting row has to be at, or before, requested start
	end_month := (end - 1) - ((end - 1) % month) // ending row has to be include the last point we might need

	pre := time.Now()
	if start_month == end_month {
		// we need a selection of the row between startTs and endTs
		row_key := fmt.Sprintf("%s_%d", key, start_month/month)
		query(start_month, "SELECT data FROM metric WHERE key = ? AND ts >= ? AND ts < ? ORDER BY ts ASC", row_key, start, end)
	} else {
		// get row_keys for each row we need to query.
		for mark := start_month; mark <= end_month; mark += month {
			row_key := fmt.Sprintf("%s_%d", key, mark/month)
			if mark == start_month {
				// we want from startTs to the end of the row.
				query(mark, "SELECT data FROM metric WHERE key = ? AND ts >= ? ORDER BY ts ASC", row_key, start)
			} else if mark == end_month {
				// we want from start of the row till the endTs.
				query(mark, "SELECT data FROM metric WHERE key = ? AND ts < ? ORDER BY ts ASC", row_key, end)
			} else {
				// we want all columns
				query(mark, "SELECT data FROM metric WHERE key = ? ORDER BY ts ASC", row_key)
			}
		}
	}
	outcomes := make([]outcome, 0)
	iters := make([]*tsz.Iter, 0)
	go func() {
		wg.Wait()
		cassandraGetDuration.Value(time.Now().Sub(pre))
		close(results)
	}()
	for o := range results {
		outcomes = append(outcomes, o)
	}
	sort.Sort(asc(outcomes))

	var b []byte
	for _, outcome := range outcomes {
		chunks := int64(0)
		for outcome.i.Scan(&b) {
			chunks += 1
			chunkSizeAtLoad.Value(int64(len(b)))
			iter, err := tsz.NewIterator(b)
			if err != nil {
				log.Fatal(err)
			}
			iters = append(iters, iter)
		}
		cassandraChunksPerRow.Value(chunks)
		err := outcome.i.Close()
		if err != nil {
			log.Println("ERROR:", err)
		}
	}
	cassandraRowsPerResponse.Value(int64(len(outcomes)))
	//log.Println(len(outcomes), "outcomes, cassandra results", len(iters))
	return iters, nil
}
<commit_msg>add default table properties.<commit_after>package main

import (
	"fmt"
	"log"
	"sort"
	"sync"
	"time"

	"github.com/dgryski/go-tsz"
	"github.com/gocql/gocql"
)

// write aggregated data to cassandra.

const month = 60 * 60 * 24 * 28

const keyspace_schema = `CREATE KEYSPACE IF NOT EXISTS raintank WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}  AND durable_writes = true`
const table_schema = `CREATE TABLE IF NOT EXISTS raintank.metric (
    key ascii,
    ts int,
    data blob,
    PRIMARY KEY (key, ts)
) WITH COMPACT STORAGE
    AND CLUSTERING ORDER BY (ts DESC)
    AND compaction = {'class': 'org.apache.cassandra.db.compaction.DateTieredCompactionStrategy'}
    AND compression = {'sstable_compression': 'org.apache.cassandra.io.compress.LZ4Compressor'}
    AND read_repair_chance = 0.0
    AND dclocal_read_repair_chance = 0`

/*
https://godoc.org/github.com/gocql/gocql#Session
Session is the interface used by users to interact with the database.
It's safe for concurrent use by multiple goroutines and a typical usage scenario is to have one global session
object to interact with the whole Cassandra cluster.
*/
var cSession *gocql.Session

func InitCassandra() error {
	cluster := gocql.NewCluster(cassandraAddrs...)
	cluster.Consistency = gocql.One
	var err error
	tmpSession, err := cluster.CreateSession()
	if err != nil {
		return err
	}
	// ensure the keyspace and table exist.
	err = tmpSession.Query(keyspace_schema).Exec()
	if err != nil {
		return err
	}
	err = tmpSession.Query(table_schema).Exec()
	if err != nil {
		return err
	}
	tmpSession.Close()
	cluster.Keyspace = "raintank"
	cSession, err = cluster.CreateSession()
	return err
}

// Insert metric into Cassandra.
//
// key: is the metric_id
// ts: is the start of the aggregated time range.
// data: is the payload as bytes.
func InsertMetric(key string, t0 uint32, data []byte, ttl int) error {
	// for unit tests
	if cSession == nil {
		return nil
	}
	query := fmt.Sprintf("INSERT INTO metric (key, ts, data) values(?,?,?) USING TTL %d", ttl)
	row_key := fmt.Sprintf("%s_%d", key, t0/month) // "month number" based on unix timestamp (rounded down)
	pre := time.Now()
	ret := cSession.Query(query, row_key, t0, data).Exec()
	cassandraPutDuration.Value(time.Now().Sub(pre))
	return ret
}

type outcome struct {
	mark uint32
	i    *gocql.Iter
}
type asc []outcome

func (o asc) Len() int           { return len(o) }
func (o asc) Swap(i, j int)      { o[i], o[j] = o[j], o[i] }
func (o asc) Less(i, j int) bool { return o[i].mark < o[j].mark }

// Basic search of cassandra.
// start inclusive, end exclusive
func searchCassandra(key string, start, end uint32) ([]*tsz.Iter, error) {
	if start > end {
		panic(fmt.Sprintf("searchCassandra start %d > end %d", start, end))
	}

	results := make(chan outcome)
	wg := &sync.WaitGroup{}

	query := func(mark uint32, q string, p ...interface{}) {
		wg.Add(1)
		go func() {
			//		if len(p) == 3 {
			//		log.Println("querying cassandra for", q, p[0], TS(p[1]), TS(p[2]))
			//		} else if len(p) == 2 {
			//		log.Println("querying cassandra for", q, p[0], TS(p[1]))
			//		} else {
			//		log.Println("querying cassandra for", q, p)
			//		}
			results <- outcome{mark, cSession.Query(q, p...).Iter()}
			wg.Done()
		}()
	}

	start_month := start - (start % month)       // starting row has to be at, or before, requested start
	end_month := (end - 1) - ((end - 1) % month) // ending row has to be include the last point we might need

	pre := time.Now()
	if start_month == end_month {
		// we need a selection of the row between startTs and endTs
		row_key := fmt.Sprintf("%s_%d", key, start_month/month)
		query(start_month, "SELECT data FROM metric WHERE key = ? AND ts >= ? AND ts < ? ORDER BY ts ASC", row_key, start, end)
	} else {
		// get row_keys for each row we need to query.
		for mark := start_month; mark <= end_month; mark += month {
			row_key := fmt.Sprintf("%s_%d", key, mark/month)
			if mark == start_month {
				// we want from startTs to the end of the row.
				query(mark, "SELECT data FROM metric WHERE key = ? AND ts >= ? ORDER BY ts ASC", row_key, start)
			} else if mark == end_month {
				// we want from start of the row till the endTs.
				query(mark, "SELECT data FROM metric WHERE key = ? AND ts < ? ORDER BY ts ASC", row_key, end)
			} else {
				// we want all columns
				query(mark, "SELECT data FROM metric WHERE key = ? ORDER BY ts ASC", row_key)
			}
		}
	}
	outcomes := make([]outcome, 0)
	iters := make([]*tsz.Iter, 0)
	go func() {
		wg.Wait()
		cassandraGetDuration.Value(time.Now().Sub(pre))
		close(results)
	}()
	for o := range results {
		outcomes = append(outcomes, o)
	}
	sort.Sort(asc(outcomes))

	var b []byte
	for _, outcome := range outcomes {
		chunks := int64(0)
		for outcome.i.Scan(&b) {
			chunks += 1
			chunkSizeAtLoad.Value(int64(len(b)))
			iter, err := tsz.NewIterator(b)
			if err != nil {
				log.Fatal(err)
			}
			iters = append(iters, iter)
		}
		cassandraChunksPerRow.Value(chunks)
		err := outcome.i.Close()
		if err != nil {
			log.Println("ERROR:", err)
		}
	}
	cassandraRowsPerResponse.Value(int64(len(outcomes)))
	//log.Println(len(outcomes), "outcomes, cassandra results", len(iters))
	return iters, nil
}
<|endoftext|>


<commit_before>package memdb

import (
	"bytes"
	"errors"
	"fmt"
	"sort"
	"sync"
)

type Key []byte

type Value []byte

type kv struct {
	offset	int
	klen	int
	vlen	int
}

type kvs []kv

type kvData []byte

type ns struct{
	mu sync.RWMutex
	comparator func(a,b []byte) int
	kvs kvs
	kvData kvData
	kvSize int
}

type Nss []ns

func (ns *ns) Len() int { return len(ns.kvs)}

func (ns *ns) Swap(i,j int) {ns.kvs[i],ns.kvs[j] = ns.kvs[j],ns.kvs[i]}

type ByKey struct {ns}

func (ns *ByKey) Less(i,j int) bool {
	o1 := ns.kvs[i]
	o2 := ns.kvs[j]
	k1 := Key(ns.kvData[o1.offset:o1.offset+o1.klen])
	k2 := Key(ns.kvData[o2.offset:o2.offset+o2.klen])
	return ns.compare(k2,k1) >= 0
	}

func (ns *ns) compare(key1,key2 Key) int {
	return ns.comparator([]byte(key1),[]byte(key2));
}

func (ns *ns) delete(key Key) {
	i,err := ns.find(key);
	if err != nil {fmt.Println("Delete:",i,key);}
}

func (ns *ns) find(key Key) (int,error){
	s := sort.Search(len(ns.kvs), func(i int) bool {
		//fmt.Println("Bytes compare:",string(key),string(ns.getKey(i)),bytes.Compare([]byte(key),[]byte(ns.getKey(i))) == 0);
		return ns.compare(ns.getKey(i),key) >=0 });
	fmt.Println("Search:",string(key),len(ns.kvs),s);
	if s < len(ns.kvs) && ns.compare(ns.getKey(s),key) == 0 {return s,nil}
	return -1,errors.New("<Not found>");
}

func (ns *ns) getKey(i int) Key {
	k := Key(ns.kvData[ns.kvs[i].offset:ns.kvs[i].offset + ns.kvs[i].klen])
	//fmt.Println("getKey:",i,string(k));
	return k
}

func (ns *ns) getValue(i int) Value {
	//fmt.Println("getValue:",i,ns.kvs[i]);
	return Value(ns.kvData[ns.kvs[i].offset + ns.kvs[i].klen:ns.kvs[i].offset + ns.kvs[i].klen + ns.kvs[i].vlen])
}

func (ns *ns) get(key Key) (Value,error) {
	k,err := ns.find(key);
	if err != nil {
		return nil,err;
	}
	return ns.getValue(k),nil;
}

func (ns *ns) has(key Key) bool {
	_, err := ns.find(key)
	return err == nil;
}

func NewKV(offset,lkey,lval int) kv {
	return kv{offset,lkey,lval}
}

func (ns *ns) put(key Key, value Value) error {
	var buf bytes.Buffer
	ns.delete(key);
	buf.Write(key);
	buf.Write(value);
	ns.kvs = append(ns.kvs,NewKV(ns.kvSize,len(key),len(value)));
	ns.kvSize = ns.kvSize + len(key) + len(value);
	d := buf.Bytes()
	//fmt.Println("Put:",d);
	for _,v := range d {
		ns.kvData = append(ns.kvData,v);
	}
	//fmt.Println("kvData:",string(ns.kvData));
	return nil
}

func (ns *ns) Delete(key []byte) {
	ns.delete(key);
}

func (ns *ns) Has(key []byte) bool {
	return ns.has(key);
}

func (ns *ns) Get(key []byte) (Value,error) {
	return ns.get(key);
}

func (ns *ns) Put(key,value []byte) error {
	return ns.put(key,value);
}

func (ns *ns) Seek(key []byte) Value {
	return Value("key")
}

func NewNs() *ns {return new(ns)}

type MemDB struct{
	nss		[]ns
	nsSize	uint64
}

func (db *MemDB) getNS(parent, ns []byte) uint64 {
	val := db.nss[0].Get(Key(append(bytes.Fields(parent),ns)))
	return 0;
}
/*

func (db *memDB) Get(ns uint64,key []byte) Value{
	return ns[ns].Get(key);
}

func (db *memDB) Delete(ns, key []byte) Value{
	id := getNS(ns);
	return ns[id].Get(key);
}

func (db *memDB) Has(ns, key []byte) Value{
	id := getNS(ns);
	return ns[id].Get(key);
}

func (db *memDB) Get(ns, key []byte) Value{
	id := getNS(ns);
	return ns[id].Get(key);
}

func (db *emDB) Put(ns,key,value []byte) Value{
	id := getNS(ns);
	return ns[id].Put(key,value);
}

func (db *Memdb) Seek(ns, key []byte) Value{
	id := getNS(ns);
	return ns[id].Seek(key);
}
*/

<commit_msg>Add sorted in method Put<commit_after>package main
//package memdb

import (
	"bytes"
	"encoding/binary"
	"errors"
	"fmt"
	"sort"
	"sync"
	"time"
)

type Key []byte

type Value []byte

type kv struct {
	offset	int
	klen	int
	vlen	int
}

type kvs []kv

type kvData []byte

type ns struct{
	mu sync.RWMutex
	Comparator func(a,b []byte) int
	index []int
	kvs kvs
	kvData kvData
	kvSize int
}

type Nss []ns

//func (ns *ns) Len() int { return len(ns.kvs)}
func (ns *ns) Len() int { return len(ns.index)}

//func (ns *ns) Swap(i,j int) {ns.kvs[i],ns.kvs[j] = ns.kvs[j],ns.kvs[i]}
func (ns *ns) Swap(i,j int) {ns.index[i],ns.index[j] = ns.index[j],ns.index[i]}

type ByKey struct {ns}
/*
func (ns *ByKey) Less(i,j int) bool {
	o1 := ns.kvs[i]
	o2 := ns.kvs[j]
	k1 := Key(ns.kvData[o1.offset:o1.offset+o1.klen])
	k2 := Key(ns.kvData[o2.offset:o2.offset+o2.klen])
	return ns.compare(k2,k1) >= 0
	}
*/
func (ns *ByKey) Less(i,j int) bool {
	o1 := ns.kvs[ns.index[i]]
	o2 := ns.kvs[ns.index[j]]
	k1 := Key(ns.kvData[o1.offset:o1.offset+o1.klen])
	k2 := Key(ns.kvData[o2.offset:o2.offset+o2.klen])
	return ns.compare(k2,k1) >= 0
	}

func (ns *ns) compare(key1,key2 Key) int {
	return ns.Comparator([]byte(key1),[]byte(key2));
}

func (ns *ns) delete(key Key) {
	i,err := ns.find(key);
	if err == nil {ns.deleteKey(i);}
}

func (ns *ns) deleteKey(i int) {
	idx := ns.index[i]
	r := ns.kvs[idx]
	ns.kvData = append(ns.kvData[:r.offset],ns.kvData[r.offset+r.klen+r.vlen:]...)
	ns.kvs = append(ns.kvs[:idx], ns.kvs[idx:]...);
	ns.index = append(ns.index[:i],ns.index[i:]...);
	ns.kvSize = ns.kvSize - r.klen - r.vlen;
}

func (ns *ns) find(key Key) (int,error){
	s := sort.Search(len(ns.index), func(i int) bool {
		//fmt.Println("Bytes compare:",string(key),string(ns.getKey(i)),bytes.Compare([]byte(key),[]byte(ns.getKey(i))) == 0);
		return ns.compare(ns.getKey(i),key) >=0 });
	//fmt.Println("Search:",string(key),len(ns.kvs),s);
	if s < len(ns.kvs) && ns.compare(ns.getKey(s),key) == 0 {return s,nil}
	return -1,errors.New("<Not found>");
}

func (ns *ns) getKey(i int) Key {
	r := ns.kvs[ns.index[i]]
	k := Key(ns.kvData[r.offset:r.offset + r.klen])
	//fmt.Println("getKey:",i,string(k));
	return k
}

func (ns *ns) getValue(i int) Value {
	//fmt.Println("getValue:",i,ns.kvs[i]);
	r := ns.kvs[ns.index[i]]
	return Value(ns.kvData[r.offset + r.klen:r.offset + r.klen + r.vlen])
}

func (ns *ns) get(key Key) (Value,error) {
	k,err := ns.find(key);
	fmt.Println("get:",k,err);
	if err != nil {
		return nil,err;
	}
	return ns.getValue(k),nil;
}

func (ns *ns) has(key Key) bool {
	_, err := ns.find(key)
	return err == nil;
}

func NewKV(offset,lkey,lval int) kv {
	return kv{offset,lkey,lval}
}

func (ns *ns) put(key Key, value Value) error {
	ns.delete(key);
	ns.kvs = append(ns.kvs,NewKV(ns.kvSize,len(key),len(value)));
	ns.kvData = append(ns.kvData,[]byte(key)...);
	ns.kvData = append(ns.kvData,[]byte(value)...);
	ns.kvSize = ns.kvSize + len(key) + len(value);
	ns.index = append(ns.index,len(ns.kvs)-1);
	//fmt.Println("kvData:",string(ns.kvData));
	return nil
}

func (ns *ns) Delete(key Key) {
	ns.delete(key);
	}

func (ns *ns) Has(key Key) bool {
	return ns.has(key);
}

func (ns *ns) Get(key Key) (Value,error) {
	return ns.get(key);
}

func (ns *ns) Put(key Key, value Value) error {
	err := ns.put(key,value);
	if err == nil {sort.Sort(&ByKey{*ns});}
	return err
}

func (ns *ns) Seek(key []byte) Value {
	return Value("key")
}

func NewNs() *ns {return new(ns)}

type MemDB struct{
	nss		[]ns
	nsSize	uint64
}

func (db *MemDB) getNS(parent, namespace []byte) int64 {
	val,err := db.nss[0].Get(append(parent,namespace...));
	if err != nil { return -1}
	v,c := binary.Varint(val);
	if c > 0 {return v;}
	return -1
}

func (db *MemDB) putNS(parent, namespace,value []byte) int64 {
	val,err := db.nss[0].Get(append(parent,namespace...));
	if err != nil { return -1}
	v,c := binary.Varint(val);
	if c > 0 {return v;}
	return -1
}

/*

func (db *memDB) Get(ns uint64,key []byte) Value{
	return ns[ns].Get(key);
}

func (db *memDB) Delete(ns, key []byte) Value{
	id := getNS(ns);
	return ns[id].Get(key);
}

func (db *memDB) Has(ns, key []byte) Value{
	id := getNS(ns);
	return ns[id].Get(key);
}

func (db *memDB) Get(ns, key []byte) Value{
	id := getNS(ns);
	return ns[id].Get(key);
}

func (db *emDB) Put(ns,key,value []byte) Value{
	id := getNS(ns);
	return ns[id].Put(key,value);
}

func (db *Memdb) Seek(ns, key []byte) Value{
	id := getNS(ns);
	return ns[id].Seek(key);
}
*/
func main(){

 n := NewNs();
 //n := ns{}
 n.Comparator = bytes.Compare
 n.Put(Key("abc2"),Value("1234+2babc"))
 n.Put(Key("abc"),Value("1234"));
 n.Put(Key("abc1"),Value("1234+1"))
 n.Put(Key("abc85"),Value("1234+85"))
 n.Put(Key("abc3"),Value("1234+3"));
 n.Put(Key("abc7"),Value("1234+7"))

tp0 := time.Now()
 for i:=10000; i >= 0; i-- {
	n.Put(Key("Iiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiii-"+string(i)),Value("Vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv-"+string(i)));
 }
tp1 := time.Now()
fmt.Printf("The call puted %v to run.\n", tp1.Sub(tp0))
/*
 for i := 0; i < len(n.index); i++ {
 fmt.Println("KV:",i,string(n.getKey(i)));
 }

t0 := time.Now()
sort.Sort(&ByKey{n});
t1 := time.Now()
fmt.Printf("The call sorting %v to run.\n", t1.Sub(t0))
 */

//sort.Sort(&ByKey{*n});
v,err := n.Get(Key("abc2"))
fmt.Println("V:",string(v),err)
//fmt.Println(string(n.kvData))
n.Delete(Key("abc2"))
//fmt.Println(string(n.kvData))
v,err = n.Get(Key("abc2"))
fmt.Println("V:",string(v),err)
//fmt.Println("INDEX:",n.index)
}

<|endoftext|>


<commit_before>// Copyright 2013 Andreas Koch. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
	Package model defines the basic
	data structures of the docs engine.
*/
package model

import (
	"crypto/sha1"
	"encoding/hex"
	"io/ioutil"
	"time"
)

type RepositoryItem struct {
	Path       string
	Files      []RepositoryItemFile
	ChildItems []RepositoryItem
}

type RepositoryItemFile struct {
	Path string
}

func NewRepositoryItemFile(path string) RepositoryItemFile {
	return RepositoryItemFile{
		Path: path,
	}
}

func NewRepositoryItem(path string, files []RepositoryItemFile, childItems []RepositoryItem) RepositoryItem {
	return RepositoryItem{
		Path:       path,
		Files:      files,
		ChildItems: childItems,
	}
}

func (item *RepositoryItem) GetHash() string {
	itemBytes, readFileErr := ioutil.ReadFile(item.Path)
	if readFileErr != nil {
		return ""
	}

	sha1Hash := sha1.New()
	sha1Hash.Write(itemBytes)
	hashBytes := sha1Hash.Sum(nil)

	return string(hex.EncodeToString(hashBytes))
}

func (item *RepositoryItem) String() string {
	s := item.Path + "(Hash: " + item.GetHash() + ")\n"

	s += "\n"
	s += "Files:\n"
	if len(item.Files) > 0 {
		for _, file := range item.Files {
			s += " - " + file.Path + "\n"
		}
	} else {
		s += "<none>\n"
	}

	s += "\n"
	s += "ChildItems:\n"
	if len(item.ChildItems) > 0 {
		for _, child := range item.ChildItems {
			s += child.String()
		}
	} else {
		s += "<none>\n"
	}
	s += "\n"

	return s
}

type Document struct {
	Path    string // The documents folder
	Content string // The document content

	Title       string // The document title
	Description string // A short description of the document content.

	// Meta information
	Language string    // [optional] The ISO language code document (e.g. "en-GB", "de-DE")
	Date     time.Time // [optional] The date the document has been created
}
<commit_msg>Reduced the hash length<commit_after>// Copyright 2013 Andreas Koch. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
	Package model defines the basic
	data structures of the docs engine.
*/
package model

import (
	"crypto/sha1"
	"encoding/hex"
	"io/ioutil"
	"time"
)

type RepositoryItem struct {
	Path       string
	Files      []RepositoryItemFile
	ChildItems []RepositoryItem
}

type RepositoryItemFile struct {
	Path string
}

func NewRepositoryItemFile(path string) RepositoryItemFile {
	return RepositoryItemFile{
		Path: path,
	}
}

func NewRepositoryItem(path string, files []RepositoryItemFile, childItems []RepositoryItem) RepositoryItem {
	return RepositoryItem{
		Path:       path,
		Files:      files,
		ChildItems: childItems,
	}
}

func (item *RepositoryItem) GetHash() string {
	itemBytes, readFileErr := ioutil.ReadFile(item.Path)
	if readFileErr != nil {
		return ""
	}

	sha1Hash := sha1.New()
	sha1Hash.Write(itemBytes)
	hashBytes := sha1Hash.Sum(nil)

	return string(hex.EncodeToString(hashBytes[0:6]))
}

func (item *RepositoryItem) String() string {
	s := item.Path + "(Hash: " + item.GetHash() + ")\n"

	s += "\n"
	s += "Files:\n"
	if len(item.Files) > 0 {
		for _, file := range item.Files {
			s += " - " + file.Path + "\n"
		}
	} else {
		s += "<none>\n"
	}

	s += "\n"
	s += "ChildItems:\n"
	if len(item.ChildItems) > 0 {
		for _, child := range item.ChildItems {
			s += child.String()
		}
	} else {
		s += "<none>\n"
	}
	s += "\n"

	return s
}

type Document struct {
	Path    string // The documents folder
	Content string // The document content

	Title       string // The document title
	Description string // A short description of the document content.

	// Meta information
	Language string    // [optional] The ISO language code document (e.g. "en-GB", "de-DE")
	Date     time.Time // [optional] The date the document has been created
}
<|endoftext|>


<commit_before>package models

import (
	"database/sql"

	e "github.com/techjanitor/pram-get/errors"
	u "github.com/techjanitor/pram-get/utils"
)

// PostModel holds the parameters from the request and also the key for the cache
type PostModel struct {
	Ib     uint
	Thread uint
	Id     uint
	Result PostType
}

// IndexType is the top level of the JSON response
type PostType struct {
	Body Post `json:"post"`
}

type Post struct {
	ThreadId    uint    `json:"thread_id"`
	PostId      uint    `json:"post_id"`
	Num         uint    `json:"num"`
	Name        string  `json:"name"`
	Group       uint    `json:"group"`
	Avatar      uint    `json:"avatar"`
	Time        *string `json:"time"`
	Text        *string `json:"comment"`
	ImgId       *uint   `json:"img_id,omitempty"`
	File        *string `json:"filename,omitempty"`
	Thumb       *string `json:"thumbnail,omitempty"`
	ThumbHeight *uint   `json:"tn_height,omitempty"`
	ThumbWidth  *uint   `json:"tn_width,omitempty"`
}

// Get will gather the information from the database and return it as JSON serialized data
func (i *PostModel) Get() (err error) {

	// Initialize response header
	response := PostType{}

	// Get Database handle
	db, err := u.GetDb()
	if err != nil {
		return
	}

	post := Post{}

	err = db.QueryRow(`SELECT threads.thread_id,posts.post_id,post_num,user_name,usergroup_id,user_avatar,post_time,post_text,image_id,image_file,image_thumbnail,image_tn_height,image_tn_width
	FROM posts
	LEFT JOIN images on posts.post_id = images.post_id
	INNER JOIN threads on posts.thread_id = threads.thread_id
	INNER JOIN users on posts.user_id = users.user_id
	WHERE posts.post_num = ? AND posts.thread_id = ? AND ib_id = ? AND thread_deleted != 1 AND post_deleted != 1`, i.Id, i.Thread, i.Ib).Scan(&post.ThreadId, &post.PostId, &post.Num, &post.Name, &post.Group, &user.Avatar, &post.Time, &post.Text, &post.ImgId, &post.File, &post.Thumb, &post.ThumbHeight, &post.ThumbWidth)
	if err == sql.ErrNoRows {
		return e.ErrNotFound
	} else if err != nil {
		return
	}

	// Add pagedresponse to the response struct
	response.Body = post

	// This is the data we will serialize
	i.Result = response

	return

}
<commit_msg>add avatar<commit_after>package models

import (
	"database/sql"

	e "github.com/techjanitor/pram-get/errors"
	u "github.com/techjanitor/pram-get/utils"
)

// PostModel holds the parameters from the request and also the key for the cache
type PostModel struct {
	Ib     uint
	Thread uint
	Id     uint
	Result PostType
}

// IndexType is the top level of the JSON response
type PostType struct {
	Body Post `json:"post"`
}

type Post struct {
	ThreadId    uint    `json:"thread_id"`
	PostId      uint    `json:"post_id"`
	Num         uint    `json:"num"`
	Name        string  `json:"name"`
	Group       uint    `json:"group"`
	Avatar      uint    `json:"avatar"`
	Time        *string `json:"time"`
	Text        *string `json:"comment"`
	ImgId       *uint   `json:"img_id,omitempty"`
	File        *string `json:"filename,omitempty"`
	Thumb       *string `json:"thumbnail,omitempty"`
	ThumbHeight *uint   `json:"tn_height,omitempty"`
	ThumbWidth  *uint   `json:"tn_width,omitempty"`
}

// Get will gather the information from the database and return it as JSON serialized data
func (i *PostModel) Get() (err error) {

	// Initialize response header
	response := PostType{}

	// Get Database handle
	db, err := u.GetDb()
	if err != nil {
		return
	}

	post := Post{}

	err = db.QueryRow(`SELECT threads.thread_id,posts.post_id,post_num,user_name,usergroup_id,user_avatar,post_time,post_text,image_id,image_file,image_thumbnail,image_tn_height,image_tn_width
	FROM posts
	LEFT JOIN images on posts.post_id = images.post_id
	INNER JOIN threads on posts.thread_id = threads.thread_id
	INNER JOIN users on posts.user_id = users.user_id
	WHERE posts.post_num = ? AND posts.thread_id = ? AND ib_id = ? AND thread_deleted != 1 AND post_deleted != 1`, i.Id, i.Thread, i.Ib).Scan(&post.ThreadId, &post.PostId, &post.Num, &post.Name, &post.Group, &post.Avatar, &post.Time, &post.Text, &post.ImgId, &post.File, &post.Thumb, &post.ThumbHeight, &post.ThumbWidth)
	if err == sql.ErrNoRows {
		return e.ErrNotFound
	} else if err != nil {
		return
	}

	// Add pagedresponse to the response struct
	response.Body = post

	// This is the data we will serialize
	i.Result = response

	return

}
<|endoftext|>