<commit_before>package main

import (
	"os/exec"
	"io"
	"os"
	"io/ioutil"
	"strings"
	"syscall"
	"fmt"

	"github.com/flynn/go-discover/discover"
)

func main() {
	root := "/var/lib/demo/apps"
	hostname := shell("curl -s icanhazip.com")

	client, err := discover.NewClient()
	if err != nil {
		panic(err)
	}
	set, _ := client.Services("shelf")
	addrs := set.OnlineAddrs()
	if len(addrs) < 1 {
		panic("Shelf is not discoverable")
	}
	shelfHost := addrs[0]

	app := os.Args[2]
	os.MkdirAll(root + "/" + app, 0755)

	fmt.Printf("-----> Building %s on %s ...\n", app, hostname)

	cmd := exec.Command("docker", "run", "-i", "-a=stdin", "-a=stdout", "flynn/slugbuilder", "http://"+shelfHost+"/"+app+".tgz")
	errCh, startCh := attachCmd(cmd, os.Stdout, os.Stderr, os.Stdin)
	err = cmd.Start()
	if err != nil {
		panic(err)
	}
	close(startCh)
	exitCh := exitStatusCh(cmd)
	if err = <-errCh; err != nil {
		panic(err)
	}
	<-exitCh

	fmt.Printf("-----> Deploying %s ...\n", app)
	if _, err := os.Stat(root + "/" + app + "/CONTAINER"); err == nil {
    	oldid := readFile(root + "/" + app + "/CONTAINER")
    	shell("docker kill " + oldid)
	}

	id := shell("docker run -d -p 5000 -e PORT=5000 -e SLUG_URL=http://"+shelfHost+"/"+app+".tgz flynn/slugrunner start web")
	writeFile(root + "/" + app + "/CONTAINER", id)
	port := shell("docker port "+id+" 5000 | sed 's/0.0.0.0://'")
	writeFile(root + "/" + app + "/PORT", port)
	writeFile(root + "/" + app + "/URL", "http://"+hostname+":"+port)

	fmt.Printf("=====> Application deployed:\n")
	fmt.Printf("       %s\n", readFile(root + "/" + app + "/URL"))
	fmt.Println("")

}

func readFile(filename string) string {
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		panic(err)
	}
	return string(data)
}

func writeFile(filename, data string) {
	err := ioutil.WriteFile(filename, []byte(data), 0644)
	if err != nil {
		panic(err)
	}
}

func shell(cmdline string) string {
	out, err := exec.Command("bash", "-c", cmdline).Output()
	if err != nil {
		panic(err)
	}
	return strings.Trim(string(out), " \n")
}

func attachCmd(cmd *exec.Cmd, stdout, stderr io.Writer, stdin io.Reader) (chan error, chan interface{}) {
	errCh := make(chan error)
	startCh := make(chan interface{})

	stdinIn, err := cmd.StdinPipe()
	if err != nil {
		panic(err)
	}
	stdoutOut, err := cmd.StdoutPipe()
	if err != nil {
		panic(err)
	}
	stderrOut, err := cmd.StderrPipe()
	if err != nil {
		panic(err)
	}

	go func() {
		<-startCh
		go func() {
			_, e := io.Copy(stdinIn, stdin)
			errCh <- e
		}()
		go func() {
			_, e := io.Copy(stdout, stdoutOut)
			errCh <- e
		}()
		go func() {
			_, e := io.Copy(stderr, stderrOut)
			errCh <- e
		}()
	}()

	return errCh, startCh
}

func exitStatusCh(cmd *exec.Cmd) chan uint {
	exitCh := make(chan uint)
	go func() {
		err := cmd.Wait()
		if err != nil {
			if exiterr, ok := err.(*exec.ExitError); ok {
				// There is no plattform independent way to retrieve
				// the exit code, but the following will work on Unix
				if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
					exitCh <- uint(status.ExitStatus())
				}
			} else {
				panic(err)
			}
			return
		}
		exitCh <- uint(0)
	}()
	return exitCh
}<commit_msg>demo: never mind<commit_after>package main

import (
	"os/exec"
	"io"
	"os"
	"io/ioutil"
	"strings"
	"syscall"
	"fmt"

	"github.com/flynn/go-discover/discover"
)

func main() {
	root := "/var/lib/demo/apps"
	hostname := shell("curl -s icanhazip.com")

	client, err := discover.NewClient()
	if err != nil {
		panic(err)
	}
	set, _ := client.Services("shelf")
	addrs := set.OnlineAddrs()
	if len(addrs) < 1 {
		panic("Shelf is not discoverable")
	}
	shelfHost := addrs[0]

	app := os.Args[2]
	os.MkdirAll(root + "/" + app, 0755)

	fmt.Printf("-----> Building %s on %s ...\n", app, hostname)

	cmd := exec.Command("docker", "run", "-i", "-a=stdin", "-a=stdout", "flynn/slugbuilder", "http://"+shelfHost+"/"+app+".tgz")
	errCh := attachCmd(cmd, os.Stdout, os.Stderr, os.Stdin)
	err = cmd.Start()
	if err != nil {
		panic(err)
	}
	exitCh := exitStatusCh(cmd)
	if err = <-errCh; err != nil {
		panic(err)
	}
	<-exitCh

	fmt.Printf("-----> Deploying %s ...\n", app)
	if _, err := os.Stat(root + "/" + app + "/CONTAINER"); err == nil {
    	oldid := readFile(root + "/" + app + "/CONTAINER")
    	shell("docker kill " + oldid)
	}

	id := shell("docker run -d -p 5000 -e PORT=5000 -e SLUG_URL=http://"+shelfHost+"/"+app+".tgz flynn/slugrunner start web")
	writeFile(root + "/" + app + "/CONTAINER", id)
	port := shell("docker port "+id+" 5000 | sed 's/0.0.0.0://'")
	writeFile(root + "/" + app + "/PORT", port)
	writeFile(root + "/" + app + "/URL", "http://"+hostname+":"+port)

	fmt.Printf("=====> Application deployed:\n")
	fmt.Printf("       %s\n", readFile(root + "/" + app + "/URL"))
	fmt.Println("")

}

func readFile(filename string) string {
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		panic(err)
	}
	return string(data)
}

func writeFile(filename, data string) {
	err := ioutil.WriteFile(filename, []byte(data), 0644)
	if err != nil {
		panic(err)
	}
}

func shell(cmdline string) string {
	out, err := exec.Command("bash", "-c", cmdline).Output()
	if err != nil {
		panic(err)
	}
	return strings.Trim(string(out), " \n")
}

func attachCmd(cmd *exec.Cmd, stdout, stderr io.Writer, stdin io.Reader) chan error {
	errCh := make(chan error)

	stdinIn, err := cmd.StdinPipe()
	if err != nil {
		panic(err)
	}
	stdoutOut, err := cmd.StdoutPipe()
	if err != nil {
		panic(err)
	}
	stderrOut, err := cmd.StderrPipe()
	if err != nil {
		panic(err)
	}

	go func() {
		_, e := io.Copy(stdinIn, stdin)
		errCh <- e
	}()
	go func() {
		_, e := io.Copy(stdout, stdoutOut)
		errCh <- e
	}()
	go func() {
		_, e := io.Copy(stderr, stderrOut)
		errCh <- e
	}()

	return errCh
}

func exitStatusCh(cmd *exec.Cmd) chan uint {
	exitCh := make(chan uint)
	go func() {
		err := cmd.Wait()
		if err != nil {
			if exiterr, ok := err.(*exec.ExitError); ok {
				// There is no plattform independent way to retrieve
				// the exit code, but the following will work on Unix
				if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
					exitCh <- uint(status.ExitStatus())
				}
			} else {
				panic(err)
			}
			return
		}
		exitCh <- uint(0)
	}()
	return exitCh
}<|endoftext|>


<commit_before>package main

import (
	"os/exec"
	"strings"
	"encoding/gob"
	"fmt"
	"bufio"
	"io"
	"log"
	"net"
	"time"
	"os"

	"github.com/flynn/go-discover/discover"
	"github.com/flynn/lorne/types"
	"github.com/titanous/go-dockerclient"
	"github.com/flynn/sampi/types"
	sc "github.com/flynn/sampi/client"
	lc "github.com/flynn/lorne/client"
)

// WARNING: assumes one host at the moment (firstHost will always be the same)

var sd *discover.Client
var sched *sc.Client
var host *lc.Client
var hostid string

func init() {
	var err error
	sd, err = discover.NewClient()
	if err != nil {
		log.Fatal(err)
	}

	sched, err = sc.New()
	if err != nil {
		log.Fatal(err)
	}

	hostid = findHost()
	host, err = lc.New(hostid)
	if err != nil {
		log.Fatal(err)
	}
}

func main() {
	root := "/var/lib/demo/apps"
	hostname := shell("curl -s icanhazip.com")

	set, _ := sd.Services("shelf")
	addrs := set.OnlineAddrs()
	if len(addrs) < 1 {
		panic("Shelf is not discoverable")
	}
	shelfHost := addrs[0]

	app := os.Args[2]
	os.MkdirAll(root+"/"+app, 0755)

	fmt.Printf("-----> Building %s on %s ...\n", app, hostname)

	scheduleAndAttach(app + ".build", docker.Config{
		Image:        "flynn/slugbuilder",
		Cmd:          []string{"http://" + shelfHost + "/" + app + ".tgz"},
		Tty:          false,
		AttachStdin:  true,
		AttachStdout: true,
		AttachStderr: true,
		OpenStdin:    true,
		StdinOnce:    true,
	})

	fmt.Printf("-----> Deploying %s ...\n", app)

	jobid := app + ".web"

	stopIfExists(jobid)
	scheduleWithTcpPort(jobid, docker.Config{
		Image:        "flynn/slugrunner",
		Cmd:          []string{"start", "web"},
		Tty:          false,
		AttachStdin:  false,
		AttachStdout: false,
		AttachStderr: false,
		OpenStdin:    false,
		StdinOnce:    false,
		Env: []string{
			"SLUG_URL=http://" + shelfHost + "/" + app + ".tgz",
		},
	})

	time.Sleep(1 * time.Second)
	fmt.Printf("=====> Application deployed:\n")
	fmt.Printf("       http://%s:%s\n", hostname, getPort(jobid))
	fmt.Println("")

}

func shell(cmdline string) string {
        out, err := exec.Command("bash", "-c", cmdline).Output()
        if err != nil {
                panic(err)
        }
        return strings.Trim(string(out), " \n")
}

func stopIfExists(jobid string) {
	_, err := host.GetJob(jobid)
	if err != nil {
		return
	}
	if err := host.StopJob(jobid); err != nil {
		return
	}
}

func scheduleWithTcpPort(jobid string, config docker.Config) {

	schedReq := &sampi.ScheduleReq{
		Incremental: true,
		HostJobs:    map[string][]*sampi.Job{hostid: {{ID: jobid, Config: &config, TCPPorts: 1}}},
	}
	if _, err := sched.Schedule(schedReq); err != nil {
		log.Fatal(err)
	}
}

func getPort(jobid string) string {
	job, err := host.GetJob(jobid)
	if err != nil {
		log.Fatal(err)
	}
	for portspec := range job.Job.Config.ExposedPorts {
		return strings.Split(portspec, "/")[0]
	}
}

func findHost() string {
	state, err := sched.State()
	if err != nil {
		log.Fatal(err)
	}

	var firstHost string
	for k := range state {
		firstHost = k
		break
	}
	if firstHost == "" {
		log.Fatal("no hosts")
	}
	return firstHost
}

func scheduleAndAttach(jobid string, config docker.Config) {

	services, err := sd.Services("flynn-lorne-attach." + hostid)
	if err != nil {
		log.Fatal(err)
	}
	conn, err := net.Dial("tcp", services.OnlineAddrs()[0])
	if err != nil {
		log.Fatal(err)
	}
	err = gob.NewEncoder(conn).Encode(&lorne.AttachReq{
		JobID: jobid,
		Flags: lorne.AttachFlagStdout | lorne.AttachFlagStderr | lorne.AttachFlagStdin | lorne.AttachFlagStream,
	})
	if err != nil {
		log.Fatal(err)
	}
	attachState := make([]byte, 1)
	if _, err := conn.Read(attachState); err != nil {
		log.Fatal(err)
	}
	switch attachState[0] {
	case lorne.AttachError:
		log.Fatal("attach error")
	}

	schedReq := &sampi.ScheduleReq{
		Incremental: true,
		HostJobs:    map[string][]*sampi.Job{hostid: {{ID: jobid, Config: &config}}},
	}
	if _, err := sched.Schedule(schedReq); err != nil {
		log.Fatal(err)
	}

	if _, err := conn.Read(attachState); err != nil {
		log.Fatal(err)
	}

	go func() {
		io.Copy(conn, os.Stdin)
		conn.(*net.TCPConn).CloseWrite()
	}()
	scanner := bufio.NewScanner(conn)
	for scanner.Scan() {
		fmt.Fprintln(os.Stdout, scanner.Text()[8:])
	}
	conn.Close()
}
<commit_msg>receiver: need a return<commit_after>package main

import (
	"os/exec"
	"strings"
	"encoding/gob"
	"fmt"
	"bufio"
	"io"
	"log"
	"net"
	"time"
	"os"

	"github.com/flynn/go-discover/discover"
	"github.com/flynn/lorne/types"
	"github.com/titanous/go-dockerclient"
	"github.com/flynn/sampi/types"
	sc "github.com/flynn/sampi/client"
	lc "github.com/flynn/lorne/client"
)

// WARNING: assumes one host at the moment (firstHost will always be the same)

var sd *discover.Client
var sched *sc.Client
var host *lc.Client
var hostid string

func init() {
	var err error
	sd, err = discover.NewClient()
	if err != nil {
		log.Fatal(err)
	}

	sched, err = sc.New()
	if err != nil {
		log.Fatal(err)
	}

	hostid = findHost()
	host, err = lc.New(hostid)
	if err != nil {
		log.Fatal(err)
	}
}

func main() {
	root := "/var/lib/demo/apps"
	hostname := shell("curl -s icanhazip.com")

	set, _ := sd.Services("shelf")
	addrs := set.OnlineAddrs()
	if len(addrs) < 1 {
		panic("Shelf is not discoverable")
	}
	shelfHost := addrs[0]

	app := os.Args[2]
	os.MkdirAll(root+"/"+app, 0755)

	fmt.Printf("-----> Building %s on %s ...\n", app, hostname)

	scheduleAndAttach(app + ".build", docker.Config{
		Image:        "flynn/slugbuilder",
		Cmd:          []string{"http://" + shelfHost + "/" + app + ".tgz"},
		Tty:          false,
		AttachStdin:  true,
		AttachStdout: true,
		AttachStderr: true,
		OpenStdin:    true,
		StdinOnce:    true,
	})

	fmt.Printf("-----> Deploying %s ...\n", app)

	jobid := app + ".web"

	stopIfExists(jobid)
	scheduleWithTcpPort(jobid, docker.Config{
		Image:        "flynn/slugrunner",
		Cmd:          []string{"start", "web"},
		Tty:          false,
		AttachStdin:  false,
		AttachStdout: false,
		AttachStderr: false,
		OpenStdin:    false,
		StdinOnce:    false,
		Env: []string{
			"SLUG_URL=http://" + shelfHost + "/" + app + ".tgz",
		},
	})

	time.Sleep(1 * time.Second)
	fmt.Printf("=====> Application deployed:\n")
	fmt.Printf("       http://%s:%s\n", hostname, getPort(jobid))
	fmt.Println("")

}

func shell(cmdline string) string {
        out, err := exec.Command("bash", "-c", cmdline).Output()
        if err != nil {
                panic(err)
        }
        return strings.Trim(string(out), " \n")
}

func stopIfExists(jobid string) {
	_, err := host.GetJob(jobid)
	if err != nil {
		return
	}
	if err := host.StopJob(jobid); err != nil {
		return
	}
}

func scheduleWithTcpPort(jobid string, config docker.Config) {

	schedReq := &sampi.ScheduleReq{
		Incremental: true,
		HostJobs:    map[string][]*sampi.Job{hostid: {{ID: jobid, Config: &config, TCPPorts: 1}}},
	}
	if _, err := sched.Schedule(schedReq); err != nil {
		log.Fatal(err)
	}
}

func getPort(jobid string) string {
	job, err := host.GetJob(jobid)
	if err != nil {
		log.Fatal(err)
	}
	for portspec := range job.Job.Config.ExposedPorts {
		return strings.Split(portspec, "/")[0]
	}
	return ""
}

func findHost() string {
	state, err := sched.State()
	if err != nil {
		log.Fatal(err)
	}

	var firstHost string
	for k := range state {
		firstHost = k
		break
	}
	if firstHost == "" {
		log.Fatal("no hosts")
	}
	return firstHost
}

func scheduleAndAttach(jobid string, config docker.Config) {

	services, err := sd.Services("flynn-lorne-attach." + hostid)
	if err != nil {
		log.Fatal(err)
	}
	conn, err := net.Dial("tcp", services.OnlineAddrs()[0])
	if err != nil {
		log.Fatal(err)
	}
	err = gob.NewEncoder(conn).Encode(&lorne.AttachReq{
		JobID: jobid,
		Flags: lorne.AttachFlagStdout | lorne.AttachFlagStderr | lorne.AttachFlagStdin | lorne.AttachFlagStream,
	})
	if err != nil {
		log.Fatal(err)
	}
	attachState := make([]byte, 1)
	if _, err := conn.Read(attachState); err != nil {
		log.Fatal(err)
	}
	switch attachState[0] {
	case lorne.AttachError:
		log.Fatal("attach error")
	}

	schedReq := &sampi.ScheduleReq{
		Incremental: true,
		HostJobs:    map[string][]*sampi.Job{hostid: {{ID: jobid, Config: &config}}},
	}
	if _, err := sched.Schedule(schedReq); err != nil {
		log.Fatal(err)
	}

	if _, err := conn.Read(attachState); err != nil {
		log.Fatal(err)
	}

	go func() {
		io.Copy(conn, os.Stdin)
		conn.(*net.TCPConn).CloseWrite()
	}()
	scanner := bufio.NewScanner(conn)
	for scanner.Scan() {
		fmt.Fprintln(os.Stdout, scanner.Text()[8:])
	}
	conn.Close()
}
<|endoftext|>


<commit_before>package bearychat

import (
	"net/http"
	"testing"
)

const (
	testWebhook = "http://localhost:3927"
)

func TestWebhookResponse_IsOk(t *testing.T) {
	var resp WebhookResponse

	resp = WebhookResponse{Code: 0}
	if !resp.IsOk() {
		t.Errorf("response should be ok when code is 0")
	}

	resp = WebhookResponse{Code: 1}
	if resp.IsOk() {
		t.Errorf("response should not be ok when code is not 0")
	}
}

func TestIncomingWebhookClient_SetWebhook(t *testing.T) {
	h := NewIncomingWebhookClient("")
	if h.SetWebhook(testWebhook) == nil {
		t.Errorf("should return webhook client")
	}

	if h.Webhook != testWebhook {
		t.Errorf("should set webhook")
	}
}

func TestIncomingWebhookClient_SetHTTPClient(t *testing.T) {
	h := NewIncomingWebhookClient(testWebhook)

	if h.httpClient != http.DefaultClient {
		t.Errorf("should use `http.DefaultClient` by default")
	}

	testHTTPClient := &http.Client{}
	if h.SetHTTPClient(testHTTPClient) == nil {
		t.Errorf("should return webhook client")
	}

	if h.httpClient != testHTTPClient {
		t.Errorf("should set http client")
	}
}

func TestIncomingWebhookClient_Send_WithoutWebhook(t *testing.T) {
	h := NewIncomingWebhookClient("")
	_, err := h.Send(nil)
	if err == nil {
		t.Errorf("should not send when webhook is not set")
	}
}

func TestIncomingWebhookClient_Send_WithoutHTTPClient(t *testing.T) {
	h := NewIncomingWebhookClient(testWebhook)
	h.SetHTTPClient(nil)
	_, err := h.Send(nil)
	if err == nil {
		t.Errorf("should not send when http client is not set")
	}
}

func ExampleIncomingWebhookClient() {
	m := Incoming{Text: "Hello, BearyChat"}
	payload, _ := m.Build()
	resp, _ := NewIncomingWebhookClient("YOUR WEBHOOK URL").Send(payload)
	if resp.IsOk() {
		// parse resp result
	} else {
		// parse resp error
	}
}
<commit_msg>doc(webhook): update example source.<commit_after>package bearychat

import (
	"net/http"
	"testing"
)

const (
	testWebhook = "http://localhost:3927"
)

func TestWebhookResponse_IsOk(t *testing.T) {
	var resp WebhookResponse

	resp = WebhookResponse{Code: 0}
	if !resp.IsOk() {
		t.Errorf("response should be ok when code is 0")
	}

	resp = WebhookResponse{Code: 1}
	if resp.IsOk() {
		t.Errorf("response should not be ok when code is not 0")
	}
}

func TestIncomingWebhookClient_SetWebhook(t *testing.T) {
	h := NewIncomingWebhookClient("")
	if h.SetWebhook(testWebhook) == nil {
		t.Errorf("should return webhook client")
	}

	if h.Webhook != testWebhook {
		t.Errorf("should set webhook")
	}
}

func TestIncomingWebhookClient_SetHTTPClient(t *testing.T) {
	h := NewIncomingWebhookClient(testWebhook)

	if h.httpClient != http.DefaultClient {
		t.Errorf("should use `http.DefaultClient` by default")
	}

	testHTTPClient := &http.Client{}
	if h.SetHTTPClient(testHTTPClient) == nil {
		t.Errorf("should return webhook client")
	}

	if h.httpClient != testHTTPClient {
		t.Errorf("should set http client")
	}
}

func TestIncomingWebhookClient_Send_WithoutWebhook(t *testing.T) {
	h := NewIncomingWebhookClient("")
	_, err := h.Send(nil)
	if err == nil {
		t.Errorf("should not send when webhook is not set")
	}
}

func TestIncomingWebhookClient_Send_WithoutHTTPClient(t *testing.T) {
	h := NewIncomingWebhookClient(testWebhook)
	h.SetHTTPClient(nil)
	_, err := h.Send(nil)
	if err == nil {
		t.Errorf("should not send when http client is not set")
	}
}

func ExampleNewIncomingWebhookClient() {
	m := Incoming{Text: "Hello, BearyChat"}
	payload, _ := m.Build()
	resp, _ := NewIncomingWebhookClient("YOUR WEBHOOK URL").Send(payload)
	if resp.IsOk() {
		// parse resp result
	} else {
		// parse resp error
	}
}
<|endoftext|>


<commit_before>package frank

import (
	"code.google.com/p/go.net/html"
	"code.google.com/p/go.net/html/atom"
	"errors"
	irc "github.com/fluffle/goirc/client"
	"io"
	"log"
	"net"
	"net/http"
	"regexp"
	"strconv"
	"strings"
	"time"
)

// how many URLs can the cache store
const cacheSize = 100

// how many hours an entry should be considered valid
const cacheValidHours = 24

// how many kilo bytes should be considered when looking for the title
// tag.
const httpReadKByte = 100

// abort HTTP requests if it takes longer than X seconds. Not sure, it’s
// definitely magic involved. Must be larger than 5.
const httpGetDeadline = 10

// don’t repost the same title within this period
const noRepostWithinSeconds = 30

// matches all whitespace and zero bytes. Additionally, all Unicode
// characters of class Cf (format chars, e.g. right-to-left) and Cc
// (control chars) are matched.
var whitespaceRegex = regexp.MustCompile(`[\s\0\p{Cf}\p{Cc}]+`)

var ignoreDomainsRegex = regexp.MustCompile(`^http://p\.nnev\.de`)

var twitterDomainRegex = regexp.MustCompile(`(?i)^https?://(?:[a-z0-9]\.)?twitter.com`)
var twitterPicsRegex = regexp.MustCompile(`(?i)(?:\b|^)pic\.twitter\.com/[a-z0-9]+(?:\b|$)`)

var noSpoilerRegex = regexp.MustCompile(`(?i)(don't|no|kein|nicht) spoiler`)

func UriFind(conn *irc.Conn, line *irc.Line) {
	defer func() {
		if r := recover(); r != nil {
			log.Printf("MEGA-WTF:pkg: %v", r)
		}
	}()

	msg := line.Args[1]

	if noSpoilerRegex.MatchString(msg) {
		log.Printf("not spoilering this line: %s", msg)
		return
	}

	urls := extract(msg)

	for _, url := range urls {
		if url == "" {
			continue
		}

		if title := cacheGetTitleByUrl(url); title != "" {
			log.Printf("using cache for URL: %s", url)
			postTitle(conn, line, title, "Cache Info")
			continue
		}

		go func(url string) {
			if ignoreDomainsRegex.MatchString(url) {
				log.Printf("ignoring this URL: %s", url)
				return
			}

			log.Printf("testing URL: %s", url)
			title, _, err := TitleGet(url)
			if err != nil {
				//postTitle(conn, line, err.Error(), "Error")
			} else if title != "" {
				postTitle(conn, line, title, "")
				cacheAdd(url, title)
			}
		}(url)
	}
}

// regexing ////////////////////////////////////////////////////////////

func extract(msg string) []string {
	results := make([]string, 0)
	for idx := strings.Index(msg, "http"); idx > -1; idx = strings.Index(msg, "http") {
		url := msg[idx:]
		if !strings.HasPrefix(url, "http://") &&
			!strings.HasPrefix(url, "https://") {
			msg = msg[idx+len("http"):]
			continue
		}

		// End on commas, but only if they are followed by a space.
		// spiegel.de URLs have commas in them, that would be a
		// false positive otherwise.
		if end := strings.Index(url, ", "); end > -1 {
			url = url[:end]
		}

		// End on closing paren, but only if there is an opening
		// paren before the URL (should fix most false-positives).
		if end := strings.Index(url, ")"); idx > 0 && msg[idx-1] == '(' && end > -1 {
			url = url[:end]
		}

		// Whitespace always ends a URL.
		if end := strings.IndexAny(url, " \t"); end > -1 {
			url = url[:end]
		}

		results = append(results, url)
		msg = msg[idx+len(url):]
	}
	return results
}

// http/html stuff /////////////////////////////////////////////////////

func TitleGet(url string) (string, string, error) {
	// via http://www.reddit.com/r/golang/comments/10awvj/timeout_on_httpget/c6bz49s
	c := http.Client{
		Transport: &http.Transport{
			Dial: func(netw, addr string) (net.Conn, error) {
				deadline := time.Now().Add(time.Second * httpGetDeadline)
				c, err := net.DialTimeout(netw, addr, time.Second*(httpGetDeadline-5))
				if err != nil {
					return nil, err
				}
				c.SetDeadline(deadline)
				return c, nil
			},
		},
	}

	r, err := c.Get(url)
	if err != nil {
		log.Printf("WTF: could not resolve %s: %s\n", url, err)
		return "", url, err
	}
	defer r.Body.Close()

	lastUrl := r.Request.URL.String()

	// TODO: r.Body → utf8?
	title, tweet := titleParseHtml(io.LimitReader(r.Body, 1024*httpReadKByte))

	if r.StatusCode != 200 {
		return "", lastUrl, errors.New("[" + strconv.Itoa(r.StatusCode) + "] " + title)
	}

	if tweet != "" && twitterDomainRegex.MatchString(lastUrl) {
		title = tweet
	}

	log.Printf("Title for URL %s: %s\n", url, title)

	return title, lastUrl, nil
}

// parses the incoming HTML fragment and tries to extract text from
// suitable tags. Currently this is the page’s title tag and tweets
// when the HTML-code is similar enough to twitter.com. Returns
// title and tweet.
func titleParseHtml(r io.Reader) (string, string) {
	doc, err := html.Parse(r)
	if err != nil {
		log.Printf("WTF: html parser blew up: %s\n", err)
		return "", ""
	}

	title := ""
	tweetText := ""
	tweetUser := ""
	tweetPicUrl := ""

	var f func(*html.Node)
	f = func(n *html.Node) {
		if title == "" && n.Type == html.ElementNode && n.DataAtom == atom.Title {
			title = extractText(n)
			return
		}

		if tweetText == "" && hasClass(n, "tweet-text") {
			tweetText = extractText(n)
			return
		}

		if tweetUser == "" && hasClass(n, "js-user-profile-link") {
			tweetUser = extractText(n)
			return
		}

		if tweetPicUrl == "" && hasClass(n, "media-thumbnail") && !hasClass(n, "profile-picture") {
			attrVal := getAttr(n, "data-url")
			if attrVal != "" {
				tweetPicUrl = attrVal
				return
			}
		}

		// recurse down
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			f(c)
		}

	}
	f(doc)

	// cleanup
	tweet := ""
	if tweetText != "" {
		tweetText = twitterPicsRegex.ReplaceAllString(tweetText, "")
		tweetUser = strings.Replace(tweetUser, "@", "(@", 1) + "): "
		tweet = tweetUser + tweetText + " " + tweetPicUrl
		tweet = clean(tweet)
	}

	return strings.TrimSpace(title), strings.TrimSpace(tweet)
}

func extractText(n *html.Node) string {
	text := ""
	for c := n.FirstChild; c != nil; c = c.NextSibling {
		if c.Type == html.TextNode {
			text += c.Data
		} else {
			text += extractText(c)
		}
	}
	return clean(text)
}

func hasClass(n *html.Node, class string) bool {
	if n.Type != html.ElementNode {
		return false
	}

	class = " " + strings.TrimSpace(class) + " "
	if strings.Contains(" "+getAttr(n, "class")+" ", class) {
		return true
	}
	return false
}

func getAttr(n *html.Node, findAttr string) string {
	for _, attr := range n.Attr {
		if attr.Key == findAttr {
			return attr.Val
		}
	}
	return ""
}

// Cache ///////////////////////////////////////////////////////////////
type Cache struct {
	url   string
	title string
	date  time.Time
}

var cache = [cacheSize]Cache{}
var cacheIndex = 0

func cacheAdd(url string, title string) {
	if len(cache) == cacheIndex {
		cacheIndex = 0
	}
	cache[cacheIndex] = Cache{url, title, time.Now()}
	cacheIndex += 1
}

func cacheGetTitleByUrl(url string) string {
	for _, cc := range cache {
		if cc.url == url && time.Since(cc.date).Hours() <= cacheValidHours {
			return cc.title
		}
	}
	return ""
}

func cacheGetSecondsToLastPost(title string) int {
	var secondsAgo = int(^uint(0) >> 1)
	for _, cc := range cache {
		var a = int(time.Since(cc.date).Seconds())
		if cc.title == title && a < secondsAgo {
			secondsAgo = a
		}
	}
	return secondsAgo
}

// util ////////////////////////////////////////////////////////////////

func postTitle(conn *irc.Conn, line *irc.Line, title string, prefix string) {
	tgt := line.Args[0]

	secondsAgo := cacheGetSecondsToLastPost(title)
	if secondsAgo <= noRepostWithinSeconds {
		log.Printf("Skipping, because posted %d seconds ago (“%s”)", secondsAgo, title)
		return
	}

	log.Printf("nick=%s, target=%s, title=%s", line.Nick, tgt, title)
	// if target is our current nick, it was a private message.
	// Answer the users in this case.
	if tgt == conn.Me().Nick {
		tgt = line.Nick
	}
	if prefix == "" {
		prefix = "Link Info"
	} else {
		prefix = clean(prefix)
	}
	title = clean(title)
	// the IRC spec states that notice should be used instead of msg
	// and that bots should not react to notice at all. However, no
	// real world bot adheres to this. Furthermore, people who can’t
	// configure their client to not highlight them on notices will
	// complain.
	conn.Privmsg(tgt, "["+prefix+"] "+title)
}

func clean(text string) string {
	text = whitespaceRegex.ReplaceAllString(text, " ")
	return strings.TrimSpace(text)
}
<commit_msg>detected embedded content in tweets as well<commit_after>package frank

import (
	"code.google.com/p/go.net/html"
	"code.google.com/p/go.net/html/atom"
	"errors"
	irc "github.com/fluffle/goirc/client"
	"io"
	"log"
	"net"
	"net/http"
	"regexp"
	"strconv"
	"strings"
	"time"
)

// how many URLs can the cache store
const cacheSize = 100

// how many hours an entry should be considered valid
const cacheValidHours = 24

// how many kilo bytes should be considered when looking for the title
// tag.
const httpReadKByte = 100

// abort HTTP requests if it takes longer than X seconds. Not sure, it’s
// definitely magic involved. Must be larger than 5.
const httpGetDeadline = 10

// don’t repost the same title within this period
const noRepostWithinSeconds = 30

// matches all whitespace and zero bytes. Additionally, all Unicode
// characters of class Cf (format chars, e.g. right-to-left) and Cc
// (control chars) are matched.
var whitespaceRegex = regexp.MustCompile(`[\s\0\p{Cf}\p{Cc}]+`)

var ignoreDomainsRegex = regexp.MustCompile(`^http://p\.nnev\.de`)

var twitterDomainRegex = regexp.MustCompile(`(?i)^https?://(?:[a-z0-9]\.)?twitter.com`)
var twitterPicsRegex = regexp.MustCompile(`(?i)(?:\b|^)pic\.twitter\.com/[a-z0-9]+(?:\b|$)`)

var noSpoilerRegex = regexp.MustCompile(`(?i)(don't|no|kein|nicht) spoiler`)

func UriFind(conn *irc.Conn, line *irc.Line) {
	defer func() {
		if r := recover(); r != nil {
			log.Printf("MEGA-WTF:pkg: %v", r)
		}
	}()

	msg := line.Args[1]

	if noSpoilerRegex.MatchString(msg) {
		log.Printf("not spoilering this line: %s", msg)
		return
	}

	urls := extract(msg)

	for _, url := range urls {
		if url == "" {
			continue
		}

		if title := cacheGetTitleByUrl(url); title != "" {
			log.Printf("using cache for URL: %s", url)
			postTitle(conn, line, title, "Cache Info")
			continue
		}

		go func(url string) {
			if ignoreDomainsRegex.MatchString(url) {
				log.Printf("ignoring this URL: %s", url)
				return
			}

			log.Printf("testing URL: %s", url)
			title, _, err := TitleGet(url)
			if err != nil {
				//postTitle(conn, line, err.Error(), "Error")
			} else if title != "" {
				postTitle(conn, line, title, "")
				cacheAdd(url, title)
			}
		}(url)
	}
}

// regexing ////////////////////////////////////////////////////////////

func extract(msg string) []string {
	results := make([]string, 0)
	for idx := strings.Index(msg, "http"); idx > -1; idx = strings.Index(msg, "http") {
		url := msg[idx:]
		if !strings.HasPrefix(url, "http://") &&
			!strings.HasPrefix(url, "https://") {
			msg = msg[idx+len("http"):]
			continue
		}

		// End on commas, but only if they are followed by a space.
		// spiegel.de URLs have commas in them, that would be a
		// false positive otherwise.
		if end := strings.Index(url, ", "); end > -1 {
			url = url[:end]
		}

		// End on closing paren, but only if there is an opening
		// paren before the URL (should fix most false-positives).
		if end := strings.Index(url, ")"); idx > 0 && msg[idx-1] == '(' && end > -1 {
			url = url[:end]
		}

		// Whitespace always ends a URL.
		if end := strings.IndexAny(url, " \t"); end > -1 {
			url = url[:end]
		}

		results = append(results, url)
		msg = msg[idx+len(url):]
	}
	return results
}

// http/html stuff /////////////////////////////////////////////////////

func TitleGet(url string) (string, string, error) {
	// via http://www.reddit.com/r/golang/comments/10awvj/timeout_on_httpget/c6bz49s
	c := http.Client{
		Transport: &http.Transport{
			Dial: func(netw, addr string) (net.Conn, error) {
				deadline := time.Now().Add(time.Second * httpGetDeadline)
				c, err := net.DialTimeout(netw, addr, time.Second*(httpGetDeadline-5))
				if err != nil {
					return nil, err
				}
				c.SetDeadline(deadline)
				return c, nil
			},
		},
	}

	r, err := c.Get(url)
	if err != nil {
		log.Printf("WTF: could not resolve %s: %s\n", url, err)
		return "", url, err
	}
	defer r.Body.Close()

	lastUrl := r.Request.URL.String()

	// TODO: r.Body → utf8?
	title, tweet := titleParseHtml(io.LimitReader(r.Body, 1024*httpReadKByte))

	if r.StatusCode != 200 {
		return "", lastUrl, errors.New("[" + strconv.Itoa(r.StatusCode) + "] " + title)
	}

	if tweet != "" && twitterDomainRegex.MatchString(lastUrl) {
		title = tweet
	}

	log.Printf("Title for URL %s: %s\n", url, title)

	return title, lastUrl, nil
}

// parses the incoming HTML fragment and tries to extract text from
// suitable tags. Currently this is the page’s title tag and tweets
// when the HTML-code is similar enough to twitter.com. Returns
// title and tweet.
func titleParseHtml(r io.Reader) (string, string) {
	doc, err := html.Parse(r)
	if err != nil {
		log.Printf("WTF: html parser blew up: %s\n", err)
		return "", ""
	}

	title := ""
	tweetText := ""
	tweetUser := ""
	tweetPicUrl := ""

	var f func(*html.Node)
	f = func(n *html.Node) {
		if title == "" && n.Type == html.ElementNode && n.DataAtom == atom.Title {
			title = extractText(n)
			return
		}

		if tweetText == "" && hasClass(n, "tweet-text") {
			tweetText = extractText(n)
			return
		}

		if tweetUser == "" && hasClass(n, "js-user-profile-link") {
			tweetUser = extractText(n)
			return
		}

		isMedia := hasClass(n, "media") || hasClass(n, "media-thumbnail")
		if tweetPicUrl == "" && isMedia && !hasClass(n, "profile-picture") {
			attrVal := getAttr(n, "data-url")
			if attrVal != "" {
				tweetPicUrl = attrVal
				return
			}
		}

		// recurse down
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			f(c)
		}

	}
	f(doc)

	// cleanup
	tweet := ""
	if tweetText != "" {
		tweetText = twitterPicsRegex.ReplaceAllString(tweetText, "")
		tweetUser = strings.Replace(tweetUser, "@", "(@", 1) + "): "
		tweet = tweetUser + tweetText + " " + tweetPicUrl
		tweet = clean(tweet)
	}

	return strings.TrimSpace(title), strings.TrimSpace(tweet)
}

func extractText(n *html.Node) string {
	text := ""
	for c := n.FirstChild; c != nil; c = c.NextSibling {
		if c.Type == html.TextNode {
			text += c.Data
		} else {
			text += extractText(c)
		}
	}
	return clean(text)
}

func hasClass(n *html.Node, class string) bool {
	if n.Type != html.ElementNode {
		return false
	}

	class = " " + strings.TrimSpace(class) + " "
	if strings.Contains(" "+getAttr(n, "class")+" ", class) {
		return true
	}
	return false
}

func getAttr(n *html.Node, findAttr string) string {
	for _, attr := range n.Attr {
		if attr.Key == findAttr {
			return attr.Val
		}
	}
	return ""
}

// Cache ///////////////////////////////////////////////////////////////
type Cache struct {
	url   string
	title string
	date  time.Time
}

var cache = [cacheSize]Cache{}
var cacheIndex = 0

func cacheAdd(url string, title string) {
	if len(cache) == cacheIndex {
		cacheIndex = 0
	}
	cache[cacheIndex] = Cache{url, title, time.Now()}
	cacheIndex += 1
}

func cacheGetTitleByUrl(url string) string {
	for _, cc := range cache {
		if cc.url == url && time.Since(cc.date).Hours() <= cacheValidHours {
			return cc.title
		}
	}
	return ""
}

func cacheGetSecondsToLastPost(title string) int {
	var secondsAgo = int(^uint(0) >> 1)
	for _, cc := range cache {
		var a = int(time.Since(cc.date).Seconds())
		if cc.title == title && a < secondsAgo {
			secondsAgo = a
		}
	}
	return secondsAgo
}

// util ////////////////////////////////////////////////////////////////

func postTitle(conn *irc.Conn, line *irc.Line, title string, prefix string) {
	tgt := line.Args[0]

	secondsAgo := cacheGetSecondsToLastPost(title)
	if secondsAgo <= noRepostWithinSeconds {
		log.Printf("Skipping, because posted %d seconds ago (“%s”)", secondsAgo, title)
		return
	}

	log.Printf("nick=%s, target=%s, title=%s", line.Nick, tgt, title)
	// if target is our current nick, it was a private message.
	// Answer the users in this case.
	if tgt == conn.Me().Nick {
		tgt = line.Nick
	}
	if prefix == "" {
		prefix = "Link Info"
	} else {
		prefix = clean(prefix)
	}
	title = clean(title)
	// the IRC spec states that notice should be used instead of msg
	// and that bots should not react to notice at all. However, no
	// real world bot adheres to this. Furthermore, people who can’t
	// configure their client to not highlight them on notices will
	// complain.
	conn.Privmsg(tgt, "["+prefix+"] "+title)
}

func clean(text string) string {
	text = whitespaceRegex.ReplaceAllString(text, " ")
	return strings.TrimSpace(text)
}
<|endoftext|>


<commit_before>package db

import (
	"fmt"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	"strconv"
	"strings"
	"time"
)

const (
	TRADB      = ".trago.db"
	bytes      = "abcdefghijklmnopqrstuvwxyz1234567890"
	currentDir = "./"
)

type TraDb struct {
	ReplicaId  string
	VersionVec map[string]int
	Files      map[string]FileState
}

type FileState struct {
	Size    int
	MTime   int64
	Version int
	Replica string
	// TODO: use a hash as well
}

func Parse(data string) (TraDb, error) {
	tradb := TraDb{}
	versionVector := make(map[string]int)

	tradb.Files = make(map[string]FileState)

	for _, line := range strings.Split(data, "\n") {
		line = strings.TrimSpace(line)

		if strings.HasPrefix(line, "#") {
			continue
		}

		fields := strings.Fields(line)
		if len(fields) == 0 {
			continue
		}

		switch fields[0] {
		case "file": // file name size mtime replica:version
			if len(fields) != 5 {
				continue
			}

			size, err := strconv.Atoi(fields[2])
			if err != nil {
				return tradb, err
			}

			mtime, err := strconv.ParseInt(fields[3], 10, 64)
			if err != nil {
				return tradb, err
			}

			pair := strings.Split(fields[4], ":")
			replicaId := pair[0]
			ver, err := strconv.Atoi(pair[1])
			if err != nil {
				return tradb, err
			}

			tradb.Files[fields[1]] = FileState{size, mtime, ver, replicaId}
		case "version": // version r1:v1 r2:v2 ...
			for _, entry := range fields[1:] {
				pair := strings.Split(entry, ":") // replica:version pair

				v, err := strconv.Atoi(pair[1])
				if err != nil {
					return tradb, err
				}

				versionVector[pair[0]] = v
			}
			tradb.VersionVec = versionVector

		case "replica": // replica replica-id
			if len(fields) != 2 {
				continue
			}
			tradb.ReplicaId = fields[1]
		}
	}

	return tradb, nil
}

func ParseFile() (TraDb, error) {
	tradb := TraDb{}

	dbfile, err := os.Open(TRADB)
	if os.IsNotExist(err) {
		log.Println("didn't find .trago.db")
		tradb = *New()
		tradb.Write()

		return tradb, nil
	} else if err != nil {
		return tradb, err
	}

	defer dbfile.Close()

	bs, err := ioutil.ReadFile(TRADB)
	if err != nil {
		return tradb, err
	}

	return Parse(string(bs))
}

func New() *TraDb {
	replicaId := make([]byte, 16)
	versionVector := make(map[string]int)

	rand.Seed(time.Now().UTC().UnixNano())
	for i, _ := range replicaId {
		replicaId[i] = bytes[rand.Intn(len(bytes))]
	}
	versionVector[string(replicaId)] = 1

	files, err := ioutil.ReadDir(currentDir)
	checkError(err)

	filemap := make(map[string]FileState)
	for _, file := range files {
		if file.IsDir() {
			continue // ignore directories for now
		}
		fs := FileState{
			Size:    int(file.Size()),
			MTime:   file.ModTime().UTC().UnixNano(),
			Version: 1,
			Replica: string(replicaId),
		}
		filemap[file.Name()] = fs
	}

	return &TraDb{string(replicaId), versionVector, filemap}
}

func (tradb *TraDb) Write() error {
	var pairs []string

	for replicaId, version := range tradb.VersionVec {
		entry := strings.Join([]string{replicaId, strconv.Itoa(version)}, ":")
		pairs = append(pairs, entry)
	}

	versionVector := strings.Join(pairs, " ")

	preamble := fmt.Sprintf(
		"replica %s\nversion %s\n# files\n",
		tradb.ReplicaId,
		versionVector,
	)

	fileEntries := make([]string, len(tradb.Files))

	i := 0
	for filename, info := range tradb.Files {
		fileEntries[i] = fmt.Sprintf(
			"file %s %d %d %s:%d",
			filename,
			info.Size,
			info.MTime,
			info.Replica,
			info.Version,
		)
		i = i + 1
	}

	entryString := strings.Join(fileEntries, "\n")
	dataToWrite := []byte(preamble + entryString)

	err := ioutil.WriteFile(TRADB, dataToWrite, 0644)
	return err
}

func (db *TraDb) Update() error {
	files, err := ioutil.ReadDir(currentDir)
	if err != nil {
		return err
	}

	for _, file := range files {
		if file.IsDir() {
			continue
		}

		filename := file.Name()
		dbRecord := db.Files[filename]
		if dbRecord.MTime == 0 {
			log.Printf("found a new file: %s\n", filename)
		} else if dbRecord.MTime < file.ModTime().UTC().UnixNano() {
			log.Printf("found an updated file: %s\n", filename)
			dbRecord.MTime = file.ModTime().UTC().UnixNano()
			dbRecord.Version = db.VersionVec[db.ReplicaId]
		} else {
			log.Printf("file unchanged: %s\n", filename)
		}
	}

	db.VersionVec[db.ReplicaId] += 1
	return nil
}

func (local *TraDb) Compare(remote *TraDb) {
	remoteFiles := remote.Files

	for file, state := range local.Files {
		remoteState := remoteFiles[file]

		if remoteState.Version == 0 { // file not present on server
			// TODO: download only if we have a more "recent" copy
			continue
		}

		if isFileChanged(state, remoteState) {
			if local.VersionVec[remoteState.Replica] >= remoteState.Version {
				continue // we already know about changes on remote
			} else if remote.VersionVec[state.Replica] >= state.Version {
				log.Printf("downloading: %s\n", file)
				continue
			} else {
				log.Printf("conflict: %s\n", file)
			}
		}
	}
}

func isFileChanged(fs1 FileState, fs2 FileState) bool {
	if fs1.MTime != fs2.MTime || fs1.Size != fs2.Size {
		return false
	}
	return true
}

func checkError(err error) {
	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>parse functions return pointers<commit_after>package db

import (
	"fmt"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	"strconv"
	"strings"
	"time"
)

const (
	TRADB      = ".trago.db"
	bytes      = "abcdefghijklmnopqrstuvwxyz1234567890"
	currentDir = "./"
)

type TraDb struct {
	ReplicaId  string
	VersionVec map[string]int
	Files      map[string]FileState
}

type FileState struct {
	Size    int
	MTime   int64
	Version int
	Replica string
	// TODO: use a hash as well
}

func Parse(data string) (*TraDb, error) {
	tradb := &TraDb{}
	versionVector := make(map[string]int)

	tradb.Files = make(map[string]FileState)

	for _, line := range strings.Split(data, "\n") {
		line = strings.TrimSpace(line)

		if strings.HasPrefix(line, "#") {
			continue
		}

		fields := strings.Fields(line)
		if len(fields) == 0 {
			continue
		}

		switch fields[0] {
		case "file": // file name size mtime replica:version
			if len(fields) != 5 {
				continue
			}

			size, err := strconv.Atoi(fields[2])
			if err != nil {
				return tradb, err
			}

			mtime, err := strconv.ParseInt(fields[3], 10, 64)
			if err != nil {
				return tradb, err
			}

			pair := strings.Split(fields[4], ":")
			replicaId := pair[0]
			ver, err := strconv.Atoi(pair[1])
			if err != nil {
				return tradb, err
			}

			tradb.Files[fields[1]] = FileState{size, mtime, ver, replicaId}
		case "version": // version r1:v1 r2:v2 ...
			for _, entry := range fields[1:] {
				pair := strings.Split(entry, ":") // replica:version pair

				v, err := strconv.Atoi(pair[1])
				if err != nil {
					return tradb, err
				}

				versionVector[pair[0]] = v
			}
			tradb.VersionVec = versionVector

		case "replica": // replica replica-id
			if len(fields) != 2 {
				continue
			}
			tradb.ReplicaId = fields[1]
		}
	}

	return tradb, nil
}

func ParseFile() (*TraDb, error) {
	tradb := &TraDb{}

	dbfile, err := os.Open(TRADB)
	if os.IsNotExist(err) {
		log.Println("didn't find .trago.db")
		tradb = New()
		tradb.Write()

		return tradb, nil
	} else if err != nil {
		return tradb, err
	}

	defer dbfile.Close()

	bs, err := ioutil.ReadFile(TRADB)
	if err != nil {
		return tradb, err
	}

	return Parse(string(bs))
}

func New() *TraDb {
	replicaId := make([]byte, 16)
	versionVector := make(map[string]int)

	rand.Seed(time.Now().UTC().UnixNano())
	for i, _ := range replicaId {
		replicaId[i] = bytes[rand.Intn(len(bytes))]
	}
	versionVector[string(replicaId)] = 1

	files, err := ioutil.ReadDir(currentDir)
	checkError(err)

	filemap := make(map[string]FileState)
	for _, file := range files {
		if file.IsDir() {
			continue // ignore directories for now
		}
		fs := FileState{
			Size:    int(file.Size()),
			MTime:   file.ModTime().UTC().UnixNano(),
			Version: 1,
			Replica: string(replicaId),
		}
		filemap[file.Name()] = fs
	}

	return &TraDb{string(replicaId), versionVector, filemap}
}

func (tradb *TraDb) Write() error {
	var pairs []string

	for replicaId, version := range tradb.VersionVec {
		entry := strings.Join([]string{replicaId, strconv.Itoa(version)}, ":")
		pairs = append(pairs, entry)
	}

	versionVector := strings.Join(pairs, " ")

	preamble := fmt.Sprintf(
		"replica %s\nversion %s\n# files\n",
		tradb.ReplicaId,
		versionVector,
	)

	fileEntries := make([]string, len(tradb.Files))

	i := 0
	for filename, info := range tradb.Files {
		fileEntries[i] = fmt.Sprintf(
			"file %s %d %d %s:%d",
			filename,
			info.Size,
			info.MTime,
			info.Replica,
			info.Version,
		)
		i = i + 1
	}

	entryString := strings.Join(fileEntries, "\n")
	dataToWrite := []byte(preamble + entryString)

	err := ioutil.WriteFile(TRADB, dataToWrite, 0644)
	return err
}

func (db *TraDb) Update() error {
	files, err := ioutil.ReadDir(currentDir)
	if err != nil {
		return err
	}

	for _, file := range files {
		if file.IsDir() {
			continue
		}

		filename := file.Name()
		dbRecord := db.Files[filename]
		if dbRecord.MTime == 0 {
			log.Printf("found a new file: %s\n", filename)
		} else if dbRecord.MTime < file.ModTime().UTC().UnixNano() {
			log.Printf("found an updated file: %s\n", filename)
			dbRecord.MTime = file.ModTime().UTC().UnixNano()
			dbRecord.Version = db.VersionVec[db.ReplicaId]
		} else {
			log.Printf("file unchanged: %s\n", filename)
		}
	}

	db.VersionVec[db.ReplicaId] += 1
	return nil
}

func (local *TraDb) Compare(remote *TraDb) {
	remoteFiles := remote.Files

	for file, state := range local.Files {
		remoteState := remoteFiles[file]

		if remoteState.Version == 0 { // file not present on server
			// TODO: download only if we have a more "recent" copy
			continue
		}

		if isFileChanged(state, remoteState) {
			if local.VersionVec[remoteState.Replica] >= remoteState.Version {
				continue // we already know about changes on remote
			} else if remote.VersionVec[state.Replica] >= state.Version {
				log.Printf("downloading: %s\n", file)
				continue
			} else {
				log.Printf("conflict: %s\n", file)
			}
		}
	}
}

func isFileChanged(fs1 FileState, fs2 FileState) bool {
	if fs1.MTime != fs2.MTime || fs1.Size != fs2.Size {
		return false
	}
	return true
}

func checkError(err error) {
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package config

import (
	"os"
	"sync"
	"time"

	"github.com/sirupsen/logrus"
)

// Agent watches a path and automatically loads the config stored
// therein.
type Agent struct {
	sync.Mutex
	c             *Config
	subscriptions []chan<- ConfigDelta
}

// Start will begin polling the config file at the path. If the first load
// fails, Start with return the error and abort. Future load failures will log
// the failure message but continue attempting to load.
func (ca *Agent) Start(prowConfig, jobConfig string) error {
	c, err := Load(prowConfig, jobConfig)
	if err != nil {
		return err
	}
	ca.c = c
	go func() {
		var lastModTime time.Time
		// Rarely, if two changes happen in the same second, mtime will
		// be the same for the second change, and an mtime-based check would
		// fail. Reload periodically just in case.
		skips := 0
		for range time.Tick(1 * time.Second) {
			if skips < 600 {
				// Check if the file changed to see if it needs to be re-read.
				// os.Stat follows symbolic links, which is how ConfigMaps work.
				prowStat, err := os.Stat(prowConfig)
				if err != nil {
					logrus.WithField("prowConfig", prowConfig).WithError(err).Error("Error loading prow config.")
					continue
				}

				recentModTime := prowStat.ModTime()

				// TODO(krzyzacy): allow empty jobConfig till fully migrate config to subdirs
				if jobConfig != "" {
					jobConfigStat, err := os.Stat(jobConfig)
					if err != nil {
						logrus.WithField("jobConfig", jobConfig).WithError(err).Error("Error loading job configs.")
						continue
					}

					if jobConfigStat.ModTime().After(recentModTime) {
						recentModTime = jobConfigStat.ModTime()
					}
				}

				if !recentModTime.After(lastModTime) {
					skips++
					continue // file hasn't been modified
				}
				lastModTime = recentModTime
			}
			if c, err := Load(prowConfig, jobConfig); err != nil {
				logrus.WithField("prowConfig", prowConfig).
					WithField("jobConfig", jobConfig).
					WithError(err).Error("Error loading config.")
			} else {
				skips = 0
				ca.Lock()
				delta := ConfigDelta{*ca.c, *c}
				for _, subscription := range ca.subscriptions {
					// we can't let unbuffered channels for subscriptions lock us up
					// here, so we will send events best-effort into the channels we have
					go func(out chan<- ConfigDelta) { out <- delta }(subscription)
				}
				ca.c = c
				ca.Unlock()
			}
		}
	}()
	return nil
}

type ConfigDelta struct {
	Before, After Config
}

// Subscribe registers the channel for messages on config reload.
// The caller can expect a copy of the previous and current config
// to be sent down the subscribed channel when a new configuration
// is loaded.
func (ca *Agent) Subscribe(subscription chan<- ConfigDelta) {
	ca.Lock()
	defer ca.Unlock()
	ca.subscriptions = append(ca.subscriptions, subscription)
}

// Config returns the latest config. Do not modify the config.
func (ca *Agent) Config() *Config {
	ca.Lock()
	defer ca.Unlock()
	return ca.c
}

// Set sets the config. Useful for testing.
func (ca *Agent) Set(c *Config) {
	ca.Lock()
	defer ca.Unlock()
	ca.c = c
}
<commit_msg>Move subscription update to Set()<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package config

import (
	"os"
	"sync"
	"time"

	"github.com/sirupsen/logrus"
)

// Agent watches a path and automatically loads the config stored
// therein.
type Agent struct {
	sync.Mutex
	c             *Config
	subscriptions []chan<- ConfigDelta
}

// Start will begin polling the config file at the path. If the first load
// fails, Start with return the error and abort. Future load failures will log
// the failure message but continue attempting to load.
func (ca *Agent) Start(prowConfig, jobConfig string) error {
	c, err := Load(prowConfig, jobConfig)
	if err != nil {
		return err
	}
	ca.Set(c)
	go func() {
		var lastModTime time.Time
		// Rarely, if two changes happen in the same second, mtime will
		// be the same for the second change, and an mtime-based check would
		// fail. Reload periodically just in case.
		skips := 0
		for range time.Tick(1 * time.Second) {
			if skips < 600 {
				// Check if the file changed to see if it needs to be re-read.
				// os.Stat follows symbolic links, which is how ConfigMaps work.
				prowStat, err := os.Stat(prowConfig)
				if err != nil {
					logrus.WithField("prowConfig", prowConfig).WithError(err).Error("Error loading prow config.")
					continue
				}

				recentModTime := prowStat.ModTime()

				// TODO(krzyzacy): allow empty jobConfig till fully migrate config to subdirs
				if jobConfig != "" {
					jobConfigStat, err := os.Stat(jobConfig)
					if err != nil {
						logrus.WithField("jobConfig", jobConfig).WithError(err).Error("Error loading job configs.")
						continue
					}

					if jobConfigStat.ModTime().After(recentModTime) {
						recentModTime = jobConfigStat.ModTime()
					}
				}

				if !recentModTime.After(lastModTime) {
					skips++
					continue // file hasn't been modified
				}
				lastModTime = recentModTime
			}
			if c, err := Load(prowConfig, jobConfig); err != nil {
				logrus.WithField("prowConfig", prowConfig).
					WithField("jobConfig", jobConfig).
					WithError(err).Error("Error loading config.")
			} else {
				skips = 0
				ca.Set(c)
			}
		}
	}()
	return nil
}

type ConfigDelta struct {
	Before, After Config
}

// Subscribe registers the channel for messages on config reload.
// The caller can expect a copy of the previous and current config
// to be sent down the subscribed channel when a new configuration
// is loaded.
func (ca *Agent) Subscribe(subscription chan<- ConfigDelta) {
	ca.Lock()
	defer ca.Unlock()
	ca.subscriptions = append(ca.subscriptions, subscription)
}

// Config returns the latest config. Do not modify the config.
func (ca *Agent) Config() *Config {
	ca.Lock()
	defer ca.Unlock()
	return ca.c
}

// Set sets the config. Useful for testing.
func (ca *Agent) Set(c *Config) {
	ca.Lock()
	defer ca.Unlock()
	var oldConfig Config
	if ca.c != nil {
		oldConfig = *ca.c
	}
	delta := ConfigDelta{oldConfig, *c}
	for _, subscription := range ca.subscriptions {
		// we can't let unbuffered channels for subscriptions lock us up
		// here, so we will send events best-effort into the channels we have
		go func(out chan<- ConfigDelta) { out <- delta }(subscription)
	}
	ca.c = c
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"os"

	"go.pedge.io/env"
	"go.pedge.io/proto/server"
	"go.pedge.io/protolog"
	"google.golang.org/grpc"
	kube "k8s.io/kubernetes/pkg/client/unversioned"

	"github.com/pachyderm/pachyderm"
	"github.com/pachyderm/pachyderm/src/pfs"
	"github.com/pachyderm/pachyderm/src/pps"
	"github.com/pachyderm/pachyderm/src/pps/jobserver"
	"github.com/pachyderm/pachyderm/src/pps/persist"
	persistserver "github.com/pachyderm/pachyderm/src/pps/persist/server"
	"github.com/pachyderm/pachyderm/src/pps/pipelineserver"
)

type appEnv struct {
	PachydermPfsd1Port string `env:"PACHYDERM_PFSD_1_PORT"`
	PfsAddress         string `env:"PFS_ADDRESS"`
	PfsMountDir        string `env:"PFS_MOUNT_DIR"`
	Address            string `env:"PPS_ADDRESS,default=0.0.0.0"`
	Port               int    `env:"PPS_PORT,default=651"`
	DatabaseAddress    string `env:"PPS_DATABASE_ADDRESS"`
	DatabaseName       string `env:"PPS_DATABASE_NAME,default=pachyderm"`
	DebugPort          int    `env:"PPS_TRACE_PORT,default=1051"`
	RemoveContainers   bool   `env:"PPS_REMOVE_CONTAINERS"`
}

func main() {
	env.Main(do, &appEnv{})
}

func do(appEnvObj interface{}) error {
	appEnv := appEnvObj.(*appEnv)
	rethinkAPIClient, err := getRethinkAPIClient(appEnv.DatabaseAddress, appEnv.DatabaseName)
	if err != nil {
		return err
	}
	pfsdAddress, err := getPfsdAddress()
	if err != nil {
		return err
	}
	clientConn, err := grpc.Dial(pfsdAddress, grpc.WithInsecure())
	if err != nil {
		return err
	}
	pfsAPIClient := pfs.NewAPIClient(clientConn)
	config := &kube.Config{
		Host:     "kubernetes",
		Insecure: true,
	}
	kubeClient, err := kube.New(config)
	if err != nil {
		protolog.Printf("Error creating kubernetes client: %s", err.Error())
	}
	jobAPIServer := jobserver.NewAPIServer(
		pfsAPIClient,
		rethinkAPIClient,
		kubeClient,
	)
	jobAPIClient := pps.NewLocalJobAPIClient(jobAPIServer)
	pipelineAPIServer := pipelineserver.NewAPIServer(pfsAPIClient, jobAPIClient, rethinkAPIClient)
	if err := pipelineAPIServer.Start(); err != nil {
		return err
	}
	return protoserver.Serve(
		uint16(appEnv.Port),
		func(s *grpc.Server) {
			pps.RegisterJobAPIServer(s, jobAPIServer)
			pps.RegisterPipelineAPIServer(s, pipelineAPIServer)
		},
		protoserver.ServeOptions{
			DebugPort: uint16(appEnv.DebugPort),
			Version:   pachyderm.Version,
		},
	)
}

func getRethinkAPIClient(address string, databaseName string) (persist.APIClient, error) {
	var err error
	if address == "" {
		address, err = getRethinkAddress()
		if err != nil {
			return nil, err
		}
	}
	if err := persistserver.InitDBs(address, databaseName); err != nil {
		return nil, err
	}
	rethinkAPIServer, err := persistserver.NewRethinkAPIServer(address, databaseName)
	if err != nil {
		return nil, err
	}
	return persist.NewLocalAPIClient(rethinkAPIServer), nil
}

func getRethinkAddress() (string, error) {
	rethinkAddr := os.Getenv("RETHINK_PORT_28015_TCP_ADDR")
	if rethinkAddr == "" {
		return "", errors.New("RETHINK_PORT_28015_TCP_ADDR not set")
	}
	return fmt.Sprintf("%s:28015", rethinkAddr), nil
}

func getPfsdAddress() (string, error) {
	pfsdAddr := os.Getenv("PFSD_PORT_650_TCP_ADDR")
	if pfsdAddr == "" {
		return "", errors.New("PFSD_PORT_650_TCP_ADDR not set")
	}
	return fmt.Sprintf("%s:650", pfsdAddr), nil
}
<commit_msg>pps can actually connect to kube.<commit_after>package main

import (
	"errors"
	"fmt"
	"os"

	"go.pedge.io/env"
	"go.pedge.io/proto/server"
	"go.pedge.io/protolog"
	"google.golang.org/grpc"
	kube "k8s.io/kubernetes/pkg/client/unversioned"

	"github.com/pachyderm/pachyderm"
	"github.com/pachyderm/pachyderm/src/pfs"
	"github.com/pachyderm/pachyderm/src/pps"
	"github.com/pachyderm/pachyderm/src/pps/jobserver"
	"github.com/pachyderm/pachyderm/src/pps/persist"
	persistserver "github.com/pachyderm/pachyderm/src/pps/persist/server"
	"github.com/pachyderm/pachyderm/src/pps/pipelineserver"
)

type appEnv struct {
	PachydermPfsd1Port string `env:"PACHYDERM_PFSD_1_PORT"`
	PfsAddress         string `env:"PFS_ADDRESS"`
	PfsMountDir        string `env:"PFS_MOUNT_DIR"`
	Address            string `env:"PPS_ADDRESS,default=0.0.0.0"`
	Port               int    `env:"PPS_PORT,default=651"`
	DatabaseAddress    string `env:"PPS_DATABASE_ADDRESS"`
	DatabaseName       string `env:"PPS_DATABASE_NAME,default=pachyderm"`
	DebugPort          int    `env:"PPS_TRACE_PORT,default=1051"`
	RemoveContainers   bool   `env:"PPS_REMOVE_CONTAINERS"`
}

func main() {
	env.Main(do, &appEnv{})
}

func do(appEnvObj interface{}) error {
	appEnv := appEnvObj.(*appEnv)
	rethinkAPIClient, err := getRethinkAPIClient(appEnv.DatabaseAddress, appEnv.DatabaseName)
	if err != nil {
		return err
	}
	pfsdAddress, err := getPfsdAddress()
	if err != nil {
		return err
	}
	clientConn, err := grpc.Dial(pfsdAddress, grpc.WithInsecure())
	if err != nil {
		return err
	}
	pfsAPIClient := pfs.NewAPIClient(clientConn)
	kubeAddr, err := getKubeAddress()
	if err != nil {
		return err
	}
	config := &kube.Config{
		Host:     kubeAddr,
		Insecure: true,
	}
	kubeClient, err := kube.New(config)
	if err != nil {
		protolog.Printf("Error creating kubernetes client: %s", err.Error())
	}
	jobAPIServer := jobserver.NewAPIServer(
		pfsAPIClient,
		rethinkAPIClient,
		kubeClient,
	)
	jobAPIClient := pps.NewLocalJobAPIClient(jobAPIServer)
	pipelineAPIServer := pipelineserver.NewAPIServer(pfsAPIClient, jobAPIClient, rethinkAPIClient)
	if err := pipelineAPIServer.Start(); err != nil {
		return err
	}
	return protoserver.Serve(
		uint16(appEnv.Port),
		func(s *grpc.Server) {
			pps.RegisterJobAPIServer(s, jobAPIServer)
			pps.RegisterPipelineAPIServer(s, pipelineAPIServer)
		},
		protoserver.ServeOptions{
			DebugPort: uint16(appEnv.DebugPort),
			Version:   pachyderm.Version,
		},
	)
}

func getRethinkAPIClient(address string, databaseName string) (persist.APIClient, error) {
	var err error
	if address == "" {
		address, err = getRethinkAddress()
		if err != nil {
			return nil, err
		}
	}
	if err := persistserver.InitDBs(address, databaseName); err != nil {
		return nil, err
	}
	rethinkAPIServer, err := persistserver.NewRethinkAPIServer(address, databaseName)
	if err != nil {
		return nil, err
	}
	return persist.NewLocalAPIClient(rethinkAPIServer), nil
}

func getRethinkAddress() (string, error) {
	rethinkAddr := os.Getenv("RETHINK_PORT_28015_TCP_ADDR")
	if rethinkAddr == "" {
		return "", errors.New("RETHINK_PORT_28015_TCP_ADDR not set")
	}
	return fmt.Sprintf("%s:28015", rethinkAddr), nil
}

func getPfsdAddress() (string, error) {
	pfsdAddr := os.Getenv("PFSD_PORT_650_TCP_ADDR")
	if pfsdAddr == "" {
		return "", errors.New("PFSD_PORT_650_TCP_ADDR not set")
	}
	return fmt.Sprintf("%s:650", pfsdAddr), nil
}

func getKubeAddress() (string, error) {
	kubedAddr := os.Getenv("KUBERNETES_PORT_443_TCP_ADDR")
	if kubedAddr == "" {
		return "", errors.New("KUBERNETES_PORT_443_TCP_ADDR not set")
	}
	return fmt.Sprintf("%s:443", kubedAddr), nil
}
<|endoftext|>


<commit_before>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Minimum mutator utilization (MMU) graphing.

package main

import (
	"encoding/json"
	"fmt"
	trace "internal/traceparser"
	"log"
	"math"
	"net/http"
	"strconv"
	"strings"
	"sync"
	"time"
)

func init() {
	http.HandleFunc("/mmu", httpMMU)
	http.HandleFunc("/mmuPlot", httpMMUPlot)
	http.HandleFunc("/mmuDetails", httpMMUDetails)
}

var utilFlagNames = map[string]trace.UtilFlags{
	"perProc":    trace.UtilPerProc,
	"stw":        trace.UtilSTW,
	"background": trace.UtilBackground,
	"assist":     trace.UtilAssist,
	"sweep":      trace.UtilSweep,
}

type mmuCacheEntry struct {
	init     sync.Once
	util     [][]trace.MutatorUtil
	mmuCurve *trace.MMUCurve
	err      error
}

var mmuCache struct {
	m    map[trace.UtilFlags]*mmuCacheEntry
	lock sync.Mutex
}

func init() {
	mmuCache.m = make(map[trace.UtilFlags]*mmuCacheEntry)
}

func getMMUCurve(r *http.Request) ([][]trace.MutatorUtil, *trace.MMUCurve, error) {
	var flags trace.UtilFlags
	for _, flagStr := range strings.Split(r.FormValue("flags"), "|") {
		flags |= utilFlagNames[flagStr]
	}

	mmuCache.lock.Lock()
	c := mmuCache.m[flags]
	if c == nil {
		c = new(mmuCacheEntry)
		mmuCache.m[flags] = c
	}
	mmuCache.lock.Unlock()

	c.init.Do(func() {
		tr, err := parseTrace()
		if err != nil {
			c.err = err
		} else {
			c.util = tr.MutatorUtilization(flags)
			c.mmuCurve = trace.NewMMUCurve(c.util)
		}
	})
	return c.util, c.mmuCurve, c.err
}

// httpMMU serves the MMU plot page.
func httpMMU(w http.ResponseWriter, r *http.Request) {
	http.ServeContent(w, r, "", time.Time{}, strings.NewReader(templMMU))
}

// httpMMUPlot serves the JSON data for the MMU plot.
func httpMMUPlot(w http.ResponseWriter, r *http.Request) {
	mu, mmuCurve, err := getMMUCurve(r)
	if err != nil {
		http.Error(w, fmt.Sprintf("failed to parse events: %v", err), http.StatusInternalServerError)
		return
	}

	var quantiles []float64
	for _, flagStr := range strings.Split(r.FormValue("flags"), "|") {
		if flagStr == "mut" {
			quantiles = []float64{0, 1 - .999, 1 - .99, 1 - .95}
			break
		}
	}

	// Find a nice starting point for the plot.
	xMin := time.Second
	for xMin > 1 {
		if mmu := mmuCurve.MMU(xMin); mmu < 0.0001 {
			break
		}
		xMin /= 1000
	}
	// Cover six orders of magnitude.
	xMax := xMin * 1e6
	// But no more than the length of the trace.
	minEvent, maxEvent := mu[0][0].Time, mu[0][len(mu[0])-1].Time
	for _, mu1 := range mu[1:] {
		if mu1[0].Time < minEvent {
			minEvent = mu1[0].Time
		}
		if mu1[len(mu1)-1].Time > maxEvent {
			maxEvent = mu1[len(mu1)-1].Time
		}
	}
	if maxMax := time.Duration(maxEvent - minEvent); xMax > maxMax {
		xMax = maxMax
	}
	// Compute MMU curve.
	logMin, logMax := math.Log(float64(xMin)), math.Log(float64(xMax))
	const samples = 100
	plot := make([][]float64, samples)
	for i := 0; i < samples; i++ {
		window := time.Duration(math.Exp(float64(i)/(samples-1)*(logMax-logMin) + logMin))
		if quantiles == nil {
			plot[i] = make([]float64, 2)
			plot[i][1] = mmuCurve.MMU(window)
		} else {
			plot[i] = make([]float64, 1+len(quantiles))
			copy(plot[i][1:], mmuCurve.MUD(window, quantiles))
		}
		plot[i][0] = float64(window)
	}

	// Create JSON response.
	err = json.NewEncoder(w).Encode(map[string]interface{}{"xMin": int64(xMin), "xMax": int64(xMax), "quantiles": quantiles, "curve": plot})
	if err != nil {
		log.Printf("failed to serialize response: %v", err)
		return
	}
}

var templMMU = `<!doctype html>

    <meta charset="utf-8">

Loading plot...

          View

          <input type="radio" name="view" id="system" checked>System
          ?Consider whole system utilization. For example, if one of four procs is available to the mutator, mutator utilization will be 0.25. This is the standard definition of an MMU.

          <input type="radio" name="view" id="perProc">Per-goroutine
          ?Consider per-goroutine utilization. When even one goroutine is interrupted by GC, mutator utilization is 0.

          Include

          <input type="checkbox" id="stw" checked>STW
          ?Stop-the-world stops all goroutines simultaneously.

          <input type="checkbox" id="background" checked>Background workers
          ?Background workers are GC-specific goroutines. 25% of the CPU is dedicated to background workers during GC.

          <input type="checkbox" id="assist" checked>Mark assist
          ?Mark assists are performed by allocation to prevent the mutator from outpacing GC.

          <input type="checkbox" id="sweep">Sweep
          ?Sweep reclaims unused memory between GCs. (Enabling this may be very slow.).

          Display

          <input type="checkbox" id="mut">Show percentiles
          ?Display percentile mutator utilization in addition to minimum. E.g., p99 MU drops the worst 1% of windows.

Select a point for details.

`

// httpMMUDetails serves details of an MMU graph at a particular window.
func httpMMUDetails(w http.ResponseWriter, r *http.Request) {
	_, mmuCurve, err := getMMUCurve(r)
	if err != nil {
		http.Error(w, fmt.Sprintf("failed to parse events: %v", err), http.StatusInternalServerError)
		return
	}

	windowStr := r.FormValue("window")
	window, err := strconv.ParseUint(windowStr, 10, 64)
	if err != nil {
		http.Error(w, fmt.Sprintf("failed to parse window parameter %q: %v", windowStr, err), http.StatusBadRequest)
		return
	}
	worst := mmuCurve.Examples(time.Duration(window), 10)

	// Construct a link for each window.
	var links []linkedUtilWindow
	for _, ui := range worst {
		links = append(links, newLinkedUtilWindow(ui, time.Duration(window)))
	}

	err = json.NewEncoder(w).Encode(links)
	if err != nil {
		log.Printf("failed to serialize trace: %v", err)
		return
	}
}

type linkedUtilWindow struct {
	trace.UtilWindow
	URL string
}

func newLinkedUtilWindow(ui trace.UtilWindow, window time.Duration) linkedUtilWindow {
	// Find the range containing this window.
	var r Range
	for _, r = range ranges {
		if r.EndTime > ui.Time {
			break
		}
	}
	return linkedUtilWindow{ui, fmt.Sprintf("%s#%v:%v", r.URL(), float64(ui.Time)/1e6, float64(ui.Time+int64(window))/1e6)}
}
<commit_msg>cmd/trace: notes on MMU view improvements<commit_after>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Minimum mutator utilization (MMU) graphing.

// TODO:
//
// In worst window list, show break-down of GC utilization sources
// (STW, assist, etc). Probably requires a different MutatorUtil
// representation.
//
// When a window size is selected, show a second plot of the mutator
// utilization distribution for that window size.
//
// Render plot progressively so rough outline is visible quickly even
// for very complex MUTs. Start by computing just a few window sizes
// and then add more window sizes.
//
// Consider using sampling to compute an approximate MUT. This would
// work by sampling the mutator utilization at randomly selected
// points in time in the trace to build an empirical distribution. We
// could potentially put confidence intervals on these estimates and
// render this progressively as we refine the distributions.

package main

import (
	"encoding/json"
	"fmt"
	trace "internal/traceparser"
	"log"
	"math"
	"net/http"
	"strconv"
	"strings"
	"sync"
	"time"
)

func init() {
	http.HandleFunc("/mmu", httpMMU)
	http.HandleFunc("/mmuPlot", httpMMUPlot)
	http.HandleFunc("/mmuDetails", httpMMUDetails)
}

var utilFlagNames = map[string]trace.UtilFlags{
	"perProc":    trace.UtilPerProc,
	"stw":        trace.UtilSTW,
	"background": trace.UtilBackground,
	"assist":     trace.UtilAssist,
	"sweep":      trace.UtilSweep,
}

type mmuCacheEntry struct {
	init     sync.Once
	util     [][]trace.MutatorUtil
	mmuCurve *trace.MMUCurve
	err      error
}

var mmuCache struct {
	m    map[trace.UtilFlags]*mmuCacheEntry
	lock sync.Mutex
}

func init() {
	mmuCache.m = make(map[trace.UtilFlags]*mmuCacheEntry)
}

func getMMUCurve(r *http.Request) ([][]trace.MutatorUtil, *trace.MMUCurve, error) {
	var flags trace.UtilFlags
	for _, flagStr := range strings.Split(r.FormValue("flags"), "|") {
		flags |= utilFlagNames[flagStr]
	}

	mmuCache.lock.Lock()
	c := mmuCache.m[flags]
	if c == nil {
		c = new(mmuCacheEntry)
		mmuCache.m[flags] = c
	}
	mmuCache.lock.Unlock()

	c.init.Do(func() {
		tr, err := parseTrace()
		if err != nil {
			c.err = err
		} else {
			c.util = tr.MutatorUtilization(flags)
			c.mmuCurve = trace.NewMMUCurve(c.util)
		}
	})
	return c.util, c.mmuCurve, c.err
}

// httpMMU serves the MMU plot page.
func httpMMU(w http.ResponseWriter, r *http.Request) {
	http.ServeContent(w, r, "", time.Time{}, strings.NewReader(templMMU))
}

// httpMMUPlot serves the JSON data for the MMU plot.
func httpMMUPlot(w http.ResponseWriter, r *http.Request) {
	mu, mmuCurve, err := getMMUCurve(r)
	if err != nil {
		http.Error(w, fmt.Sprintf("failed to parse events: %v", err), http.StatusInternalServerError)
		return
	}

	var quantiles []float64
	for _, flagStr := range strings.Split(r.FormValue("flags"), "|") {
		if flagStr == "mut" {
			quantiles = []float64{0, 1 - .999, 1 - .99, 1 - .95}
			break
		}
	}

	// Find a nice starting point for the plot.
	xMin := time.Second
	for xMin > 1 {
		if mmu := mmuCurve.MMU(xMin); mmu < 0.0001 {
			break
		}
		xMin /= 1000
	}
	// Cover six orders of magnitude.
	xMax := xMin * 1e6
	// But no more than the length of the trace.
	minEvent, maxEvent := mu[0][0].Time, mu[0][len(mu[0])-1].Time
	for _, mu1 := range mu[1:] {
		if mu1[0].Time < minEvent {
			minEvent = mu1[0].Time
		}
		if mu1[len(mu1)-1].Time > maxEvent {
			maxEvent = mu1[len(mu1)-1].Time
		}
	}
	if maxMax := time.Duration(maxEvent - minEvent); xMax > maxMax {
		xMax = maxMax
	}
	// Compute MMU curve.
	logMin, logMax := math.Log(float64(xMin)), math.Log(float64(xMax))
	const samples = 100
	plot := make([][]float64, samples)
	for i := 0; i < samples; i++ {
		window := time.Duration(math.Exp(float64(i)/(samples-1)*(logMax-logMin) + logMin))
		if quantiles == nil {
			plot[i] = make([]float64, 2)
			plot[i][1] = mmuCurve.MMU(window)
		} else {
			plot[i] = make([]float64, 1+len(quantiles))
			copy(plot[i][1:], mmuCurve.MUD(window, quantiles))
		}
		plot[i][0] = float64(window)
	}

	// Create JSON response.
	err = json.NewEncoder(w).Encode(map[string]interface{}{"xMin": int64(xMin), "xMax": int64(xMax), "quantiles": quantiles, "curve": plot})
	if err != nil {
		log.Printf("failed to serialize response: %v", err)
		return
	}
}

var templMMU = `<!doctype html>

    <meta charset="utf-8">

Loading plot...

          View

          <input type="radio" name="view" id="system" checked>System
          ?Consider whole system utilization. For example, if one of four procs is available to the mutator, mutator utilization will be 0.25. This is the standard definition of an MMU.

          <input type="radio" name="view" id="perProc">Per-goroutine
          ?Consider per-goroutine utilization. When even one goroutine is interrupted by GC, mutator utilization is 0.

          Include

          <input type="checkbox" id="stw" checked>STW
          ?Stop-the-world stops all goroutines simultaneously.

          <input type="checkbox" id="background" checked>Background workers
          ?Background workers are GC-specific goroutines. 25% of the CPU is dedicated to background workers during GC.

          <input type="checkbox" id="assist" checked>Mark assist
          ?Mark assists are performed by allocation to prevent the mutator from outpacing GC.

          <input type="checkbox" id="sweep">Sweep
          ?Sweep reclaims unused memory between GCs. (Enabling this may be very slow.).

          Display

          <input type="checkbox" id="mut">Show percentiles
          ?Display percentile mutator utilization in addition to minimum. E.g., p99 MU drops the worst 1% of windows.

Select a point for details.

`

// httpMMUDetails serves details of an MMU graph at a particular window.
func httpMMUDetails(w http.ResponseWriter, r *http.Request) {
	_, mmuCurve, err := getMMUCurve(r)
	if err != nil {
		http.Error(w, fmt.Sprintf("failed to parse events: %v", err), http.StatusInternalServerError)
		return
	}

	windowStr := r.FormValue("window")
	window, err := strconv.ParseUint(windowStr, 10, 64)
	if err != nil {
		http.Error(w, fmt.Sprintf("failed to parse window parameter %q: %v", windowStr, err), http.StatusBadRequest)
		return
	}
	worst := mmuCurve.Examples(time.Duration(window), 10)

	// Construct a link for each window.
	var links []linkedUtilWindow
	for _, ui := range worst {
		links = append(links, newLinkedUtilWindow(ui, time.Duration(window)))
	}

	err = json.NewEncoder(w).Encode(links)
	if err != nil {
		log.Printf("failed to serialize trace: %v", err)
		return
	}
}

type linkedUtilWindow struct {
	trace.UtilWindow
	URL string
}

func newLinkedUtilWindow(ui trace.UtilWindow, window time.Duration) linkedUtilWindow {
	// Find the range containing this window.
	var r Range
	for _, r = range ranges {
		if r.EndTime > ui.Time {
			break
		}
	}
	return linkedUtilWindow{ui, fmt.Sprintf("%s#%v:%v", r.URL(), float64(ui.Time)/1e6, float64(ui.Time+int64(window))/1e6)}
}
<|endoftext|>


<commit_before>package moh

import (
	"code.google.com/p/go.net/websocket"
	"log"
	"sync"
)

type Publisher struct {
	MessagingServer

	// Registered connections
	connections map[*connection]bool

	// Registered filters
	filters      map[string]([]*connection)
	filtersMutex sync.Mutex

	// Register requests from the connections
	register chan *connection

	// Unregister requests from the connections
	unregister chan *connection
}

func NewPublisher(addr string) (*Publisher, error) {
	s, err := NewClosableServer(addr)
	if err != nil {
		return nil, err
	}

	p := &Publisher{
		MessagingServer: *s,
		connections:     make(map[*connection]bool),
		filters:         make(map[string]([]*connection)),
		register:        make(chan *connection),
		unregister:      make(chan *connection),
	}

	p.Mux.Handle("/", p.makeWsHandler())

	go s.Serve() // Starts HTTP server
	go p.registrar()

	return p, nil
}

func (p *Publisher) Publish(key string, message []byte) {
	p.filtersMutex.Lock()
	defer p.filtersMutex.Unlock()

	connections, ok := p.filters[key]
	if !ok {
		log.Println("No matching filters")
		return
	}

	log.Println("Sending message to send channel")
	for _, c := range connections {
		select {
		case (*c).send <- message:
			log.Println("Message sent to send channel")
		default:
			// TODO remove from filters
			// delete(connections, c)
			// close(c.send)
			go c.ws.Close()
		}
	}
}

func (p *Publisher) Broadcast(message []byte) {
	for c := range p.connections {
		select {
		case c.send <- message:
		default:
			delete(p.connections, c)
			close(c.send)
			go c.ws.Close()
		}
	}
}

func (p *Publisher) makeWsHandler() websocket.Handler {
	return func(ws *websocket.Conn) {
		c := connection{
			ws:   ws,
			send: make(chan []byte, 256),
		}
		p.register <- &c
		defer func() { p.unregister <- &c }()
		go c.writer()
		c.reader(&p.filters, &p.filtersMutex)
	}
}

// registrar selects over register and unregister channels and updates connections map.
func (p *Publisher) registrar() {
	for {
		select {
		case c := <-p.register:
			p.connections[c] = true
		case c := <-p.unregister:
			delete(p.connections, c)
			close(c.send)
		}
	}
}

type connection struct {
	ws *websocket.Conn

	// Buffered channel of outbount messages
	send chan []byte
}

// reader reads the subscription requests from websocket and saves it in a map for accessing later.
func (c *connection) reader(filters *map[string]([]*connection), m *sync.Mutex) {
	for {
		var key string
		err := websocket.Message.Receive(c.ws, &key)
		if err != nil {
			log.Println("reader: Cannot receive message from websocket")
			break
		}
		log.Println("reader: Received a message from websocket")

		// TODO looks ugly, refactor below
		m.Lock()
		connections, ok := (*filters)[key]
		if !ok {
			log.Println("reader: no slice of connections for this key:", key)
			(*filters)[key] = make([]*connection, 0)
		}

		connections = (*filters)[key]
		(*filters)[key] = append(connections, c)

		log.Println("reader: filters after inserting connection for key:", *filters)
		m.Unlock()
	}
	c.ws.Close()
}

// writer writes the messages to the websocket from the send channel.
func (c *connection) writer() {
	for message := range c.send {
		err := websocket.Message.Send(c.ws, message)
		if err != nil {
			break
		}
	}
	c.ws.Close()
}
<commit_msg>fix syntax<commit_after>package moh

import (
	"code.google.com/p/go.net/websocket"
	"log"
	"sync"
)

type Publisher struct {
	MessagingServer

	// Registered connections
	connections map[*connection]bool

	// Registered filters
	filters      map[string]([]*connection)
	filtersMutex sync.Mutex

	// Register requests from the connections
	register chan *connection

	// Unregister requests from the connections
	unregister chan *connection
}

func NewPublisher(addr string) (*Publisher, error) {
	s, err := NewClosableServer(addr)
	if err != nil {
		return nil, err
	}

	p := &Publisher{
		MessagingServer: *s,
		connections:     make(map[*connection]bool),
		filters:         make(map[string]([]*connection)),
		register:        make(chan *connection),
		unregister:      make(chan *connection),
	}

	p.Mux.Handle("/", p.makeWsHandler())

	go s.Serve() // Starts HTTP server
	go p.registrar()

	return p, nil
}

func (p *Publisher) Publish(key string, message []byte) {
	p.filtersMutex.Lock()
	defer p.filtersMutex.Unlock()

	connections, ok := p.filters[key]
	if !ok {
		log.Println("No matching filters")
		return
	}

	log.Println("Sending message to send channel")
	for _, c := range connections {
		select {
		case c.send <- message:
			log.Println("Message sent to send channel")
		default:
			// TODO remove from filters
			// delete(connections, c)
			// close(c.send)
			go c.ws.Close()
		}
	}
}

func (p *Publisher) Broadcast(message []byte) {
	for c := range p.connections {
		select {
		case c.send <- message:
		default:
			delete(p.connections, c)
			close(c.send)
			go c.ws.Close()
		}
	}
}

func (p *Publisher) makeWsHandler() websocket.Handler {
	return func(ws *websocket.Conn) {
		c := connection{
			ws:   ws,
			send: make(chan []byte, 256),
		}
		p.register <- &c
		defer func() { p.unregister <- &c }()
		go c.writer()
		c.reader(&p.filters, &p.filtersMutex)
	}
}

// registrar selects over register and unregister channels and updates connections map.
func (p *Publisher) registrar() {
	for {
		select {
		case c := <-p.register:
			p.connections[c] = true
		case c := <-p.unregister:
			delete(p.connections, c)
			close(c.send)
		}
	}
}

type connection struct {
	ws *websocket.Conn

	// Buffered channel of outbount messages
	send chan []byte
}

// reader reads the subscription requests from websocket and saves it in a map for accessing later.
func (c *connection) reader(filters *map[string]([]*connection), m *sync.Mutex) {
	for {
		var key string
		err := websocket.Message.Receive(c.ws, &key)
		if err != nil {
			log.Println("reader: Cannot receive message from websocket")
			break
		}
		log.Println("reader: Received a message from websocket")

		// TODO looks ugly, refactor below
		m.Lock()
		connections, ok := (*filters)[key]
		if !ok {
			log.Println("reader: no slice of connections for this key:", key)
			(*filters)[key] = make([]*connection, 0)
		}

		connections = (*filters)[key]
		(*filters)[key] = append(connections, c)

		log.Println("reader: filters after inserting connection for key:", *filters)
		m.Unlock()
	}
	c.ws.Close()
}

// writer writes the messages to the websocket from the send channel.
func (c *connection) writer() {
	for message := range c.send {
		err := websocket.Message.Send(c.ws, message)
		if err != nil {
			break
		}
	}
	c.ws.Close()
}
<|endoftext|>


<commit_before>package rtsengine

import (
	"container/list"
	"fmt"
	"image"
)

/*
 Maintains the A* Pathing algorithm

*/

// AStarPathing will implement the A* pathing algorithm
// A simple description here: http://www.policyalmanac.org/games/aStarTutorial.htm
// Psuedocode here at the bottom of this: http://web.mit.edu/eranki/www/tutorials/search/
// https://github.com/beefsack/go-astar/blob/master/astar.go
type AStarPathing struct {
	// We need to only path-find one at a time otherwise
	// if we path-find as the world changes it will end badly.
	//muPathing sync.Mutex
}

// FindPath will find a path between source and destination Points and
// returns a list of Squares of the proper path.
// All coordinates in world coordinates (absolute coordinates) please.
func (path *AStarPathing) FindPath(pool *Pool, grid *Grid, source *image.Point, destination *image.Point) (*list.List, error) {

	// Check if both source and destination are not colliding
	if !grid.In(source) {
		return nil, fmt.Errorf("Source not in grid! (%d,%d)", source.X, source.Y)
	}

	/*
		if grid.Collision(source) {
			return nil, fmt.Errorf("Source collision! (%d,%d)", source.X, source.Y)
		}
	*/

	if !grid.In(destination) || grid.Collision(destination) {
		return nil, fmt.Errorf("Destination not in grid or collision! (%d,%d)", destination.X, destination.Y)
	}

	closedList := list.New()
	openList := list.New()

	// Starting square. 0 out the cost.
	q := pool.Squares(1)[0]
	q.F = 0
	q.G = 0
	q.H = 0
	q.Locus.X = source.X
	q.Locus.Y = source.Y
	q.Position = 0

	// Push onto the openlist to prime the pathing engine
	openList.PushFront(q)

	// While the open list is not empty
	for openList.Len() > 0 {
		//find the square with the least f on the open list, call it "q"
		//remove q from the open list
		q = path.leastF(openList)

		// generate q's 8 successors and set their parents to q
		successors := path.constructSuccessor(pool, q)
		for i, successor := range successors {

			// ensure it is in the grid and there isn't a collision
			if !grid.In(&successor.Locus) || grid.Collision(&successor.Locus) {
				pool.Free(successor)
				continue
			}

			//if successor is the goal, stop the search
			if destination.Eq(successor.Locus) {
				closedList.PushBack(q)
				closedList.PushBack(successor)
				path.FreeList(pool, openList)
				path.freeArray(pool, i+1, successors)
				return path.optimizePath(pool, closedList), nil
			}

			//successor.g = q.g + distance between successor and q
			D := grid.DistanceDiagonelShortcut(&q.Locus, &successor.Locus)
			successor.G = q.G + D

			// successor.h = distance from goal to successor
			successor.H = grid.DistanceDiagonelShortcut(&successor.Locus, destination)

			// successor.f = successor.g + successor.h
			successor.F = successor.G + successor.H

			//  if a square with the same position as successor is in the OPEN list
			//  exists and has a lower f than successor, skip this successor
			if path.skipSuccessor(successor, openList) {
				pool.Free(successor)
				continue
			}

			// if a square with the same position as successor is in the CLOSED list
			// exists has a lower f than successor, skip this successor
			if path.skipSuccessor(successor, closedList) {
				pool.Free(successor)
				continue
			}

			// otherwise, add the square to the open list
			openList.PushBack(successor)

		} // for successors

		// push q on the closed list
		closedList.PushBack(q)

	} // openList non empty

	// Free all the remaining successors in the open list.
	path.FreeList(pool, openList)

	return path.optimizePath(pool, closedList), nil
}

// freeArray will free all squares in array from i .. len(squares)-1
func (path *AStarPathing) freeArray(pool *Pool, i int, squares []*Square) {
	if i >= len(squares) {
		return
	}

	for ; i < len(squares); i++ {
		pool.Free(squares[i])
	}
}

// FreeList will free every Square in the list l
func (path *AStarPathing) FreeList(pool *Pool, l *list.List) {
	// Free all the remaining successors in the open list.
	for e := l.Front(); e != nil; e = e.Next() {
		pool.Free(e.Value.(*Square))
	}
}

// optimizePath will optimize the path list passed as a parameter. Any culled
// squares are freed from the pool.
//
// A path list will contain duplicates at each _position_. Thus you want to
// iterate over the list and remove duplicates at each _position_ leaving the
// square with the least F in the path list.
// For F ties only one is chosen.
func (path *AStarPathing) optimizePath(pool *Pool, l *list.List) *list.List {
	var m map[int]*Square

	m = make(map[int]*Square)
	for e := l.Front(); e != nil; e = e.Next() {
		square := e.Value.(*Square)

		p, ok := m[square.Position]

		if !ok {
			m[square.Position] = square
		} else {
			if p.F <= square.F {
				pool.Free(square)
			} else {
				m[square.Position] = square
				pool.Free(p)
			}
		}

	}
	result := list.New()

	length := len(m)
	for i := 0; i < length; i++ {
		result.PushBack(m[i])
	}

	return result
}

//

// skipSuccessor will scan list l and if the list l contains an element with a smaller
// F than the successor at the same position, returns TRUE.
func (path *AStarPathing) skipSuccessor(successor *Square, l *list.List) bool {
	for e := l.Front(); e != nil; e = e.Next() {
		square := e.Value.(*Square)

		if square.Position == successor.Position && square.F <= successor.F {
			return true
		}
	}

	return false
}

// constructSuccessor will construct 8 successors with parent q from the Pool pool.
func (path *AStarPathing) constructSuccessor(pool *Pool, q *Square) []*Square {
	// The successors are the adjoining squares with the source S
	// in the middle. See below. It moves clockwise. We index
	// from zero so index 0 is 1 below.
	// 1  2  3
	// 8  S  4
	// 7  6  5
	successors := pool.Squares(8)

	// 1
	successors[0].Locus.X = q.Locus.X - 1
	successors[0].Locus.Y = q.Locus.Y - 1
	successors[0].Parent = q
	successors[0].Position = q.Position + 1
	successors[0].F = 0
	successors[0].G = 0
	successors[0].H = 0

	// 2
	successors[1].Locus.X = q.Locus.X - 1
	successors[1].Parent = q
	successors[1].Position = q.Position + 1
	successors[1].F = 0
	successors[1].G = 0
	successors[1].H = 0

	// 3
	successors[2].Locus.X = q.Locus.X - 1
	successors[2].Locus.Y = q.Locus.Y + 1
	successors[2].Parent = q
	successors[2].Position = q.Position + 1
	successors[2].F = 0
	successors[2].G = 0
	successors[2].H = 0

	// 4
	successors[3].Locus.Y = q.Locus.Y + 1
	successors[3].Parent = q
	successors[3].Position = q.Position + 1
	successors[3].F = 0
	successors[3].G = 0
	successors[3].H = 0

	// 5
	successors[4].Locus.X = q.Locus.X + 1
	successors[4].Locus.Y = q.Locus.Y + 1
	successors[4].Parent = q
	successors[4].Position = q.Position + 1
	successors[4].F = 0
	successors[4].G = 0
	successors[4].H = 0

	// 6
	successors[5].Locus.X = q.Locus.X + 1
	successors[5].Parent = q
	successors[5].Position = q.Position + 1
	successors[5].F = 0
	successors[5].G = 0
	successors[5].H = 0

	// 7
	successors[6].Locus.X = q.Locus.X + 1
	successors[6].Locus.Y = q.Locus.Y - 1
	successors[6].Parent = q
	successors[6].Position = q.Position + 1
	successors[6].F = 0
	successors[6].G = 0
	successors[6].H = 0

	// 8
	successors[7].Locus.Y = q.Locus.Y - 1
	successors[7].Parent = q
	successors[7].Position = q.Position + 1
	successors[7].F = 0
	successors[7].G = 0
	successors[7].H = 0

	return successors
}

// leastF returns the Square with the least F within list l
// AND remove that Square from list l.
// Returns nil if no item exists.
func (path *AStarPathing) leastF(l *list.List) *Square {

	var leastSquare *Square
	var leastSquareE *list.Element
	for e := l.Front(); e != nil; e = e.Next() {
		square := e.Value.(*Square)
		if leastSquare == nil || square.F < leastSquare.F {
			leastSquare = square
			leastSquareE = e
		}
	}

	l.Remove(leastSquareE)
	return leastSquare
}
<commit_msg>Add comments<commit_after>package rtsengine

import (
	"container/list"
	"fmt"
	"image"
)

/*
 Maintains the A* Pathing algorithm

*/

// AStarPathing will implement the A* pathing algorithm
// A simple description here: http://www.policyalmanac.org/games/aStarTutorial.htm
// Psuedocode here at the bottom of this: http://web.mit.edu/eranki/www/tutorials/search/
// https://github.com/beefsack/go-astar/blob/master/astar.go
// Smoothing to avoid diagonels: http://www.gamasutra.com/view/feature/131505/toward_more_realistic_pathfinding.php?page=1
type AStarPathing struct {
	// We need to only path-find one at a time otherwise
	// if we path-find as the world changes it will end badly.
	//muPathing sync.Mutex
}

// FindPath will find a path between source and destination Points and
// returns a list of Squares of the proper path.
// All coordinates in world coordinates (absolute coordinates) please.
func (path *AStarPathing) FindPath(pool *Pool, grid *Grid, source *image.Point, destination *image.Point) (*list.List, error) {

	// Check if both source and destination are not colliding
	if !grid.In(source) {
		return nil, fmt.Errorf("Source not in grid! (%d,%d)", source.X, source.Y)
	}

	/*
		if grid.Collision(source) {
			return nil, fmt.Errorf("Source collision! (%d,%d)", source.X, source.Y)
		}
	*/

	if !grid.In(destination) || grid.Collision(destination) {
		return nil, fmt.Errorf("Destination not in grid or collision! (%d,%d)", destination.X, destination.Y)
	}

	closedList := list.New()
	openList := list.New()

	// Starting square. 0 out the cost.
	q := pool.Squares(1)[0]
	q.F = 0
	q.G = 0
	q.H = 0
	q.Locus.X = source.X
	q.Locus.Y = source.Y
	q.Position = 0

	// Push onto the openlist to prime the pathing engine
	openList.PushFront(q)

	// While the open list is not empty
	for openList.Len() > 0 {
		//find the square with the least f on the open list, call it "q"
		//remove q from the open list
		q = path.leastF(openList)

		// generate q's 8 successors and set their parents to q
		successors := path.constructSuccessor(pool, q)
		for i, successor := range successors {

			// ensure it is in the grid and there isn't a collision
			if !grid.In(&successor.Locus) || grid.Collision(&successor.Locus) {
				pool.Free(successor)
				continue
			}

			//if successor is the goal, stop the search
			if destination.Eq(successor.Locus) {
				closedList.PushBack(q)
				closedList.PushBack(successor)
				path.FreeList(pool, openList)
				path.freeArray(pool, i+1, successors)
				return path.optimizePath(pool, closedList), nil
			}

			//successor.g = q.g + distance between successor and q
			D := grid.DistanceDiagonelShortcut(&q.Locus, &successor.Locus)
			successor.G = q.G + D

			// successor.h = distance from goal to successor
			successor.H = grid.DistanceDiagonelShortcut(&successor.Locus, destination)

			// successor.f = successor.g + successor.h
			successor.F = successor.G + successor.H

			//  if a square with the same position as successor is in the OPEN list
			//  exists and has a lower f than successor, skip this successor
			if path.skipSuccessor(successor, openList) {
				pool.Free(successor)
				continue
			}

			// if a square with the same position as successor is in the CLOSED list
			// exists has a lower f than successor, skip this successor
			if path.skipSuccessor(successor, closedList) {
				pool.Free(successor)
				continue
			}

			// otherwise, add the square to the open list
			openList.PushBack(successor)

		} // for successors

		// push q on the closed list
		closedList.PushBack(q)

	} // openList non empty

	// Free all the remaining successors in the open list.
	path.FreeList(pool, openList)

	return path.optimizePath(pool, closedList), nil
}

// freeArray will free all squares in array from i .. len(squares)-1
func (path *AStarPathing) freeArray(pool *Pool, i int, squares []*Square) {
	if i >= len(squares) {
		return
	}

	for ; i < len(squares); i++ {
		pool.Free(squares[i])
	}
}

// FreeList will free every Square in the list l
func (path *AStarPathing) FreeList(pool *Pool, l *list.List) {
	// Free all the remaining successors in the open list.
	for e := l.Front(); e != nil; e = e.Next() {
		pool.Free(e.Value.(*Square))
	}
}

// optimizePath will optimize the path list passed as a parameter. Any culled
// squares are freed from the pool.
//
// A path list will contain duplicates at each _position_. Thus you want to
// iterate over the list and remove duplicates at each _position_ leaving the
// square with the least F in the path list.
// For F ties only one is chosen.
func (path *AStarPathing) optimizePath(pool *Pool, l *list.List) *list.List {
	var m map[int]*Square

	m = make(map[int]*Square)
	for e := l.Front(); e != nil; e = e.Next() {
		square := e.Value.(*Square)

		p, ok := m[square.Position]

		if !ok {
			m[square.Position] = square
		} else {
			if p.F <= square.F {
				pool.Free(square)
			} else {
				m[square.Position] = square
				pool.Free(p)
			}
		}

	}
	result := list.New()

	length := len(m)
	for i := 0; i < length; i++ {
		result.PushBack(m[i])
	}

	return result
}

//

// skipSuccessor will scan list l and if the list l contains an element with a smaller
// F than the successor at the same position, returns TRUE.
func (path *AStarPathing) skipSuccessor(successor *Square, l *list.List) bool {
	for e := l.Front(); e != nil; e = e.Next() {
		square := e.Value.(*Square)

		if square.Position == successor.Position && square.F <= successor.F {
			return true
		}
	}

	return false
}

// constructSuccessor will construct 8 successors with parent q from the Pool pool.
func (path *AStarPathing) constructSuccessor(pool *Pool, q *Square) []*Square {
	// The successors are the adjoining squares with the source S
	// in the middle. See below. It moves clockwise. We index
	// from zero so index 0 is 1 below.
	// 1  2  3
	// 8  S  4
	// 7  6  5
	successors := pool.Squares(8)

	// 1
	successors[0].Locus.X = q.Locus.X - 1
	successors[0].Locus.Y = q.Locus.Y - 1
	successors[0].Parent = q
	successors[0].Position = q.Position + 1
	successors[0].F = 0
	successors[0].G = 0
	successors[0].H = 0

	// 2
	successors[1].Locus.X = q.Locus.X - 1
	successors[1].Parent = q
	successors[1].Position = q.Position + 1
	successors[1].F = 0
	successors[1].G = 0
	successors[1].H = 0

	// 3
	successors[2].Locus.X = q.Locus.X - 1
	successors[2].Locus.Y = q.Locus.Y + 1
	successors[2].Parent = q
	successors[2].Position = q.Position + 1
	successors[2].F = 0
	successors[2].G = 0
	successors[2].H = 0

	// 4
	successors[3].Locus.Y = q.Locus.Y + 1
	successors[3].Parent = q
	successors[3].Position = q.Position + 1
	successors[3].F = 0
	successors[3].G = 0
	successors[3].H = 0

	// 5
	successors[4].Locus.X = q.Locus.X + 1
	successors[4].Locus.Y = q.Locus.Y + 1
	successors[4].Parent = q
	successors[4].Position = q.Position + 1
	successors[4].F = 0
	successors[4].G = 0
	successors[4].H = 0

	// 6
	successors[5].Locus.X = q.Locus.X + 1
	successors[5].Parent = q
	successors[5].Position = q.Position + 1
	successors[5].F = 0
	successors[5].G = 0
	successors[5].H = 0

	// 7
	successors[6].Locus.X = q.Locus.X + 1
	successors[6].Locus.Y = q.Locus.Y - 1
	successors[6].Parent = q
	successors[6].Position = q.Position + 1
	successors[6].F = 0
	successors[6].G = 0
	successors[6].H = 0

	// 8
	successors[7].Locus.Y = q.Locus.Y - 1
	successors[7].Parent = q
	successors[7].Position = q.Position + 1
	successors[7].F = 0
	successors[7].G = 0
	successors[7].H = 0

	return successors
}

// leastF returns the Square with the least F within list l
// AND remove that Square from list l.
// Returns nil if no item exists.
func (path *AStarPathing) leastF(l *list.List) *Square {

	var leastSquare *Square
	var leastSquareE *list.Element
	for e := l.Front(); e != nil; e = e.Next() {
		square := e.Value.(*Square)
		if leastSquare == nil || square.F < leastSquare.F {
			leastSquare = square
			leastSquareE = e
		}
	}

	l.Remove(leastSquareE)
	return leastSquare
}
<|endoftext|>