<commit_before>package rollbar

import (
	"bytes"
	"encoding/json"
	"fmt"
	"hash/adler32"
	"net/http"
	"net/url"
	"os"
	"reflect"
	"regexp"
	"runtime"
	"strings"
	"sync"
	"time"
)

const (
	NAME    = "go-rollbar"
	VERSION = "0.3.0"

	// Severity levels
	CRIT  = "critical"
	ERR   = "error"
	WARN  = "warning"
	INFO  = "info"
	DEBUG = "debug"

	FILTERED = "[FILTERED]"
)

var (
	// Rollbar access token. If this is blank, no errors will be reported to
	// Rollbar.
	Token = ""

	// All errors and messages will be submitted under this environment.
	Environment = "development"

	// API endpoint for Rollbar.
	Endpoint = "https://api.rollbar.com/api/1/item/"

	// Maximum number of errors allowed in the sending queue before we start
	// dropping new errors on the floor.
	Buffer = 1000

	// Filter GET and POST parameters from being sent to Rollbar.
	FilterFields = regexp.MustCompile("password|secret|token")

	// Output of error, by default stderr
	ErrorWriter = os.Stderr

	// Queue of messages to be sent.
	bodyChannel chan map[string]interface{}
	waitGroup   sync.WaitGroup
)

// Fields can be used to pass arbitrary data to the Rollbar API.
type Field struct {
	Name string
	Data interface{}
}

// -- Setup

func init() {
	bodyChannel = make(chan map[string]interface{}, Buffer)

	go func() {
		for body := range bodyChannel {
			post(body)
			waitGroup.Done()
		}
	}()
}

// -- Error reporting

// Error asynchronously sends an error to Rollbar with the given severity
// level. You can pass, optionally, custom Fields to be passed on to Rollbar.
func Error(level string, err error, fields ...*Field) {
	ErrorWithStackSkip(level, err, 1, fields...)
}

// ErrorWithStackSkip asynchronously sends an error to Rollbar with the given
// severity level and a given number of stack trace frames skipped. You can
// pass, optionally, custom Fields to be passed on to Rollbar.
func ErrorWithStackSkip(level string, err error, skip int, fields ...*Field) {
	stack := BuildStack(2 + skip)
	ErrorWithStack(level, err, stack, fields...)
}

// ErrorWithStack asynchronously sends and error to Rollbar with the given
// stacktrace and (optionally) custom Fields to be passed on to Rollbar.
func ErrorWithStack(level string, err error, stack Stack, fields ...*Field) {
	buildAndPushError(level, err, stack, fields...)
}

// RequestError asynchronously sends an error to Rollbar with the given
// severity level and request-specific information. You can pass, optionally,
// custom Fields to be passed on to Rollbar.
func RequestError(level string, r *http.Request, err error, fields ...*Field) {
	RequestErrorWithStackSkip(level, r, err, 1, fields...)
}

// RequestErrorWithStackSkip asynchronously sends an error to Rollbar with the
// given severity level and a given number of stack trace frames skipped, in
// addition to extra request-specific information. You can pass, optionally,
// custom Fields to be passed on to Rollbar.
func RequestErrorWithStackSkip(level string, r *http.Request, err error, skip int, fields ...*Field) {
	stack := BuildStack(2 + skip)
	RequestErrorWithStack(level, r, err, stack, fields...)
}

// RequestErrorWithStack asynchronously sends an error to Rollbar with the
// given severity level, request-specific information provided by the given
// http.Request, and a custom Stack. You You can pass, optionally, custom
// Fields to be passed on to Rollbar.
func RequestErrorWithStack(level string, r *http.Request, err error, stack Stack, fields ...*Field) {
	buildAndPushError(level, err, stack, &Field{Name: "request", Data: errorRequest(r)})
}

func buildError(level string, err error, stack Stack, fields ...*Field) map[string]interface{} {
	body := buildBody(level, err.Error())
	data := body["data"].(map[string]interface{})
	errBody, fingerprint := errorBody(err, stack)
	data["body"] = errBody
	data["fingerprint"] = fingerprint

	for _, field := range fields {
		data[field.Name] = field.Data
	}

	return body
}

func buildAndPushError(level string, err error, stack Stack, fields ...*Field) {
	push(buildError(level, err, stack, fields...))
}

// -- Message reporting

// Message asynchronously sends a message to Rollbar with the given severity
// level.
func Message(level string, msg string) {
	body := buildBody(level, msg)
	data := body["data"].(map[string]interface{})
	data["body"] = messageBody(msg)

	push(body)
}

// -- Misc.

// Wait will block until the queue of errors / messages is empty.
func Wait() {
	waitGroup.Wait()
}

// Build the main JSON structure that will be sent to Rollbar with the
// appropriate metadata.
func buildBody(level, title string) map[string]interface{} {
	timestamp := time.Now().Unix()
	hostname, _ := os.Hostname()

	return map[string]interface{}{
		"access_token": Token,
		"data": map[string]interface{}{
			"environment": Environment,
			"title":       title,
			"level":       level,
			"timestamp":   timestamp,
			"platform":    runtime.GOOS,
			"language":    "go",
			"server": map[string]interface{}{
				"host": hostname,
			},
			"notifier": map[string]interface{}{
				"name":    NAME,
				"version": VERSION,
			},
		},
	}
}

// errorBody generates a Rollbar error body with a given stack trace.
func errorBody(err error, stack Stack) (map[string]interface{}, string) {
	fingerprint := stack.Fingerprint()
	errBody := map[string]interface{}{
		"trace": map[string]interface{}{
			"frames": stack,
			"exception": map[string]interface{}{
				"class":   errorClass(err),
				"message": err.Error(),
			},
		},
	}
	return errBody, fingerprint
}

// errorRequest extracts details from a Request in a format that Rollbar
// accepts.
func errorRequest(r *http.Request) map[string]interface{} {
	cleanQuery := filterParams(r.URL.Query())

	return map[string]interface{}{
		"url":     r.URL.String(),
		"method":  r.Method,
		"headers": flattenValues(r.Header),

		// GET params
		"query_string": url.Values(cleanQuery).Encode(),
		"GET":          flattenValues(cleanQuery),

		// POST / PUT params
		"POST": flattenValues(filterParams(r.Form)),
	}
}

// filterParams filters sensitive information like passwords from being sent to
// Rollbar.
func filterParams(values map[string][]string) map[string][]string {
	for key, _ := range values {
		if FilterFields.Match([]byte(key)) {
			values[key] = []string{FILTERED}
		}
	}

	return values
}

func flattenValues(values map[string][]string) map[string]interface{} {
	result := make(map[string]interface{})

	for k, v := range values {
		if len(v) == 1 {
			result[k] = v[0]
		} else {
			result[k] = v
		}
	}

	return result
}

// Build a message inner-body for the given message string.
func messageBody(s string) map[string]interface{} {
	return map[string]interface{}{
		"message": map[string]interface{}{
			"body": s,
		},
	}
}

func errorClass(err error) string {
	class := reflect.TypeOf(err).String()
	if class == "" {
		return "panic"
	} else if class == "*errors.errorString" {
		checksum := adler32.Checksum([]byte(err.Error()))
		return fmt.Sprintf("{%x}", checksum)
	} else {
		return strings.TrimPrefix(class, "*")
	}
}

// -- POST handling

// Queue the given JSON body to be POSTed to Rollbar.
func push(body map[string]interface{}) {
	if len(bodyChannel) < Buffer {
		waitGroup.Add(1)
		bodyChannel <- body
	} else {
		stderr("buffer full, dropping error on the floor")
	}
}

// POST the given JSON body to Rollbar synchronously.
func post(body map[string]interface{}) {
	if len(Token) == 0 {
		stderr("empty token")
		return
	}

	jsonBody, err := json.Marshal(body)
	if err != nil {
		stderr("failed to encode payload: %s", err.Error())
		return
	}

	resp, err := http.Post(Endpoint, "application/json", bytes.NewReader(jsonBody))
	if err != nil {
		stderr("POST failed: %s", err.Error())
	} else if resp.StatusCode != 200 {
		stderr("received response: %s", resp.Status)
	}
	if resp != nil {
		resp.Body.Close()
	}
}

// -- stderr
func stderr(format string, args ...interface{}) {
	if ErrorWriter != nil {
		format = "Rollbar error: " + format + "\n"
		fmt.Fprintf(os.Stderr, format, args...)
	}
}
<commit_msg>Fix writer reference for error printing<commit_after>package rollbar

import (
	"bytes"
	"encoding/json"
	"fmt"
	"hash/adler32"
	"net/http"
	"net/url"
	"os"
	"reflect"
	"regexp"
	"runtime"
	"strings"
	"sync"
	"time"
)

const (
	NAME    = "go-rollbar"
	VERSION = "0.3.0"

	// Severity levels
	CRIT  = "critical"
	ERR   = "error"
	WARN  = "warning"
	INFO  = "info"
	DEBUG = "debug"

	FILTERED = "[FILTERED]"
)

var (
	// Rollbar access token. If this is blank, no errors will be reported to
	// Rollbar.
	Token = ""

	// All errors and messages will be submitted under this environment.
	Environment = "development"

	// API endpoint for Rollbar.
	Endpoint = "https://api.rollbar.com/api/1/item/"

	// Maximum number of errors allowed in the sending queue before we start
	// dropping new errors on the floor.
	Buffer = 1000

	// Filter GET and POST parameters from being sent to Rollbar.
	FilterFields = regexp.MustCompile("password|secret|token")

	// Output of error, by default stderr
	ErrorWriter = os.Stderr

	// Queue of messages to be sent.
	bodyChannel chan map[string]interface{}
	waitGroup   sync.WaitGroup
)

// Fields can be used to pass arbitrary data to the Rollbar API.
type Field struct {
	Name string
	Data interface{}
}

// -- Setup

func init() {
	bodyChannel = make(chan map[string]interface{}, Buffer)

	go func() {
		for body := range bodyChannel {
			post(body)
			waitGroup.Done()
		}
	}()
}

// -- Error reporting

// Error asynchronously sends an error to Rollbar with the given severity
// level. You can pass, optionally, custom Fields to be passed on to Rollbar.
func Error(level string, err error, fields ...*Field) {
	ErrorWithStackSkip(level, err, 1, fields...)
}

// ErrorWithStackSkip asynchronously sends an error to Rollbar with the given
// severity level and a given number of stack trace frames skipped. You can
// pass, optionally, custom Fields to be passed on to Rollbar.
func ErrorWithStackSkip(level string, err error, skip int, fields ...*Field) {
	stack := BuildStack(2 + skip)
	ErrorWithStack(level, err, stack, fields...)
}

// ErrorWithStack asynchronously sends and error to Rollbar with the given
// stacktrace and (optionally) custom Fields to be passed on to Rollbar.
func ErrorWithStack(level string, err error, stack Stack, fields ...*Field) {
	buildAndPushError(level, err, stack, fields...)
}

// RequestError asynchronously sends an error to Rollbar with the given
// severity level and request-specific information. You can pass, optionally,
// custom Fields to be passed on to Rollbar.
func RequestError(level string, r *http.Request, err error, fields ...*Field) {
	RequestErrorWithStackSkip(level, r, err, 1, fields...)
}

// RequestErrorWithStackSkip asynchronously sends an error to Rollbar with the
// given severity level and a given number of stack trace frames skipped, in
// addition to extra request-specific information. You can pass, optionally,
// custom Fields to be passed on to Rollbar.
func RequestErrorWithStackSkip(level string, r *http.Request, err error, skip int, fields ...*Field) {
	stack := BuildStack(2 + skip)
	RequestErrorWithStack(level, r, err, stack, fields...)
}

// RequestErrorWithStack asynchronously sends an error to Rollbar with the
// given severity level, request-specific information provided by the given
// http.Request, and a custom Stack. You You can pass, optionally, custom
// Fields to be passed on to Rollbar.
func RequestErrorWithStack(level string, r *http.Request, err error, stack Stack, fields ...*Field) {
	buildAndPushError(level, err, stack, &Field{Name: "request", Data: errorRequest(r)})
}

func buildError(level string, err error, stack Stack, fields ...*Field) map[string]interface{} {
	body := buildBody(level, err.Error())
	data := body["data"].(map[string]interface{})
	errBody, fingerprint := errorBody(err, stack)
	data["body"] = errBody
	data["fingerprint"] = fingerprint

	for _, field := range fields {
		data[field.Name] = field.Data
	}

	return body
}

func buildAndPushError(level string, err error, stack Stack, fields ...*Field) {
	push(buildError(level, err, stack, fields...))
}

// -- Message reporting

// Message asynchronously sends a message to Rollbar with the given severity
// level.
func Message(level string, msg string) {
	body := buildBody(level, msg)
	data := body["data"].(map[string]interface{})
	data["body"] = messageBody(msg)

	push(body)
}

// -- Misc.

// Wait will block until the queue of errors / messages is empty.
func Wait() {
	waitGroup.Wait()
}

// Build the main JSON structure that will be sent to Rollbar with the
// appropriate metadata.
func buildBody(level, title string) map[string]interface{} {
	timestamp := time.Now().Unix()
	hostname, _ := os.Hostname()

	return map[string]interface{}{
		"access_token": Token,
		"data": map[string]interface{}{
			"environment": Environment,
			"title":       title,
			"level":       level,
			"timestamp":   timestamp,
			"platform":    runtime.GOOS,
			"language":    "go",
			"server": map[string]interface{}{
				"host": hostname,
			},
			"notifier": map[string]interface{}{
				"name":    NAME,
				"version": VERSION,
			},
		},
	}
}

// errorBody generates a Rollbar error body with a given stack trace.
func errorBody(err error, stack Stack) (map[string]interface{}, string) {
	fingerprint := stack.Fingerprint()
	errBody := map[string]interface{}{
		"trace": map[string]interface{}{
			"frames": stack,
			"exception": map[string]interface{}{
				"class":   errorClass(err),
				"message": err.Error(),
			},
		},
	}
	return errBody, fingerprint
}

// errorRequest extracts details from a Request in a format that Rollbar
// accepts.
func errorRequest(r *http.Request) map[string]interface{} {
	cleanQuery := filterParams(r.URL.Query())

	return map[string]interface{}{
		"url":     r.URL.String(),
		"method":  r.Method,
		"headers": flattenValues(r.Header),

		// GET params
		"query_string": url.Values(cleanQuery).Encode(),
		"GET":          flattenValues(cleanQuery),

		// POST / PUT params
		"POST": flattenValues(filterParams(r.Form)),
	}
}

// filterParams filters sensitive information like passwords from being sent to
// Rollbar.
func filterParams(values map[string][]string) map[string][]string {
	for key, _ := range values {
		if FilterFields.Match([]byte(key)) {
			values[key] = []string{FILTERED}
		}
	}

	return values
}

func flattenValues(values map[string][]string) map[string]interface{} {
	result := make(map[string]interface{})

	for k, v := range values {
		if len(v) == 1 {
			result[k] = v[0]
		} else {
			result[k] = v
		}
	}

	return result
}

// Build a message inner-body for the given message string.
func messageBody(s string) map[string]interface{} {
	return map[string]interface{}{
		"message": map[string]interface{}{
			"body": s,
		},
	}
}

func errorClass(err error) string {
	class := reflect.TypeOf(err).String()
	if class == "" {
		return "panic"
	} else if class == "*errors.errorString" {
		checksum := adler32.Checksum([]byte(err.Error()))
		return fmt.Sprintf("{%x}", checksum)
	} else {
		return strings.TrimPrefix(class, "*")
	}
}

// -- POST handling

// Queue the given JSON body to be POSTed to Rollbar.
func push(body map[string]interface{}) {
	if len(bodyChannel) < Buffer {
		waitGroup.Add(1)
		bodyChannel <- body
	} else {
		stderr("buffer full, dropping error on the floor")
	}
}

// POST the given JSON body to Rollbar synchronously.
func post(body map[string]interface{}) {
	if len(Token) == 0 {
		stderr("empty token")
		return
	}

	jsonBody, err := json.Marshal(body)
	if err != nil {
		stderr("failed to encode payload: %s", err.Error())
		return
	}

	resp, err := http.Post(Endpoint, "application/json", bytes.NewReader(jsonBody))
	if err != nil {
		stderr("POST failed: %s", err.Error())
	} else if resp.StatusCode != 200 {
		stderr("received response: %s", resp.Status)
	}
	if resp != nil {
		resp.Body.Close()
	}
}

// -- stderr
func stderr(format string, args ...interface{}) {
	if ErrorWriter != nil {
		format = "Rollbar error: " + format + "\n"
		fmt.Fprintf(ErrorWriter, format, args...)
	}
}
<|endoftext|>


<commit_before>package arts

import (
	"golang.org/x/net/html"
	"golang.org/x/net/html/atom"
	"fmt"
	//"github.com/matrixik/goquery"
	"code.google.com/p/cascadia"
	"github.com/bcampbell/arts/arts/byline"
	"regexp"
	"strings"
)

var authorPats = struct {
	indicativeStartTextPat *regexp.Regexp
	//bylineIndicativeText   *regexp.Regexp
	likelyClassPat *regexp.Regexp
}{
	regexp.MustCompile(`(?i)^\s*(by|text by|posted by|written by|exclusive by|reviewed by|published by|von)\b[:]?\s*`),
	//regexp.MustCompile(`(?i)\s*\b(by|text by|posted by|written by|exclusive by|reviewed by|report|published by|photographs by|von)\b[:]?\s*`),
	regexp.MustCompile(`(?i)name|byline|by-line|by_line|author|writer|credits|storycredit|firma`),
}

var bylineContainerPats = struct {
	likelyClassPat      *regexp.Regexp
	asideSel            cascadia.Selector
	sidebarSel          cascadia.Selector
	standfirstPat       *regexp.Regexp
	articleHeaderSel    cascadia.Selector
	schemaOrgArticleSel cascadia.Selector
	commentPat          *regexp.Regexp
	cruftIndicative     *regexp.Regexp
}{
	regexp.MustCompile(`(?i)byline|by-line|by_line|author|writer|credits|storycredit|firma|entry-details`),
	cascadia.MustCompile("aside"),
	cascadia.MustCompile("#sidebar, #side"),
	regexp.MustCompile(`(?i)stand-first|standfirst|kicker|dek|articleTagline|tagline`), // also sub-heading, sub-hed, deck?
	cascadia.MustCompile("article header"),
	cascadia.MustCompile(`[itemscope][itemtype="http://schema.org/Article"]`),
	regexp.MustCompile(`(?i)\b(?:comment|disqus|livefyre|remark|conversation)\b`),
	regexp.MustCompile(`(?i)\b(?:combx|comment|community|disqus|livefyre|menu|remark|rss|shoutbox|sidebar|sponsor|ad-break|agegate|pagination|pager|popup|promo|sponsor|shopping|tweet|twitter|facebook)\b`),
}

func rateBylineContainerNode(c candidate) {
	el := c.node()

	// TEST: inside likely cruft? (sidebars, related-articles boxes etc)
	/*
		for _, n := range parentNodes(el) {
			if bylineContainerPats.cruftIndicative.MatchString(getAttr(n, "class")) || bylineContainerPats.cruftIndicative.MatchString(getAttr(n, "id")) {
				c.addPoints(-3, fmt.Sprintf("inside cruft '%s'", describeNode(n)))
			}
		}
	*/

	elClass := getAttr(el, "class")
	elId := getAttr(el, "id")

	// TEST: is cruft itself?
	if bylineContainerPats.cruftIndicative.MatchString(getAttr(el, "class")) || bylineContainerPats.cruftIndicative.MatchString(getAttr(el, "id")) {
		c.addPoints(-3, fmt.Sprintf("looks like cruft"))
	}

	// TEST: is it a standfirst?
	if bylineContainerPats.standfirstPat.MatchString(elClass + " " + elId) {
		c.addPoints(-3, fmt.Sprintf("looks like standfirst"))
	}

	// TEST: likely other indicators in class/id?
	if bylineContainerPats.likelyClassPat.MatchString(elClass) {
		c.addPoints(1, "indicative class")
	}
	if bylineContainerPats.likelyClassPat.MatchString(elId) {
		c.addPoints(1, "indicative id")
	}

	// TEST: Indicative text? (eg "By...")
	// TODO: this test needs to be much better
	/*
		if authorPats.indicativeStartTextPat.MatchString(c.txt()) {
			c.addPoints(0.2, "indicative text")
		}
	*/

	// TODO: TEST: contains/adjacent to date info
}

// rate node on how much it looks like an individual author
func rateAuthorNode(c candidate, contentNodes []*html.Node) {
	el := c.node()

	// TODO: handle updated uFormats: http://www.microformats.org/wiki/h-entry

	hentrySel := cascadia.MustCompile(".hentry")
	hcardSel := cascadia.MustCompile(".vcard")
	hcardAuthorSel := cascadia.MustCompile(".vcard.author")
	itemPropAuthorSel := cascadia.MustCompile(`[itemprop="author"]`)

	// likely-looking author urls
	goodUrlPat := regexp.MustCompile(`(?i)(^mailto:)|([/](columnistarchive|biography|profile|about|author[s]?|writer|i-author|authorinfo)[/])`)
	//    'bad_url': re.compile(r'([/](category|tag[s]?|topic[s]?|thema)[/])|(#comment[s]?$)', re.I),

	// TEST: marked up with hcard?
	if hcardSel.Match(el) {
		c.addPoints(2, "hcard")
	}

	// TEST: hatom author?
	if hcardAuthorSel.Match(el) {
		c.addPoints(2, "hatom author")
		if closest(el, hentrySel) != nil {
			c.addPoints(2, "inside hentry")
		}
	}

	rel := strings.TrimSpace(strings.ToLower(getAttr(el, "rel")))

	// TEST: rel="author"
	if rel == "author" {
		c.addPoints(2, "rel-author")
	}
	if rel == "tag" {
		c.addPoints(-2, "rel-tag")
	}
	if rel == "category" {
		c.addPoints(-2, "rel-category")
	}
	// TODO:
	// test: penalise if /tag/ /category/ /topic/ whatever link

	// TEST: schema.org author
	if itemPropAuthorSel.Match(el) {
		c.addPoints(2, `itemprop="author"`)
	}

	// TEST: likely other indicators in class/id?
	if authorPats.likelyClassPat.MatchString(getAttr(el, "class")) {
		c.addPoints(1, "indicative class")
	}
	if authorPats.likelyClassPat.MatchString(getAttr(el, "id")) {
		c.addPoints(1, "indicative id")
	}

	//    TEST: looks like a name?
	nameScore := rateName(c.txt())
	if nameScore != 0 {
		c.addPoints(nameScore, "looks-like-a-name score")
	}

	// TEST: indicative text ("by ..." etc)
	if authorPats.indicativeStartTextPat.MatchString(c.txt()) {
		c.addPoints(1, "indicative text")
	}

	// TODO:
	//  test: penalise for full sentence text (eg punctuation)
	//  test: penalise for stopwords ("about" etc)
	//  test: check for adjacent indicative text

	// TEST: likely-looking link?
	if el.DataAtom == atom.A {
		href := getAttr(el, "href")
		if goodUrlPat.MatchString(href) {
			c.addPoints(2, "likely-looking link")
		}
	}

	// TEST: inside content, but not at immediate top or bottom
	//	if getLinkDensity(el.Parent) < 0.75 {
	//		c.addPoints(-2, "in block of text")
	//	}

	// TODO: TEST: contains twitter id?
}

// TODO:
// - parse bylines ("By ... ...." etc)
// - check for bylines/dates at start of content (maybe content extraction should filter them out?)
// - better scoring on indicative text
// - de-dupe results
// - use <meta> tags to rate names
//   eg <meta name="DCSext.author" content="Martin Evans" />
// - stopwords for not-a-name list ("correspondant" etc)
func grabAuthors(root *html.Node, contentNodes []*html.Node, headlineNode *html.Node) []Author {
	dbug := Debug.AuthorsLogger
	var authors = candidateList{}
	var bylines = candidateList{}

	likelyElementSel := cascadia.MustCompile("a,p,span,div,li,h3,h4,h5,h6,td,strong")

	// get the set of elements between headline and content
	intervening := map[*html.Node]struct{}{}
	if len(contentNodes) > 0 {
		foo, err := interveningElements(headlineNode, contentNodes[0])
		if err == nil {
			for _, bar := range foo {
				intervening[bar] = struct{}{}
			}
		}
	}

	// looking for:
	//  - elements containing individual authors
	//  - elements that look like byline containers
	for _, el := range likelyElementSel.MatchAll(root) {
		earlyOut := false
		txt := compressSpace(getTextContent(el))
		if len(txt) >= 150 {
			earlyOut = true
		} else if len(txt) < 3 {
			earlyOut = true
		} else {
			// inside comment?
			// if so, just ignore.
			for _, parent := range parentNodes(el) {
				if bylineContainerPats.commentPat.MatchString(getAttr(parent, "class")) {
					earlyOut = true
				}
				if bylineContainerPats.commentPat.MatchString(getAttr(parent, "id")) {
					earlyOut = true
				}
			}
		}
		if earlyOut {
			continue
		}

		authorC := newStandardCandidate(el, txt)
		containerC := newStandardCandidate(el, txt)
		if _, got := intervening[el]; got {
			authorC.addPoints(1, "between headline and content")
			containerC.addPoints(1, "between headline and content")
		}

		// any good as an author?
		rateAuthorNode(authorC, contentNodes)

		if authorC.total() > 1 {
			authors = append(authors, authorC)
		}

		// any good as a container?
		rateBylineContainerNode(containerC)
		if containerC.total() > 0 {
			bylines = append(bylines, containerC)
		}
	}

	// run over all the author candidates, and give them credit for their parents
	/*
		for _, authorC := range authors {
			descendants := authors.descendants(authorC)
			if len(descendants) > 0 {
				descendants[len(descendants)-1].addPoints(float64(len(descendants)), "likely-looking parent(s)")
			}
		}
	*/

	// discard authors which contain others
	dbug.Printf("AUTHOR before culling: %d candidates\n", len(authors))
	for _, c := range authors {
		c.dump(dbug)
	}
	authors = cullNestedAuthors(authors)

	// PASS TWO: give containers credit for containing likely-looking authors
	for _, byline := range bylines {
		cnt := 0
		for _, author := range authors {
			if byline.node() == author.node() {
				//byline.addPoints(1, "also a likely-looking author")
			} else if contains(byline.node(), author.node()) {
				cnt += 1
			}
		}
		if cnt > 0 {
			byline.addPoints(1, fmt.Sprintf("contains likely-looking author(s)"))
		}
	}

	authors.Sort()
	bylines.Sort()

	// TODO: if no containers, promote best author if good enough?

	// extract authors inside best container
	dbug.Printf("AUTHOR (post cull): %d candidates\n", len(authors))
	for _, c := range authors {
		c.dump(dbug)
	}
	dbug.Printf("BYLINECONTAINERS: %d candidates\n", len(bylines))
	for _, c := range bylines {
		c.dump(dbug)
	}

	// if there's a 'best' byline container, then discard any authors outside that
	var bestContainer candidate
	if len(bylines) > 0 {
		bestContainer = bylines[0]
		// TODO:
		// if multiple top-scorers, check they agree.
		// if not, abort.
		authors = cullNonContainedAuthors(bestContainer, authors)
	} else {
		authors = candidateList{}
	}

	// if a container, but no authors, use the container as the author
	if len(authors) == 0 && bestContainer != nil && bestContainer.total() >= 2 {
		authors = append(authors, bestContainer)
	}

	// extract authors
	return extractAuthors(authors)
}

// cull out authors which contain others
func cullNestedAuthors(authors candidateList) candidateList {
	old := authors
	authors = make(candidateList, 0, len(old))

	for _, outer := range old {
		childCnt := 0
		for _, inner := range old {
			if contains(outer.node(), inner.node()) {
				childCnt++
			}
		}
		if childCnt == 0 {
			authors = append(authors, outer)
		}
	}
	return authors
}

func cullNonContainedAuthors(container candidate, authors candidateList) candidateList {
	kept := candidateList{}
	for _, authorC := range authors {
		if authorC.node() == container.node() || contains(container.node(), authorC.node()) {
			kept = append(kept, authorC)
		}
	}
	return kept
}

func extractAuthors(authors candidateList) []Author {

	extracted := make([]Author, 0)

	for _, authorC := range authors {
		for _, a := range byline.Parse(authorC.txt()) {
			// TODO: extract vcard stuff, email, rel-author etc etc
			extracted = append(extracted, Author{Name: a.Name, Email: a.Email})
		}
	}
	return extracted
}
<commit_msg>fix author-extraction crash when no headline found<commit_after>package arts

import (
	"fmt"
	"golang.org/x/net/html"
	"golang.org/x/net/html/atom"
	//"github.com/matrixik/goquery"
	"code.google.com/p/cascadia"
	"github.com/bcampbell/arts/arts/byline"
	"regexp"
	"strings"
)

var authorPats = struct {
	indicativeStartTextPat *regexp.Regexp
	//bylineIndicativeText   *regexp.Regexp
	likelyClassPat *regexp.Regexp
}{
	regexp.MustCompile(`(?i)^\s*(by|text by|posted by|written by|exclusive by|reviewed by|published by|von)\b[:]?\s*`),
	//regexp.MustCompile(`(?i)\s*\b(by|text by|posted by|written by|exclusive by|reviewed by|report|published by|photographs by|von)\b[:]?\s*`),
	regexp.MustCompile(`(?i)name|byline|by-line|by_line|author|writer|credits|storycredit|firma`),
}

var bylineContainerPats = struct {
	likelyClassPat      *regexp.Regexp
	asideSel            cascadia.Selector
	sidebarSel          cascadia.Selector
	standfirstPat       *regexp.Regexp
	articleHeaderSel    cascadia.Selector
	schemaOrgArticleSel cascadia.Selector
	commentPat          *regexp.Regexp
	cruftIndicative     *regexp.Regexp
}{
	regexp.MustCompile(`(?i)byline|by-line|by_line|author|writer|credits|storycredit|firma|entry-details`),
	cascadia.MustCompile("aside"),
	cascadia.MustCompile("#sidebar, #side"),
	regexp.MustCompile(`(?i)stand-first|standfirst|kicker|dek|articleTagline|tagline`), // also sub-heading, sub-hed, deck?
	cascadia.MustCompile("article header"),
	cascadia.MustCompile(`[itemscope][itemtype="http://schema.org/Article"]`),
	regexp.MustCompile(`(?i)\b(?:comment|disqus|livefyre|remark|conversation)\b`),
	regexp.MustCompile(`(?i)\b(?:combx|comment|community|disqus|livefyre|menu|remark|rss|shoutbox|sidebar|sponsor|ad-break|agegate|pagination|pager|popup|promo|sponsor|shopping|tweet|twitter|facebook)\b`),
}

func rateBylineContainerNode(c candidate) {
	el := c.node()

	// TEST: inside likely cruft? (sidebars, related-articles boxes etc)
	/*
		for _, n := range parentNodes(el) {
			if bylineContainerPats.cruftIndicative.MatchString(getAttr(n, "class")) || bylineContainerPats.cruftIndicative.MatchString(getAttr(n, "id")) {
				c.addPoints(-3, fmt.Sprintf("inside cruft '%s'", describeNode(n)))
			}
		}
	*/

	elClass := getAttr(el, "class")
	elId := getAttr(el, "id")

	// TEST: is cruft itself?
	if bylineContainerPats.cruftIndicative.MatchString(getAttr(el, "class")) || bylineContainerPats.cruftIndicative.MatchString(getAttr(el, "id")) {
		c.addPoints(-3, fmt.Sprintf("looks like cruft"))
	}

	// TEST: is it a standfirst?
	if bylineContainerPats.standfirstPat.MatchString(elClass + " " + elId) {
		c.addPoints(-3, fmt.Sprintf("looks like standfirst"))
	}

	// TEST: likely other indicators in class/id?
	if bylineContainerPats.likelyClassPat.MatchString(elClass) {
		c.addPoints(1, "indicative class")
	}
	if bylineContainerPats.likelyClassPat.MatchString(elId) {
		c.addPoints(1, "indicative id")
	}

	// TEST: Indicative text? (eg "By...")
	// TODO: this test needs to be much better
	/*
		if authorPats.indicativeStartTextPat.MatchString(c.txt()) {
			c.addPoints(0.2, "indicative text")
		}
	*/

	// TODO: TEST: contains/adjacent to date info
}

// rate node on how much it looks like an individual author
func rateAuthorNode(c candidate, contentNodes []*html.Node) {
	el := c.node()

	// TODO: handle updated uFormats: http://www.microformats.org/wiki/h-entry

	hentrySel := cascadia.MustCompile(".hentry")
	hcardSel := cascadia.MustCompile(".vcard")
	hcardAuthorSel := cascadia.MustCompile(".vcard.author")
	itemPropAuthorSel := cascadia.MustCompile(`[itemprop="author"]`)

	// likely-looking author urls
	goodUrlPat := regexp.MustCompile(`(?i)(^mailto:)|([/](columnistarchive|biography|profile|about|author[s]?|writer|i-author|authorinfo)[/])`)
	//    'bad_url': re.compile(r'([/](category|tag[s]?|topic[s]?|thema)[/])|(#comment[s]?$)', re.I),

	// TEST: marked up with hcard?
	if hcardSel.Match(el) {
		c.addPoints(2, "hcard")
	}

	// TEST: hatom author?
	if hcardAuthorSel.Match(el) {
		c.addPoints(2, "hatom author")
		if closest(el, hentrySel) != nil {
			c.addPoints(2, "inside hentry")
		}
	}

	rel := strings.TrimSpace(strings.ToLower(getAttr(el, "rel")))

	// TEST: rel="author"
	if rel == "author" {
		c.addPoints(2, "rel-author")
	}
	if rel == "tag" {
		c.addPoints(-2, "rel-tag")
	}
	if rel == "category" {
		c.addPoints(-2, "rel-category")
	}
	// TODO:
	// test: penalise if /tag/ /category/ /topic/ whatever link

	// TEST: schema.org author
	if itemPropAuthorSel.Match(el) {
		c.addPoints(2, `itemprop="author"`)
	}

	// TEST: likely other indicators in class/id?
	if authorPats.likelyClassPat.MatchString(getAttr(el, "class")) {
		c.addPoints(1, "indicative class")
	}
	if authorPats.likelyClassPat.MatchString(getAttr(el, "id")) {
		c.addPoints(1, "indicative id")
	}

	//    TEST: looks like a name?
	nameScore := rateName(c.txt())
	if nameScore != 0 {
		c.addPoints(nameScore, "looks-like-a-name score")
	}

	// TEST: indicative text ("by ..." etc)
	if authorPats.indicativeStartTextPat.MatchString(c.txt()) {
		c.addPoints(1, "indicative text")
	}

	// TODO:
	//  test: penalise for full sentence text (eg punctuation)
	//  test: penalise for stopwords ("about" etc)
	//  test: check for adjacent indicative text

	// TEST: likely-looking link?
	if el.DataAtom == atom.A {
		href := getAttr(el, "href")
		if goodUrlPat.MatchString(href) {
			c.addPoints(2, "likely-looking link")
		}
	}

	// TEST: inside content, but not at immediate top or bottom
	//	if getLinkDensity(el.Parent) < 0.75 {
	//		c.addPoints(-2, "in block of text")
	//	}

	// TODO: TEST: contains twitter id?
}

// TODO:
// - parse bylines ("By ... ...." etc)
// - check for bylines/dates at start of content (maybe content extraction should filter them out?)
// - better scoring on indicative text
// - de-dupe results
// - use <meta> tags to rate names
//   eg <meta name="DCSext.author" content="Martin Evans" />
// - stopwords for not-a-name list ("correspondant" etc)
func grabAuthors(root *html.Node, contentNodes []*html.Node, headlineNode *html.Node) []Author {
	dbug := Debug.AuthorsLogger
	var authors = candidateList{}
	var bylines = candidateList{}

	likelyElementSel := cascadia.MustCompile("a,p,span,div,li,h3,h4,h5,h6,td,strong")

	// get the set of elements between headline and content
	intervening := map[*html.Node]struct{}{}
	if headlineNode != nil && len(contentNodes) > 0 {
		foo, err := interveningElements(headlineNode, contentNodes[0])
		if err == nil {
			for _, bar := range foo {
				intervening[bar] = struct{}{}
			}
		}
	}

	// looking for:
	//  - elements containing individual authors
	//  - elements that look like byline containers
	for _, el := range likelyElementSel.MatchAll(root) {
		earlyOut := false
		txt := compressSpace(getTextContent(el))
		if len(txt) >= 150 {
			earlyOut = true
		} else if len(txt) < 3 {
			earlyOut = true
		} else {
			// inside comment?
			// if so, just ignore.
			for _, parent := range parentNodes(el) {
				if bylineContainerPats.commentPat.MatchString(getAttr(parent, "class")) {
					earlyOut = true
				}
				if bylineContainerPats.commentPat.MatchString(getAttr(parent, "id")) {
					earlyOut = true
				}
			}
		}
		if earlyOut {
			continue
		}

		authorC := newStandardCandidate(el, txt)
		containerC := newStandardCandidate(el, txt)
		if _, got := intervening[el]; got {
			authorC.addPoints(1, "between headline and content")
			containerC.addPoints(1, "between headline and content")
		}

		// any good as an author?
		rateAuthorNode(authorC, contentNodes)

		if authorC.total() > 1 {
			authors = append(authors, authorC)
		}

		// any good as a container?
		rateBylineContainerNode(containerC)
		if containerC.total() > 0 {
			bylines = append(bylines, containerC)
		}
	}

	// run over all the author candidates, and give them credit for their parents
	/*
		for _, authorC := range authors {
			descendants := authors.descendants(authorC)
			if len(descendants) > 0 {
				descendants[len(descendants)-1].addPoints(float64(len(descendants)), "likely-looking parent(s)")
			}
		}
	*/

	// discard authors which contain others
	dbug.Printf("AUTHOR before culling: %d candidates\n", len(authors))
	for _, c := range authors {
		c.dump(dbug)
	}
	authors = cullNestedAuthors(authors)

	// PASS TWO: give containers credit for containing likely-looking authors
	for _, byline := range bylines {
		cnt := 0
		for _, author := range authors {
			if byline.node() == author.node() {
				//byline.addPoints(1, "also a likely-looking author")
			} else if contains(byline.node(), author.node()) {
				cnt += 1
			}
		}
		if cnt > 0 {
			byline.addPoints(1, fmt.Sprintf("contains likely-looking author(s)"))
		}
	}

	authors.Sort()
	bylines.Sort()

	// TODO: if no containers, promote best author if good enough?

	// extract authors inside best container
	dbug.Printf("AUTHOR (post cull): %d candidates\n", len(authors))
	for _, c := range authors {
		c.dump(dbug)
	}
	dbug.Printf("BYLINECONTAINERS: %d candidates\n", len(bylines))
	for _, c := range bylines {
		c.dump(dbug)
	}

	// if there's a 'best' byline container, then discard any authors outside that
	var bestContainer candidate
	if len(bylines) > 0 {
		bestContainer = bylines[0]
		// TODO:
		// if multiple top-scorers, check they agree.
		// if not, abort.
		authors = cullNonContainedAuthors(bestContainer, authors)
	} else {
		authors = candidateList{}
	}

	// if a container, but no authors, use the container as the author
	if len(authors) == 0 && bestContainer != nil && bestContainer.total() >= 2 {
		authors = append(authors, bestContainer)
	}

	// extract authors
	return extractAuthors(authors)
}

// cull out authors which contain others
func cullNestedAuthors(authors candidateList) candidateList {
	old := authors
	authors = make(candidateList, 0, len(old))

	for _, outer := range old {
		childCnt := 0
		for _, inner := range old {
			if contains(outer.node(), inner.node()) {
				childCnt++
			}
		}
		if childCnt == 0 {
			authors = append(authors, outer)
		}
	}
	return authors
}

func cullNonContainedAuthors(container candidate, authors candidateList) candidateList {
	kept := candidateList{}
	for _, authorC := range authors {
		if authorC.node() == container.node() || contains(container.node(), authorC.node()) {
			kept = append(kept, authorC)
		}
	}
	return kept
}

func extractAuthors(authors candidateList) []Author {

	extracted := make([]Author, 0)

	for _, authorC := range authors {
		for _, a := range byline.Parse(authorC.txt()) {
			// TODO: extract vcard stuff, email, rel-author etc etc
			extracted = append(extracted, Author{Name: a.Name, Email: a.Email})
		}
	}
	return extracted
}
<|endoftext|>


<commit_before>package asana

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"

	"github.com/manishrjain/asanawarrior/x"
	"github.com/pkg/errors"
)

var token = flag.String("token", "", "Token provided by Asana.")
var domain = flag.String("domain", "", "Workspace name, generally your domain name in Asana.")
var cache *acache = new(acache)

const (
	prefix = "https://app.asana.com/api/1.0"
	stamp  = "2006-01-02T15:04:05.999Z"
)

func runGetter(i interface{}, suffix string, fields ...string) error {
GETLOOP:
	url := fmt.Sprintf("%s/%s?opt_fields=%s", prefix, suffix, strings.Join(fields, ","))
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		log.Fatal(err)
	}
	req.Header.Add("Authorization", "Bearer "+*token)
	client := &http.Client{
		Timeout: 10 * time.Minute,
	}
	resp, err := client.Do(req)
	if err != nil {
		log.Printf("runGetter url: [%v] err: [%v]", url, err)
		time.Sleep(5 * time.Second)
		goto GETLOOP
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}
	if err := json.Unmarshal(body, i); err != nil {
		return err
	}
	return nil
}

type Basic struct {
	Id    uint64 `json:"id"`
	Name  string `json:"name"`
	Email string `json:"email"`
}
type BasicData struct {
	Data []Basic `json:"data"`
}
type BasicDataOne struct {
	Data Basic `json:"data"`
}

func getVarious(suffix string, opts ...string) ([]Basic, error) {
	var bd BasicData
	if err := runGetter(&bd, suffix, opts...); err != nil {
		return nil, err
	}
	return bd.Data, nil
}

type psec struct {
	Project Basic `json:"project"`
	Section Basic `json:"section"`
}

type task struct {
	Basic
	Assignee    Basic   `json:"assignee"`
	Tags        []Basic `json:"tags"`
	CompletedAt string  `json:"completed_at"`
	ModifiedAt  string  `json:"modified_at"`
	CreatedAt   string  `json:"created_at"`
	Memberships []psec  `json:"memberships"`
}

type tasks struct {
	Data []task `json:"data"`
}

type oneTask struct {
	Data task `json:"data"`
}

func convert(tsk task, proj, section string) (x.WarriorTask, error) {
	e := x.WarriorTask{}

	mts, err := time.Parse(stamp, tsk.ModifiedAt)
	if err != nil {
		return e, errors.Wrap(err, "asana modified at")
	}
	cts, err := time.Parse(stamp, tsk.CreatedAt)
	if err != nil {
		return e, errors.Wrap(err, "asana created at")
	}
	var dts time.Time
	if len(tsk.CompletedAt) > 0 {
		dts, err = time.Parse(stamp, tsk.CompletedAt)
		if err != nil {
			return e, errors.Wrap(err, "asana completed at")
		}
	}

	wt := x.WarriorTask{
		Name:      tsk.Name,
		Project:   proj,
		Xid:       tsk.Id,
		Assignee:  cache.User(tsk.Assignee.Id),
		Modified:  mts,
		Created:   cts,
		Completed: dts,
		Section:   section,
	}
	for _, tag := range tsk.Tags {
		wt.Tags = append(wt.Tags, cache.Tag(tag.Id))
	}
	return wt, nil
}

func getTasks(proj Basic, out chan x.WarriorTask, errc chan error) {
	var sectionName string
	var t tasks
	if err := runGetter(&t, fmt.Sprintf("projects/%d/tasks", proj.Id),
		"assignee,name,tags,completed_at,modified_at,created_at"); err != nil {
		errc <- errors.Wrapf(err, "getTasks for project: %v", proj.Name)
		return
	}

	for _, tsk := range t.Data {
		if len(tsk.Name) == 0 {
			// Don't sync such tasks.
			continue
		}
		if strings.HasSuffix(tsk.Name, ":") {
			sec := Basic{
				Id:   tsk.Id,
				Name: tsk.Name,
			}
			sectionName = cache.AddSection(proj.Id, sec)
			continue
		}

		wt, err := convert(tsk, proj.Name, sectionName)
		if err != nil {
			errc <- errors.Wrapf(err, "convert: getTasks for project: %v", proj.Name)
			return
		}
		out <- wt
	}
	errc <- nil
}

func GetTasks() ([]x.WarriorTask, error) {
	if err := cache.update(); err != nil {
		return nil, errors.Wrap(err, "cache.update")
	}

	out := make(chan x.WarriorTask, 100)
	projects := cache.Projects()
	errc := make(chan error, len(projects))
	for _, proj := range projects {
		go getTasks(proj, out, errc)
	}

	wtasks := make([]x.WarriorTask, 0, 100)
	done := make(chan struct{})
	go func() {
		for wt := range out {
			wtasks = append(wtasks, wt)
		}
		done <- struct{}{}
	}()

	var rerr error
	for _ = range projects {
		if err := <-errc; err != nil {
			rerr = err
		}
	}
	close(out)
	<-done // Wait for all tasks to be picked up by goroutine.
	return wtasks, rerr
}

// runPost would run a PUT or POST to Asana. No locks should be acquired.
func runPost(method, suffix string, values url.Values) ([]byte, error) {
POSTLOOP:
	url := fmt.Sprintf("%s/%s", prefix, suffix)
	fmt.Println(url, values.Encode())
	req, err := http.NewRequest(method, url, bytes.NewBufferString(values.Encode()))
	if err != nil {
		log.Fatal(errors.Wrap(err, "runPost http.NewRequest"))
	}

	req.Header.Add("Authorization", "Bearer "+*token)
	req.Header.Add("content-type", "application/x-www-form-urlencoded")
	client := &http.Client{
		Timeout: 10 * time.Minute,
	}
	resp, err := client.Do(req)
	if err != nil {
		log.Printf("runPost url: [%v] err: [%v]", url, err)
		time.Sleep(5 * time.Second)
		goto POSTLOOP
	}
	defer resp.Body.Close()

	return ioutil.ReadAll(resp.Body)
}

func toTagIds(tnames []string) []string {
	var tags []string
	for _, t := range tnames {
		tid := cache.TagId(t)
		if tid == 0 {
			tid = cache.CreateTag(t)
			fmt.Printf("New Tag created. ID: %d", tid)
		}
		if tid > 0 {
			tags = append(tags, strconv.FormatUint(tid, 10))
		}
	}
	return tags
}

func removeProject(tid, pid uint64) error {
	v := url.Values{}
	v.Add("project", strconv.FormatUint(pid, 10))
	_, err := runPost("POST", fmt.Sprintf("tasks/%d/removeProject", tid), v)
	return err
}

func updateSection(tid, pid uint64, section string) error {
	v := url.Values{}
	v.Add("project", strconv.FormatUint(pid, 10))

	sid := cache.SectionId(pid, section)
	if sid > 0 {
		v.Add("section", strconv.FormatUint(sid, 10))
	}

	_, err := runPost("POST", fmt.Sprintf("tasks/%d/addProject", tid), v)
	return err
}

func AddNew(wt x.WarriorTask) (x.WarriorTask, error) {
	e := x.WarriorTask{}

	// Ensure that project actually exists before proceeding.
	pid := cache.ProjectId(wt.Project)
	if pid == 0 {
		return e, fmt.Errorf("Project not found: %v", wt.Project)
	}

	v := url.Values{}
	v.Add("workspace", strconv.FormatUint(cache.Workspace(), 10))
	v.Add("name", wt.Name)
	fmt.Println("Assignee", wt.Assignee)
	aid := cache.UserId(wt.Assignee)
	if aid > 0 {
		v.Add("assignee", strconv.FormatUint(aid, 10))
	}

	tags := toTagIds(wt.Tags)
	v.Add("tags", strings.Join(tags, ","))
	resp, err := runPost("POST", "tasks", v)
	if err != nil {
		return e, errors.Wrap(err, "AddNew runPost")
	}
	fmt.Println(string(resp))

	var ot oneTask
	if err := json.Unmarshal(resp, &ot); err != nil {
		return e, errors.Wrap(err, "AddNew unmarshal")
	}
	if ot.Data.Id == 0 {
		return e, fmt.Errorf("Unable to find ID assigned by Asana: %+v", ot.Data)
	}

	// Now set the project and section.
	if err := updateSection(ot.Data.Id, pid, wt.Section); err != nil {
		return e, errors.Wrap(err, "AddNew updateSection")
	}

	// Now retrieve the task back again so we can sync it up with TW.
	return GetOneTask(ot.Data.Id)
}

func diff(t1 []string, t2 []string) []string {
	m := make(map[string]bool)
	for _, s := range t2 {
		m[s] = true
	}

	var result []string
	for _, t := range t1 {
		if has := m[t]; !has {
			result = append(result, t)
		}
	}
	return result
}

func updateOneTag(tagid, taskid, instruction string, errc chan error) {
	v := url.Values{}
	v.Add("tag", tagid)
	suffix := fmt.Sprintf("tasks/%s/%s", taskid, instruction)
	resp, err := runPost("POST", suffix, v)
	if err != nil {
		errc <- errors.Wrap(err, "updateTags")
		return
	}
	fmt.Println(string(resp))
	errc <- nil
}

func updateTags(tw x.WarriorTask, asana x.WarriorTask) error {
	taskid := strconv.FormatUint(tw.Xid, 10)
	add := diff(tw.Tags, asana.Tags)
	rem := diff(asana.Tags, tw.Tags)

	addids := toTagIds(add)
	remids := toTagIds(rem)
	sz := len(addids) + len(remids)

	errc := make(chan error, sz)
	for _, id := range addids {
		go updateOneTag(id, taskid, "addTag", errc)
	}
	for _, id := range remids {
		go updateOneTag(id, taskid, "removeTag", errc)
	}

	var rerr error
	for i := 0; i < sz; i++ {
		if err := <-errc; err != nil {
			rerr = err
		}
	}

	return rerr
}

func UpdateTask(tw x.WarriorTask, asana x.WarriorTask) error {
	v := url.Values{}
	if tw.Name != asana.Name {
		v.Add("name", tw.Name)
	}
	if tw.Assignee != asana.Assignee {
		a := cache.UserId(tw.Assignee)
		if a > 0 {
			v.Add("assignee", strconv.FormatUint(a, 10))
		}
	}
	if !tw.Completed.IsZero() {
		v.Add("completed", "true")
	}

	if len(v) > 0 {
		resp, err := runPost("PUT", "tasks/"+strconv.FormatUint(tw.Xid, 10), v)
		if err != nil {
			return errors.Wrap(err, "UpdateAsanaTask")
		}
		fmt.Println(string(resp))
	}

	if err := updateTags(tw, asana); err != nil {
		return errors.Wrap(err, "asana.UpdateTask updateTags")
	}

	// Update project or section if changed.
	pid := cache.ProjectId(tw.Project)
	if pid > 0 && (tw.Project != asana.Project || tw.Section != asana.Section) {
		fmt.Printf("Updating project and section: %v %v", tw.Project, tw.Section)
		if err := updateSection(tw.Xid, pid, tw.Section); err != nil {
			return errors.Wrap(err, "asana.UpdateTask updateSection")
		}
		fmt.Printf("Removing from project: %v", asana.Project)
		if previd := cache.ProjectId(asana.Project); previd > 0 {
			if err := removeProject(tw.Xid, previd); err != nil {
				return err
			}
		}
	}
	return nil
}

func GetOneTask(taskid uint64) (x.WarriorTask, error) {
	e := x.WarriorTask{}
	var ot oneTask
	if err := runGetter(&ot, "tasks/"+strconv.FormatUint(taskid, 10)); err != nil {
		return e, errors.Wrap(err, "AddNew runGetter")
	}

	if len(ot.Data.Memberships) == 0 {
		return e, fmt.Errorf("Member of no project")
	}
	member := ot.Data.Memberships[0]

	sname := cache.SectionName(member.Project.Id, member.Section.Id)
	return convert(ot.Data, member.Project.Name, sname)
}
<commit_msg>Fix a bug where we tried to remove task from the only project it was part of. Also fix print formatting.<commit_after>package asana

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"

	"github.com/manishrjain/asanawarrior/x"
	"github.com/pkg/errors"
)

var token = flag.String("token", "", "Token provided by Asana.")
var domain = flag.String("domain", "", "Workspace name, generally your domain name in Asana.")
var cache *acache = new(acache)

const (
	prefix = "https://app.asana.com/api/1.0"
	stamp  = "2006-01-02T15:04:05.999Z"
)

func runGetter(i interface{}, suffix string, fields ...string) error {
GETLOOP:
	url := fmt.Sprintf("%s/%s?opt_fields=%s", prefix, suffix, strings.Join(fields, ","))
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		log.Fatal(err)
	}
	req.Header.Add("Authorization", "Bearer "+*token)
	client := &http.Client{
		Timeout: 10 * time.Minute,
	}
	resp, err := client.Do(req)
	if err != nil {
		log.Printf("runGetter url: [%v] err: [%v]", url, err)
		time.Sleep(5 * time.Second)
		goto GETLOOP
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}
	if err := json.Unmarshal(body, i); err != nil {
		return err
	}
	return nil
}

type Basic struct {
	Id    uint64 `json:"id"`
	Name  string `json:"name"`
	Email string `json:"email"`
}
type BasicData struct {
	Data []Basic `json:"data"`
}
type BasicDataOne struct {
	Data Basic `json:"data"`
}

func getVarious(suffix string, opts ...string) ([]Basic, error) {
	var bd BasicData
	if err := runGetter(&bd, suffix, opts...); err != nil {
		return nil, err
	}
	return bd.Data, nil
}

type psec struct {
	Project Basic `json:"project"`
	Section Basic `json:"section"`
}

type task struct {
	Basic
	Assignee    Basic   `json:"assignee"`
	Tags        []Basic `json:"tags"`
	CompletedAt string  `json:"completed_at"`
	ModifiedAt  string  `json:"modified_at"`
	CreatedAt   string  `json:"created_at"`
	Memberships []psec  `json:"memberships"`
}

type tasks struct {
	Data []task `json:"data"`
}

type oneTask struct {
	Data task `json:"data"`
}

func convert(tsk task, proj, section string) (x.WarriorTask, error) {
	e := x.WarriorTask{}

	mts, err := time.Parse(stamp, tsk.ModifiedAt)
	if err != nil {
		return e, errors.Wrap(err, "asana modified at")
	}
	cts, err := time.Parse(stamp, tsk.CreatedAt)
	if err != nil {
		return e, errors.Wrap(err, "asana created at")
	}
	var dts time.Time
	if len(tsk.CompletedAt) > 0 {
		dts, err = time.Parse(stamp, tsk.CompletedAt)
		if err != nil {
			return e, errors.Wrap(err, "asana completed at")
		}
	}

	wt := x.WarriorTask{
		Name:      tsk.Name,
		Project:   proj,
		Xid:       tsk.Id,
		Assignee:  cache.User(tsk.Assignee.Id),
		Modified:  mts,
		Created:   cts,
		Completed: dts,
		Section:   section,
	}
	for _, tag := range tsk.Tags {
		wt.Tags = append(wt.Tags, cache.Tag(tag.Id))
	}
	return wt, nil
}

func getTasks(proj Basic, out chan x.WarriorTask, errc chan error) {
	var sectionName string
	var t tasks
	if err := runGetter(&t, fmt.Sprintf("projects/%d/tasks", proj.Id),
		"assignee,name,tags,completed_at,modified_at,created_at"); err != nil {
		errc <- errors.Wrapf(err, "getTasks for project: %v", proj.Name)
		return
	}

	for _, tsk := range t.Data {
		if len(tsk.Name) == 0 {
			// Don't sync such tasks.
			continue
		}
		if strings.HasSuffix(tsk.Name, ":") {
			sec := Basic{
				Id:   tsk.Id,
				Name: tsk.Name,
			}
			sectionName = cache.AddSection(proj.Id, sec)
			continue
		}

		wt, err := convert(tsk, proj.Name, sectionName)
		if err != nil {
			errc <- errors.Wrapf(err, "convert: getTasks for project: %v", proj.Name)
			return
		}
		out <- wt
	}
	errc <- nil
}

func GetTasks() ([]x.WarriorTask, error) {
	if err := cache.update(); err != nil {
		return nil, errors.Wrap(err, "cache.update")
	}

	out := make(chan x.WarriorTask, 100)
	projects := cache.Projects()
	errc := make(chan error, len(projects))
	for _, proj := range projects {
		go getTasks(proj, out, errc)
	}

	wtasks := make([]x.WarriorTask, 0, 100)
	done := make(chan struct{})
	go func() {
		for wt := range out {
			wtasks = append(wtasks, wt)
		}
		done <- struct{}{}
	}()

	var rerr error
	for _ = range projects {
		if err := <-errc; err != nil {
			rerr = err
		}
	}
	close(out)
	<-done // Wait for all tasks to be picked up by goroutine.
	return wtasks, rerr
}

// runPost would run a PUT or POST to Asana. No locks should be acquired.
func runPost(method, suffix string, values url.Values) ([]byte, error) {
POSTLOOP:
	url := fmt.Sprintf("%s/%s", prefix, suffix)
	fmt.Println(url, values.Encode())
	req, err := http.NewRequest(method, url, bytes.NewBufferString(values.Encode()))
	if err != nil {
		log.Fatal(errors.Wrap(err, "runPost http.NewRequest"))
	}

	req.Header.Add("Authorization", "Bearer "+*token)
	req.Header.Add("content-type", "application/x-www-form-urlencoded")
	client := &http.Client{
		Timeout: 10 * time.Minute,
	}
	resp, err := client.Do(req)
	if err != nil {
		log.Printf("runPost url: [%v] err: [%v]", url, err)
		time.Sleep(5 * time.Second)
		goto POSTLOOP
	}
	defer resp.Body.Close()

	return ioutil.ReadAll(resp.Body)
}

func toTagIds(tnames []string) []string {
	var tags []string
	for _, t := range tnames {
		tid := cache.TagId(t)
		if tid == 0 {
			tid = cache.CreateTag(t)
			fmt.Printf("New Tag created. ID: %d", tid)
		}
		if tid > 0 {
			tags = append(tags, strconv.FormatUint(tid, 10))
		}
	}
	return tags
}

func removeProject(tid, pid uint64) error {
	v := url.Values{}
	v.Add("project", strconv.FormatUint(pid, 10))
	_, err := runPost("POST", fmt.Sprintf("tasks/%d/removeProject", tid), v)
	return err
}

func updateSection(tid, pid uint64, section string) error {
	v := url.Values{}
	v.Add("project", strconv.FormatUint(pid, 10))

	sid := cache.SectionId(pid, section)
	if sid > 0 {
		v.Add("section", strconv.FormatUint(sid, 10))
	}

	_, err := runPost("POST", fmt.Sprintf("tasks/%d/addProject", tid), v)
	return err
}

func AddNew(wt x.WarriorTask) (x.WarriorTask, error) {
	e := x.WarriorTask{}

	// Ensure that project actually exists before proceeding.
	pid := cache.ProjectId(wt.Project)
	if pid == 0 {
		return e, fmt.Errorf("Project not found: %v", wt.Project)
	}

	v := url.Values{}
	v.Add("workspace", strconv.FormatUint(cache.Workspace(), 10))
	v.Add("name", wt.Name)
	fmt.Println("Assignee", wt.Assignee)
	aid := cache.UserId(wt.Assignee)
	if aid > 0 {
		v.Add("assignee", strconv.FormatUint(aid, 10))
	}

	tags := toTagIds(wt.Tags)
	v.Add("tags", strings.Join(tags, ","))
	resp, err := runPost("POST", "tasks", v)
	if err != nil {
		return e, errors.Wrap(err, "AddNew runPost")
	}
	fmt.Println(string(resp))

	var ot oneTask
	if err := json.Unmarshal(resp, &ot); err != nil {
		return e, errors.Wrap(err, "AddNew unmarshal")
	}
	if ot.Data.Id == 0 {
		return e, fmt.Errorf("Unable to find ID assigned by Asana: %+v", ot.Data)
	}

	// Now set the project and section.
	if err := updateSection(ot.Data.Id, pid, wt.Section); err != nil {
		return e, errors.Wrap(err, "AddNew updateSection")
	}

	// Now retrieve the task back again so we can sync it up with TW.
	return GetOneTask(ot.Data.Id)
}

func diff(t1 []string, t2 []string) []string {
	m := make(map[string]bool)
	for _, s := range t2 {
		m[s] = true
	}

	var result []string
	for _, t := range t1 {
		if has := m[t]; !has {
			result = append(result, t)
		}
	}
	return result
}

func updateOneTag(tagid, taskid, instruction string, errc chan error) {
	v := url.Values{}
	v.Add("tag", tagid)
	suffix := fmt.Sprintf("tasks/%s/%s", taskid, instruction)
	resp, err := runPost("POST", suffix, v)
	if err != nil {
		errc <- errors.Wrap(err, "updateTags")
		return
	}
	fmt.Println(string(resp))
	errc <- nil
}

func updateTags(tw x.WarriorTask, asana x.WarriorTask) error {
	taskid := strconv.FormatUint(tw.Xid, 10)
	add := diff(tw.Tags, asana.Tags)
	rem := diff(asana.Tags, tw.Tags)

	addids := toTagIds(add)
	remids := toTagIds(rem)
	sz := len(addids) + len(remids)

	errc := make(chan error, sz)
	for _, id := range addids {
		go updateOneTag(id, taskid, "addTag", errc)
	}
	for _, id := range remids {
		go updateOneTag(id, taskid, "removeTag", errc)
	}

	var rerr error
	for i := 0; i < sz; i++ {
		if err := <-errc; err != nil {
			rerr = err
		}
	}

	return rerr
}

func UpdateTask(tw x.WarriorTask, asana x.WarriorTask) error {
	v := url.Values{}
	if tw.Name != asana.Name {
		v.Add("name", tw.Name)
	}
	if tw.Assignee != asana.Assignee {
		a := cache.UserId(tw.Assignee)
		if a > 0 {
			v.Add("assignee", strconv.FormatUint(a, 10))
		}
	}
	if !tw.Completed.IsZero() {
		v.Add("completed", "true")
	}

	if len(v) > 0 {
		resp, err := runPost("PUT", "tasks/"+strconv.FormatUint(tw.Xid, 10), v)
		if err != nil {
			return errors.Wrap(err, "UpdateAsanaTask")
		}
		fmt.Println(string(resp))
	}

	if err := updateTags(tw, asana); err != nil {
		return errors.Wrap(err, "asana.UpdateTask updateTags")
	}

	// Update project or section if changed.
	pid := cache.ProjectId(tw.Project)
	if pid > 0 && (tw.Project != asana.Project || tw.Section != asana.Section) {
		fmt.Printf("Updating project and section: %v %v\n", tw.Project, tw.Section)
		if err := updateSection(tw.Xid, pid, tw.Section); err != nil {
			return errors.Wrap(err, "asana.UpdateTask updateSection")
		}
		if tw.Project == asana.Project {
			return nil
		}
		// Project was changed. So, remove the last one.
		fmt.Printf("Removing from project: %v\n", asana.Project)
		if previd := cache.ProjectId(asana.Project); previd > 0 {
			if err := removeProject(tw.Xid, previd); err != nil {
				return err
			}
		}
	}
	return nil
}

func GetOneTask(taskid uint64) (x.WarriorTask, error) {
	e := x.WarriorTask{}
	var ot oneTask
	if err := runGetter(&ot, "tasks/"+strconv.FormatUint(taskid, 10)); err != nil {
		return e, errors.Wrap(err, "AddNew runGetter")
	}

	if len(ot.Data.Memberships) == 0 {
		return e, fmt.Errorf("Member of no project")
	}
	member := ot.Data.Memberships[0]

	sname := cache.SectionName(member.Project.Id, member.Section.Id)
	return convert(ot.Data, member.Project.Name, sname)
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/go-chi/chi"
	"github.com/go-chi/chi/middleware"
	"gopkg.in/mgo.v2"
	"time"
)

//newRouter creates chi router and pass mongo session to handlers
func newRouter(session *mgo.Session) *chi.Mux {
	// Configure http routing
	r := chi.NewRouter()
	// A good base middleware stack
	r.Use(middleware.RequestID)
	r.Use(middleware.RealIP)
	r.Use(middleware.Logger)
	r.Use(middleware.Recoverer)
	r.Use(middleware.RedirectSlashes)

	// Set a timeout value on the request context (ctx), that will signal
	// through ctx.Done() that the request has timed out and further
	// processing should be stopped.
	r.Use(middleware.Timeout(60 * time.Second))

	// RESTy routes for "data" resource
	r.Route("/{collectionName}", func(r chi.Router) {
		r.Get("/", indexHandler)
		r.Post("/", makeHandler(createHandler, session))
	})

	return r
}
<commit_msg>add simple cors<commit_after>package main

import (
	"gopkg.in/mgo.v2"
	"time"

	"github.com/go-chi/chi"
	"github.com/go-chi/chi/middleware"
	"github.com/go-chi/cors"
)

//newRouter creates chi router and pass mongo session to handlers
func newRouter(session *mgo.Session) *chi.Mux {
	// Configure http routing
	r := chi.NewRouter()
	// A good base middleware stack
	r.Use(middleware.RequestID)
	r.Use(middleware.RealIP)
	r.Use(middleware.Logger)
	r.Use(middleware.Recoverer)
	r.Use(middleware.RedirectSlashes)

	// Basic CORS
	r.Use(cors.Handler(cors.Options{
		AllowedOrigins: []string{"*"},
		AllowedMethods: []string{"GET", "POST", "OPTIONS"},
		AllowedHeaders: []string{"*"},
		MaxAge: int(24 * time.Hour),
	}))

	// Set a timeout value on the request context (ctx), that will signal
	// through ctx.Done() that the request has timed out and further
	// processing should be stopped.
	r.Use(middleware.Timeout(60 * time.Second))

	// RESTy routes for "data" resource
	r.Route("/{collectionName}", func(r chi.Router) {
		r.Get("/", indexHandler)
		r.Post("/", makeHandler(createHandler, session))
	})

	return r
}
<|endoftext|>


<commit_before>package dht

import (
	"bytes"
	"encoding/json"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"

	peer "github.com/jbenet/go-ipfs/peer"
	kb "github.com/jbenet/go-ipfs/routing/kbucket"
	u "github.com/jbenet/go-ipfs/util"
)

// This file implements the Routing interface for the IpfsDHT struct.

// Basic Put/Get

// PutValue adds value corresponding to given Key.
// This is the top level "Store" operation of the DHT
func (dht *IpfsDHT) PutValue(ctx context.Context, key u.Key, value []byte) error {
	peers := []*peer.Peer{}

	// get the peers we need to announce to
	for _, route := range dht.routingTables {
		npeers := route.NearestPeers(kb.ConvertKey(key), KValue)
		peers = append(peers, npeers...)
	}

	query := newQuery(key, func(ctx context.Context, p *peer.Peer) (*dhtQueryResult, error) {
		u.DOut("[%s] PutValue qry part %v\n", dht.self.ID.Pretty(), p.ID.Pretty())
		err := dht.putValueToNetwork(ctx, p, string(key), value)
		if err != nil {
			return nil, err
		}
		return &dhtQueryResult{success: true}, nil
	})

	_, err := query.Run(ctx, peers)
	u.DOut("[%s] PutValue %v %v\n", dht.self.ID.Pretty(), key, value)
	return err
}

// GetValue searches for the value corresponding to given Key.
// If the search does not succeed, a multiaddr string of a closer peer is
// returned along with util.ErrSearchIncomplete
func (dht *IpfsDHT) GetValue(ctx context.Context, key u.Key) ([]byte, error) {
	ll := startNewRPC("GET")
	defer ll.EndAndPrint()

	// If we have it local, dont bother doing an RPC!
	// NOTE: this might not be what we want to do...
	val, err := dht.getLocal(key)
	if err == nil {
		ll.Success = true
		return val, nil
	}

	// get closest peers in the routing tables
	routeLevel := 0
	closest := dht.routingTables[routeLevel].NearestPeers(kb.ConvertKey(key), PoolSize)
	if closest == nil || len(closest) == 0 {
		return nil, kb.ErrLookupFailure
	}

	// setup the Query
	query := newQuery(key, func(ctx context.Context, p *peer.Peer) (*dhtQueryResult, error) {

		val, peers, err := dht.getValueOrPeers(ctx, p, key, routeLevel)
		if err != nil {
			return nil, err
		}

		res := &dhtQueryResult{value: val, closerPeers: peers}
		if val != nil {
			res.success = true
		}

		return res, nil
	})

	// run it!
	result, err := query.Run(ctx, closest)
	if err != nil {
		return nil, err
	}

	u.DOut("[%s] GetValue %v %v\n", dht.self.ID.Pretty(), key, result.value)
	if result.value == nil {
		return nil, u.ErrNotFound
	}

	return result.value, nil
}

// Value provider layer of indirection.
// This is what DSHTs (Coral and MainlineDHT) do to store large values in a DHT.

// Provide makes this node announce that it can provide a value for the given key
func (dht *IpfsDHT) Provide(ctx context.Context, key u.Key) error {

	dht.providers.AddProvider(key, dht.self)
	peers := dht.routingTables[0].NearestPeers(kb.ConvertKey(key), PoolSize)
	if len(peers) == 0 {
		return kb.ErrLookupFailure
	}

	//TODO FIX: this doesn't work! it needs to be sent to the actual nearest peers.
	// `peers` are the closest peers we have, not the ones that should get the value.
	for _, p := range peers {
		err := dht.putProvider(ctx, p, string(key))
		if err != nil {
			return err
		}
	}
	return nil
}

// NB: not actually async. Used to keep the interface consistent while the
// actual async method, FindProvidersAsync2 is under construction
func (dht *IpfsDHT) FindProvidersAsync(ctx context.Context, key u.Key, count int) <-chan *peer.Peer {
	ch := make(chan *peer.Peer)
	providers, err := dht.FindProviders(ctx, key)
	if err != nil {
		close(ch)
		return ch
	}
	go func() {
		defer close(ch)
		for _, p := range providers {
			ch <- p
		}
	}()
	return ch
}

// FIXME: there's a bug here!
func (dht *IpfsDHT) FindProvidersAsync2(ctx context.Context, key u.Key, count int) <-chan *peer.Peer {
	peerOut := make(chan *peer.Peer, count)
	go func() {
		ps := newPeerSet()
		provs := dht.providers.GetProviders(key)
		for _, p := range provs {
			count--
			// NOTE: assuming that this list of peers is unique
			ps.Add(p)
			peerOut <- p
			if count <= 0 {
				return
			}
		}

		peers := dht.routingTables[0].NearestPeers(kb.ConvertKey(key), AlphaValue)
		for _, pp := range peers {
			ppp := pp
			go func() {
				pmes, err := dht.findProvidersSingle(ctx, ppp, key, 0)
				if err != nil {
					u.PErr("%v\n", err)
					return
				}
				dht.addPeerListAsync(key, pmes.GetProviderPeers(), ps, count, peerOut)
			}()
		}

	}()
	return peerOut
}

//TODO: this function could also be done asynchronously
func (dht *IpfsDHT) addPeerListAsync(k u.Key, peers []*Message_Peer, ps *peerSet, count int, out chan *peer.Peer) {
	for _, pbp := range peers {

		// construct new peer
		p, err := dht.ensureConnectedToPeer(pbp)
		if err != nil {
			continue
		}

		dht.providers.AddProvider(k, p)
		if ps.AddIfSmallerThan(p, count) {
			out <- p
		} else if ps.Size() >= count {
			return
		}
	}
}

// FindProviders searches for peers who can provide the value for given key.
func (dht *IpfsDHT) FindProviders(ctx context.Context, key u.Key) ([]*peer.Peer, error) {
	ll := startNewRPC("FindProviders")
	ll.EndAndPrint()

	// get closest peer
	u.DOut("Find providers for: '%s'\n", key)
	p := dht.routingTables[0].NearestPeer(kb.ConvertKey(key))
	if p == nil {
		return nil, nil
	}

	for level := 0; level < len(dht.routingTables); {

		// attempt retrieving providers
		pmes, err := dht.findProvidersSingle(ctx, p, key, level)
		if err != nil {
			return nil, err
		}

		// handle providers
		provs := pmes.GetProviderPeers()
		if provs != nil {
			u.DOut("Got providers back from findProviders call!\n")
			return dht.addProviders(key, provs), nil
		}

		u.DOut("Didnt get providers, just closer peers.\n")
		closer := pmes.GetCloserPeers()
		if len(closer) == 0 {
			level++
			continue
		}

		np, err := dht.peerFromInfo(closer[0])
		if err != nil {
			u.DOut("no peerFromInfo")
			level++
			continue
		}
		p = np
	}
	return nil, u.ErrNotFound
}

// Find specific Peer

// FindPeer searches for a peer with given ID.
func (dht *IpfsDHT) FindPeer(ctx context.Context, id peer.ID) (*peer.Peer, error) {

	// Check if were already connected to them
	p, _ := dht.Find(id)
	if p != nil {
		return p, nil
	}

	// @whyrusleeping why is this here? doesn't the dht.Find above cover it?
	routeLevel := 0
	p = dht.routingTables[routeLevel].NearestPeer(kb.ConvertPeerID(id))
	if p == nil {
		return nil, kb.ErrLookupFailure
	}
	if p.ID.Equal(id) {
		return p, nil
	}

	for routeLevel < len(dht.routingTables) {
		pmes, err := dht.findPeerSingle(ctx, p, id, routeLevel)
		plist := pmes.GetCloserPeers()
		if plist == nil || len(plist) == 0 {
			routeLevel++
			continue
		}
		found := plist[0]

		nxtPeer, err := dht.ensureConnectedToPeer(found)
		if err != nil {
			routeLevel++
			continue
		}

		if nxtPeer.ID.Equal(id) {
			return nxtPeer, nil
		}

		p = nxtPeer
	}
	return nil, u.ErrNotFound
}

func (dht *IpfsDHT) findPeerMultiple(ctx context.Context, id peer.ID) (*peer.Peer, error) {

	// Check if were already connected to them
	p, _ := dht.Find(id)
	if p != nil {
		return p, nil
	}

	// get the peers we need to announce to
	routeLevel := 0
	peers := dht.routingTables[routeLevel].NearestPeers(kb.ConvertPeerID(id), AlphaValue)
	if len(peers) == 0 {
		return nil, kb.ErrLookupFailure
	}

	// setup query function
	query := newQuery(u.Key(id), func(ctx context.Context, p *peer.Peer) (*dhtQueryResult, error) {
		pmes, err := dht.findPeerSingle(ctx, p, id, routeLevel)
		if err != nil {
			u.DErr("getPeer error: %v\n", err)
			return nil, err
		}

		plist := pmes.GetCloserPeers()
		if len(plist) == 0 {
			routeLevel++
		}

		nxtprs := make([]*peer.Peer, len(plist))
		for i, fp := range plist {
			nxtp, err := dht.peerFromInfo(fp)
			if err != nil {
				u.DErr("findPeer error: %v\n", err)
				continue
			}

			if nxtp.ID.Equal(id) {
				return &dhtQueryResult{peer: nxtp, success: true}, nil
			}

			nxtprs[i] = nxtp
		}

		return &dhtQueryResult{closerPeers: nxtprs}, nil
	})

	result, err := query.Run(ctx, peers)
	if err != nil {
		return nil, err
	}

	if result.peer == nil {
		return nil, u.ErrNotFound
	}
	return result.peer, nil
}

// Ping a peer, log the time it took
func (dht *IpfsDHT) Ping(ctx context.Context, p *peer.Peer) error {
	// Thoughts: maybe this should accept an ID and do a peer lookup?
	u.DOut("[%s] ping %s start\n", dht.self.ID.Pretty(), p.ID.Pretty())

	pmes := newMessage(Message_PING, "", 0)
	_, err := dht.sendRequest(ctx, p, pmes)
	u.DOut("[%s] ping %s end (err = %s)\n", dht.self.ID.Pretty(), p.ID.Pretty(), err)
	return err
}

func (dht *IpfsDHT) getDiagnostic(ctx context.Context) ([]*diagInfo, error) {

	u.DOut("Begin Diagnostic")
	peers := dht.routingTables[0].NearestPeers(kb.ConvertPeerID(dht.self.ID), 10)
	var out []*diagInfo

	query := newQuery(dht.self.Key(), func(ctx context.Context, p *peer.Peer) (*dhtQueryResult, error) {
		pmes := newMessage(Message_DIAGNOSTIC, "", 0)
		rpmes, err := dht.sendRequest(ctx, p, pmes)
		if err != nil {
			return nil, err
		}

		dec := json.NewDecoder(bytes.NewBuffer(rpmes.GetValue()))
		for {
			di := new(diagInfo)
			err := dec.Decode(di)
			if err != nil {
				break
			}

			out = append(out, di)
		}
		return &dhtQueryResult{success: true}, nil
	})

	_, err := query.Run(ctx, peers)
	return out, err
}
<commit_msg>udpated commands and RPC dialing to work with new configuration changes<commit_after>package dht

import (
	"bytes"
	"encoding/json"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"

	peer "github.com/jbenet/go-ipfs/peer"
	kb "github.com/jbenet/go-ipfs/routing/kbucket"
	u "github.com/jbenet/go-ipfs/util"
)

// This file implements the Routing interface for the IpfsDHT struct.

// Basic Put/Get

// PutValue adds value corresponding to given Key.
// This is the top level "Store" operation of the DHT
func (dht *IpfsDHT) PutValue(ctx context.Context, key u.Key, value []byte) error {
	peers := []*peer.Peer{}

	// get the peers we need to announce to
	for _, route := range dht.routingTables {
		npeers := route.NearestPeers(kb.ConvertKey(key), KValue)
		peers = append(peers, npeers...)
	}

	query := newQuery(key, func(ctx context.Context, p *peer.Peer) (*dhtQueryResult, error) {
		u.DOut("[%s] PutValue qry part %v\n", dht.self.ID.Pretty(), p.ID.Pretty())
		err := dht.putValueToNetwork(ctx, p, string(key), value)
		if err != nil {
			return nil, err
		}
		return &dhtQueryResult{success: true}, nil
	})

	_, err := query.Run(ctx, peers)
	u.DOut("[%s] PutValue %v %v\n", dht.self.ID.Pretty(), key, value)
	return err
}

// GetValue searches for the value corresponding to given Key.
// If the search does not succeed, a multiaddr string of a closer peer is
// returned along with util.ErrSearchIncomplete
func (dht *IpfsDHT) GetValue(ctx context.Context, key u.Key) ([]byte, error) {
	ll := startNewRPC("GET")
	defer ll.EndAndPrint()

	// If we have it local, dont bother doing an RPC!
	// NOTE: this might not be what we want to do...
	val, err := dht.getLocal(key)
	if err == nil {
		ll.Success = true
		return val, nil
	}

	// get closest peers in the routing tables
	routeLevel := 0
	closest := dht.routingTables[routeLevel].NearestPeers(kb.ConvertKey(key), PoolSize)
	if closest == nil || len(closest) == 0 {
		return nil, nil
	}

	// setup the Query
	query := newQuery(key, func(ctx context.Context, p *peer.Peer) (*dhtQueryResult, error) {

		val, peers, err := dht.getValueOrPeers(ctx, p, key, routeLevel)
		if err != nil {
			return nil, err
		}

		res := &dhtQueryResult{value: val, closerPeers: peers}
		if val != nil {
			res.success = true
		}

		return res, nil
	})

	// run it!
	result, err := query.Run(ctx, closest)
	if err != nil {
		return nil, err
	}

	u.DOut("[%s] GetValue %v %v\n", dht.self.ID.Pretty(), key, result.value)
	if result.value == nil {
		return nil, u.ErrNotFound
	}

	return result.value, nil
}

// Value provider layer of indirection.
// This is what DSHTs (Coral and MainlineDHT) do to store large values in a DHT.

// Provide makes this node announce that it can provide a value for the given key
func (dht *IpfsDHT) Provide(ctx context.Context, key u.Key) error {

	dht.providers.AddProvider(key, dht.self)
	peers := dht.routingTables[0].NearestPeers(kb.ConvertKey(key), PoolSize)
	if len(peers) == 0 {
		return nil
	}

	//TODO FIX: this doesn't work! it needs to be sent to the actual nearest peers.
	// `peers` are the closest peers we have, not the ones that should get the value.
	for _, p := range peers {
		err := dht.putProvider(ctx, p, string(key))
		if err != nil {
			return err
		}
	}
	return nil
}

// NB: not actually async. Used to keep the interface consistent while the
// actual async method, FindProvidersAsync2 is under construction
func (dht *IpfsDHT) FindProvidersAsync(ctx context.Context, key u.Key, count int) <-chan *peer.Peer {
	ch := make(chan *peer.Peer)
	providers, err := dht.FindProviders(ctx, key)
	if err != nil {
		close(ch)
		return ch
	}
	go func() {
		defer close(ch)
		for _, p := range providers {
			ch <- p
		}
	}()
	return ch
}

// FIXME: there's a bug here!
func (dht *IpfsDHT) FindProvidersAsync2(ctx context.Context, key u.Key, count int) <-chan *peer.Peer {
	peerOut := make(chan *peer.Peer, count)
	go func() {
		ps := newPeerSet()
		provs := dht.providers.GetProviders(key)
		for _, p := range provs {
			count--
			// NOTE: assuming that this list of peers is unique
			ps.Add(p)
			peerOut <- p
			if count <= 0 {
				return
			}
		}

		peers := dht.routingTables[0].NearestPeers(kb.ConvertKey(key), AlphaValue)
		for _, pp := range peers {
			ppp := pp
			go func() {
				pmes, err := dht.findProvidersSingle(ctx, ppp, key, 0)
				if err != nil {
					u.PErr("%v\n", err)
					return
				}
				dht.addPeerListAsync(key, pmes.GetProviderPeers(), ps, count, peerOut)
			}()
		}

	}()
	return peerOut
}

//TODO: this function could also be done asynchronously
func (dht *IpfsDHT) addPeerListAsync(k u.Key, peers []*Message_Peer, ps *peerSet, count int, out chan *peer.Peer) {
	for _, pbp := range peers {

		// construct new peer
		p, err := dht.ensureConnectedToPeer(pbp)
		if err != nil {
			continue
		}

		dht.providers.AddProvider(k, p)
		if ps.AddIfSmallerThan(p, count) {
			out <- p
		} else if ps.Size() >= count {
			return
		}
	}
}

// FindProviders searches for peers who can provide the value for given key.
func (dht *IpfsDHT) FindProviders(ctx context.Context, key u.Key) ([]*peer.Peer, error) {
	ll := startNewRPC("FindProviders")
	ll.EndAndPrint()

	// get closest peer
	u.DOut("Find providers for: '%s'\n", key)
	p := dht.routingTables[0].NearestPeer(kb.ConvertKey(key))
	if p == nil {
		return nil, nil
	}

	for level := 0; level < len(dht.routingTables); {

		// attempt retrieving providers
		pmes, err := dht.findProvidersSingle(ctx, p, key, level)
		if err != nil {
			return nil, err
		}

		// handle providers
		provs := pmes.GetProviderPeers()
		if provs != nil {
			u.DOut("Got providers back from findProviders call!\n")
			return dht.addProviders(key, provs), nil
		}

		u.DOut("Didnt get providers, just closer peers.\n")
		closer := pmes.GetCloserPeers()
		if len(closer) == 0 {
			level++
			continue
		}

		np, err := dht.peerFromInfo(closer[0])
		if err != nil {
			u.DOut("no peerFromInfo")
			level++
			continue
		}
		p = np
	}
	return nil, u.ErrNotFound
}

// Find specific Peer

// FindPeer searches for a peer with given ID.
func (dht *IpfsDHT) FindPeer(ctx context.Context, id peer.ID) (*peer.Peer, error) {

	// Check if were already connected to them
	p, _ := dht.Find(id)
	if p != nil {
		return p, nil
	}

	// @whyrusleeping why is this here? doesn't the dht.Find above cover it?
	routeLevel := 0
	p = dht.routingTables[routeLevel].NearestPeer(kb.ConvertPeerID(id))
	if p == nil {
		return nil, nil
	}
	if p.ID.Equal(id) {
		return p, nil
	}

	for routeLevel < len(dht.routingTables) {
		pmes, err := dht.findPeerSingle(ctx, p, id, routeLevel)
		plist := pmes.GetCloserPeers()
		if plist == nil || len(plist) == 0 {
			routeLevel++
			continue
		}
		found := plist[0]

		nxtPeer, err := dht.ensureConnectedToPeer(found)
		if err != nil {
			routeLevel++
			continue
		}

		if nxtPeer.ID.Equal(id) {
			return nxtPeer, nil
		}

		p = nxtPeer
	}
	return nil, u.ErrNotFound
}

func (dht *IpfsDHT) findPeerMultiple(ctx context.Context, id peer.ID) (*peer.Peer, error) {

	// Check if were already connected to them
	p, _ := dht.Find(id)
	if p != nil {
		return p, nil
	}

	// get the peers we need to announce to
	routeLevel := 0
	peers := dht.routingTables[routeLevel].NearestPeers(kb.ConvertPeerID(id), AlphaValue)
	if len(peers) == 0 {
		return nil, nil
	}

	// setup query function
	query := newQuery(u.Key(id), func(ctx context.Context, p *peer.Peer) (*dhtQueryResult, error) {
		pmes, err := dht.findPeerSingle(ctx, p, id, routeLevel)
		if err != nil {
			u.DErr("getPeer error: %v\n", err)
			return nil, err
		}

		plist := pmes.GetCloserPeers()
		if len(plist) == 0 {
			routeLevel++
		}

		nxtprs := make([]*peer.Peer, len(plist))
		for i, fp := range plist {
			nxtp, err := dht.peerFromInfo(fp)
			if err != nil {
				u.DErr("findPeer error: %v\n", err)
				continue
			}

			if nxtp.ID.Equal(id) {
				return &dhtQueryResult{peer: nxtp, success: true}, nil
			}

			nxtprs[i] = nxtp
		}

		return &dhtQueryResult{closerPeers: nxtprs}, nil
	})

	result, err := query.Run(ctx, peers)
	if err != nil {
		return nil, err
	}

	if result.peer == nil {
		return nil, u.ErrNotFound
	}
	return result.peer, nil
}

// Ping a peer, log the time it took
func (dht *IpfsDHT) Ping(ctx context.Context, p *peer.Peer) error {
	// Thoughts: maybe this should accept an ID and do a peer lookup?
	u.DOut("[%s] ping %s start\n", dht.self.ID.Pretty(), p.ID.Pretty())

	pmes := newMessage(Message_PING, "", 0)
	_, err := dht.sendRequest(ctx, p, pmes)
	u.DOut("[%s] ping %s end (err = %s)\n", dht.self.ID.Pretty(), p.ID.Pretty(), err)
	return err
}

func (dht *IpfsDHT) getDiagnostic(ctx context.Context) ([]*diagInfo, error) {

	u.DOut("Begin Diagnostic")
	peers := dht.routingTables[0].NearestPeers(kb.ConvertPeerID(dht.self.ID), 10)
	var out []*diagInfo

	query := newQuery(dht.self.Key(), func(ctx context.Context, p *peer.Peer) (*dhtQueryResult, error) {
		pmes := newMessage(Message_DIAGNOSTIC, "", 0)
		rpmes, err := dht.sendRequest(ctx, p, pmes)
		if err != nil {
			return nil, err
		}

		dec := json.NewDecoder(bytes.NewBuffer(rpmes.GetValue()))
		for {
			di := new(diagInfo)
			err := dec.Decode(di)
			if err != nil {
				break
			}

			out = append(out, di)
		}
		return &dhtQueryResult{success: true}, nil
	})

	_, err := query.Run(ctx, peers)
	return out, err
}
<|endoftext|>


<commit_before>package rpc

import (
	"encoding/json"
	"errors"
	"fmt"
	. "github.com/zrepl/zrepl/model"
	. "github.com/zrepl/zrepl/util"
	"io"
	"reflect"
)

type RPCRequester interface {
	FilesystemRequest(r FilesystemRequest) (roots []Filesystem, err error)
	InitialTransferRequest(r InitialTransferRequest) (io.Reader, error)
	IncrementalTransferRequest(r IncrementalTransferRequest) (io.Reader, error)
}

type RPCHandler interface {
	HandleFilesystemRequest(r FilesystemRequest) (roots []Filesystem, err error)
	HandleInitialTransferRequest(r InitialTransferRequest) (io.Reader, error)
	HandleIncrementalTransferRequest(r IncrementalTransferRequest) (io.Reader, error)
}

const ByteStreamRPCProtocolVersion = 1

type ByteStreamRPC struct {
	conn    io.ReadWriteCloser
	encoder *json.Encoder
	decoder *json.Decoder
}

func ConnectByteStreamRPC(conn io.ReadWriteCloser) (RPCRequester, error) {
	// TODO do ssh connection to transport, establish TCP-like communication channel
	rpc := ByteStreamRPC{
		conn:    conn,
		encoder: json.NewEncoder(conn),
		decoder: json.NewDecoder(conn),
	}

	// Assert protocol versions are equal
	err := rpc.ProtocolVersionRequest()
	if err != nil {
		return nil, err
	}

	return rpc, nil
}

func ListenByteStreamRPC(conn io.ReadWriteCloser, handler RPCHandler) error {

	// A request consists of two subsequent JSON objects
	// Object 1: RequestHeader => contains type of Request Body
	// Object 2: RequestBody, e.g. IncrementalTransferRequest
	// A response is always a ResponseHeader followed by bytes to be interpreted
	// 	as indicated by the ResponseHeader.ResponseType, e.g.
	// 	 a) a chunked response
	//   b) or another JSON object

	defer conn.Close()

	decoder := json.NewDecoder(conn)
	encoder := json.NewEncoder(conn)

	for {

		var header RequestHeader = RequestHeader{}
		if err := decoder.Decode(&header); err != nil {
			respondWithError(encoder, EDecodeHeader, err)
			return conn.Close()
		}

		switch header.Type {
		case RTProtocolVersionRequest:
			var rq ByteStreamRPCProtocolVersionRequest
			if err := decoder.Decode(&rq); err != nil {
				respondWithError(encoder, EDecodeRequestBody, nil)
				return conn.Close()
			}

			if rq.ClientVersion != ByteStreamRPCProtocolVersion {
				respondWithError(encoder, EProtocolVersionMismatch, nil)
				return conn.Close()
			}

			r := ResponseHeader{
				RequestId:    header.Id,
				ResponseType: ROK,
			}
			if err := encoder.Encode(&r); err != nil {
				panic(err)
			}

		case RTFilesystemRequest:

			var rq FilesystemRequest
			if err := decoder.Decode(&rq); err != nil {
				respondWithError(encoder, EDecodeRequestBody, nil)
				return conn.Close()
			}

			roots, err := handler.HandleFilesystemRequest(rq)
			if err != nil {
				respondWithError(encoder, EHandler, err)
				return conn.Close()
			} else {
				if err := encoder.Encode(&roots); err != nil {
					panic(err)
				}
			}

		case RTInitialTransferRequest:
			var rq InitialTransferRequest
			if err := decoder.Decode(&rq); err != nil {
				respondWithError(encoder, EDecodeRequestBody, nil)
				return conn.Close()
			}

			snapReader, err := handler.HandleInitialTransferRequest(rq)
			if err != nil {
				respondWithError(encoder, EHandler, err)
				return conn.Close()
			} else {
				chunker := NewChunker(snapReader)
				_, err := io.Copy(conn, &chunker)
				if err != nil {
					panic(err)
				}
			}

		case RTIncrementalTransferRequest:

			var rq IncrementalTransferRequest
			if err := decoder.Decode(&rq); err != nil {
				respondWithError(encoder, EDecodeRequestBody, nil)
				return conn.Close()
			}

			snapReader, err := handler.HandleIncrementalTransferRequest(rq)
			if err != nil {
				respondWithError(encoder, EHandler, err)
			} else {
				chunker := NewChunker(snapReader)
				_, err := io.Copy(conn, &chunker)
				if err != nil {
					panic(err)
				}
			}

		default:
			respondWithError(encoder, EUnknownRequestType, nil)
			return conn.Close()
		}
	}

	return nil
}

func respondWithError(encoder *json.Encoder, id ErrorId, err error) {

	r := ResponseHeader{
		ErrorId:      id,
		ResponseType: RNONE,
		Message:      err.Error(),
	}
	if err := encoder.Encode(&r); err != nil {
		panic(err)
	}

}

func inferRequestType(v interface{}) (RequestType, error) {
	switch v.(type) {
	case ByteStreamRPCProtocolVersionRequest:
		return RTProtocolVersionRequest, nil
	case FilesystemRequest:
		return RTFilesystemRequest, nil
	case InitialTransferRequest:
		return RTInitialTransferRequest, nil
	default:
		return 0, errors.New(fmt.Sprintf("cannot infer request type for type '%v'",
			reflect.TypeOf(v)))
	}
}

func genUUID() [16]byte {
	return [16]byte{} // TODO
}

func (c ByteStreamRPC) sendRequest(v interface{}) (err error) {

	var rt RequestType

	if rt, err = inferRequestType(v); err != nil {
		return
	}

	h := RequestHeader{
		Type: rt,
		Id:   genUUID(),
	}

	if err = c.encoder.Encode(h); err != nil {
		return
	}
	if err = c.encoder.Encode(v); err != nil {
		return
	}

	return
}

func (c ByteStreamRPC) expectResponseType(rt ResponseType) (err error) {
	var h ResponseHeader
	if err = c.decoder.Decode(&h); err != nil {
		return
	}

	if h.ResponseType != rt {
		return errors.New("unexpected response type in response header")
	}
	return
}

func (c ByteStreamRPC) sendRequestReceiveHeader(request interface{}, rt ResponseType) (err error) {

	if err = c.sendRequest(request); err != nil {
		return err
	}

	if err = c.expectResponseType(rt); err != nil {
		return err
	}

	return nil
}

func (c ByteStreamRPC) ProtocolVersionRequest() (err error) {
	b := ByteStreamRPCProtocolVersionRequest{
		ClientVersion: ByteStreamRPCProtocolVersion,
	}

	// OK response means the remote side can cope with our protocol version
	return c.sendRequestReceiveHeader(b, ROK)
}

func (c ByteStreamRPC) FilesystemRequest(r FilesystemRequest) (roots []Filesystem, err error) {

	if err = c.sendRequestReceiveHeader(r, RFilesystems); err != nil {
		return
	}

	roots = make([]Filesystem, 0)

	if err = c.decoder.Decode(&roots); err != nil {
		return
	}

	return
}

func (c ByteStreamRPC) InitialTransferRequest(r InitialTransferRequest) (unchunker io.Reader, err error) {

	if err = c.sendRequestReceiveHeader(r, RChunkedStream); err != nil {
		return
	}
	unchunker = NewUnchunker(c.conn)
	return
}

func (c ByteStreamRPC) IncrementalTransferRequest(r IncrementalTransferRequest) (unchunker io.Reader, err error) {
	if err = c.sendRequestReceiveHeader(r, RChunkedStream); err != nil {
		return
	}
	unchunker = NewUnchunker(c.conn)
	return
}

type LocalRPC struct {
	handler RPCHandler
}

func ConnectLocalRPC(handler RPCHandler) RPCRequester {
	return LocalRPC{handler}
}

func (c LocalRPC) FilesystemRequest(r FilesystemRequest) (roots []Filesystem, err error) {
	return c.handler.HandleFilesystemRequest(r)
}

func (c LocalRPC) InitialTransferRequest(r InitialTransferRequest) (io.Reader, error) {
	return c.handler.HandleInitialTransferRequest(r)
}

func (c LocalRPC) IncrementalTransferRequest(r IncrementalTransferRequest) (reader io.Reader, err error) {
	reader, err = c.handler.HandleIncrementalTransferRequest(r)
	return
}
<commit_msg>rpc: bytestream: listen: filesystems request: reply with header<commit_after>package rpc

import (
	"encoding/json"
	"errors"
	"fmt"
	. "github.com/zrepl/zrepl/model"
	. "github.com/zrepl/zrepl/util"
	"io"
	"reflect"
)

type RPCRequester interface {
	FilesystemRequest(r FilesystemRequest) (roots []Filesystem, err error)
	InitialTransferRequest(r InitialTransferRequest) (io.Reader, error)
	IncrementalTransferRequest(r IncrementalTransferRequest) (io.Reader, error)
}

type RPCHandler interface {
	HandleFilesystemRequest(r FilesystemRequest) (roots []Filesystem, err error)
	HandleInitialTransferRequest(r InitialTransferRequest) (io.Reader, error)
	HandleIncrementalTransferRequest(r IncrementalTransferRequest) (io.Reader, error)
}

const ByteStreamRPCProtocolVersion = 1

type ByteStreamRPC struct {
	conn    io.ReadWriteCloser
	encoder *json.Encoder
	decoder *json.Decoder
}

func ConnectByteStreamRPC(conn io.ReadWriteCloser) (RPCRequester, error) {
	// TODO do ssh connection to transport, establish TCP-like communication channel
	rpc := ByteStreamRPC{
		conn:    conn,
		encoder: json.NewEncoder(conn),
		decoder: json.NewDecoder(conn),
	}

	// Assert protocol versions are equal
	err := rpc.ProtocolVersionRequest()
	if err != nil {
		return nil, err
	}

	return rpc, nil
}

func ListenByteStreamRPC(conn io.ReadWriteCloser, handler RPCHandler) error {

	// A request consists of two subsequent JSON objects
	// Object 1: RequestHeader => contains type of Request Body
	// Object 2: RequestBody, e.g. IncrementalTransferRequest
	// A response is always a ResponseHeader followed by bytes to be interpreted
	// 	as indicated by the ResponseHeader.ResponseType, e.g.
	// 	 a) a chunked response
	//   b) or another JSON object

	defer conn.Close()

	decoder := json.NewDecoder(conn)
	encoder := json.NewEncoder(conn)

	for {

		var header RequestHeader = RequestHeader{}
		if err := decoder.Decode(&header); err != nil {
			respondWithError(encoder, EDecodeHeader, err)
			return conn.Close()
		}

		switch header.Type {
		case RTProtocolVersionRequest:
			var rq ByteStreamRPCProtocolVersionRequest
			if err := decoder.Decode(&rq); err != nil {
				respondWithError(encoder, EDecodeRequestBody, nil)
				return conn.Close()
			}

			if rq.ClientVersion != ByteStreamRPCProtocolVersion {
				respondWithError(encoder, EProtocolVersionMismatch, nil)
				return conn.Close()
			}

			r := ResponseHeader{
				RequestId:    header.Id,
				ResponseType: ROK,
			}
			if err := encoder.Encode(&r); err != nil {
				panic(err)
			}

		case RTFilesystemRequest:

			var rq FilesystemRequest
			if err := decoder.Decode(&rq); err != nil {
				respondWithError(encoder, EDecodeRequestBody, nil)
				return conn.Close()
			}

			roots, err := handler.HandleFilesystemRequest(rq)
			if err != nil {
				respondWithError(encoder, EHandler, err)
				return conn.Close()
			} else {
				r := ResponseHeader{
					RequestId:    header.Id,
					ResponseType: RFilesystems,
				}
				if err := encoder.Encode(&r); err != nil {
					panic(err)
				}
				if err := encoder.Encode(&roots); err != nil {
					panic(err)
				}
			}

		case RTInitialTransferRequest:
			var rq InitialTransferRequest
			if err := decoder.Decode(&rq); err != nil {
				respondWithError(encoder, EDecodeRequestBody, nil)
				return conn.Close()
			}

			snapReader, err := handler.HandleInitialTransferRequest(rq)
			if err != nil {
				respondWithError(encoder, EHandler, err)
				return conn.Close()
			} else {
				chunker := NewChunker(snapReader)
				_, err := io.Copy(conn, &chunker)
				if err != nil {
					panic(err)
				}
			}

		case RTIncrementalTransferRequest:

			var rq IncrementalTransferRequest
			if err := decoder.Decode(&rq); err != nil {
				respondWithError(encoder, EDecodeRequestBody, nil)
				return conn.Close()
			}

			snapReader, err := handler.HandleIncrementalTransferRequest(rq)
			if err != nil {
				respondWithError(encoder, EHandler, err)
			} else {
				chunker := NewChunker(snapReader)
				_, err := io.Copy(conn, &chunker)
				if err != nil {
					panic(err)
				}
			}

		default:
			respondWithError(encoder, EUnknownRequestType, nil)
			return conn.Close()
		}
	}

	return nil
}

func respondWithError(encoder *json.Encoder, id ErrorId, err error) {

	r := ResponseHeader{
		ErrorId:      id,
		ResponseType: RNONE,
		Message:      err.Error(),
	}
	if err := encoder.Encode(&r); err != nil {
		panic(err)
	}

}

func inferRequestType(v interface{}) (RequestType, error) {
	switch v.(type) {
	case ByteStreamRPCProtocolVersionRequest:
		return RTProtocolVersionRequest, nil
	case FilesystemRequest:
		return RTFilesystemRequest, nil
	case InitialTransferRequest:
		return RTInitialTransferRequest, nil
	default:
		return 0, errors.New(fmt.Sprintf("cannot infer request type for type '%v'",
			reflect.TypeOf(v)))
	}
}

func genUUID() [16]byte {
	return [16]byte{} // TODO
}

func (c ByteStreamRPC) sendRequest(v interface{}) (err error) {

	var rt RequestType

	if rt, err = inferRequestType(v); err != nil {
		return
	}

	h := RequestHeader{
		Type: rt,
		Id:   genUUID(),
	}

	if err = c.encoder.Encode(h); err != nil {
		return
	}
	if err = c.encoder.Encode(v); err != nil {
		return
	}

	return
}

func (c ByteStreamRPC) expectResponseType(rt ResponseType) (err error) {
	var h ResponseHeader
	if err = c.decoder.Decode(&h); err != nil {
		return
	}

	if h.ResponseType != rt {
		return errors.New("unexpected response type in response header")
	}
	return
}

func (c ByteStreamRPC) sendRequestReceiveHeader(request interface{}, rt ResponseType) (err error) {

	if err = c.sendRequest(request); err != nil {
		return err
	}

	if err = c.expectResponseType(rt); err != nil {
		return err
	}

	return nil
}

func (c ByteStreamRPC) ProtocolVersionRequest() (err error) {
	b := ByteStreamRPCProtocolVersionRequest{
		ClientVersion: ByteStreamRPCProtocolVersion,
	}

	// OK response means the remote side can cope with our protocol version
	return c.sendRequestReceiveHeader(b, ROK)
}

func (c ByteStreamRPC) FilesystemRequest(r FilesystemRequest) (roots []Filesystem, err error) {

	if err = c.sendRequestReceiveHeader(r, RFilesystems); err != nil {
		return
	}

	roots = make([]Filesystem, 0)

	if err = c.decoder.Decode(&roots); err != nil {
		return
	}

	return
}

func (c ByteStreamRPC) InitialTransferRequest(r InitialTransferRequest) (unchunker io.Reader, err error) {

	if err = c.sendRequestReceiveHeader(r, RChunkedStream); err != nil {
		return
	}
	unchunker = NewUnchunker(c.conn)
	return
}

func (c ByteStreamRPC) IncrementalTransferRequest(r IncrementalTransferRequest) (unchunker io.Reader, err error) {
	if err = c.sendRequestReceiveHeader(r, RChunkedStream); err != nil {
		return
	}
	unchunker = NewUnchunker(c.conn)
	return
}

type LocalRPC struct {
	handler RPCHandler
}

func ConnectLocalRPC(handler RPCHandler) RPCRequester {
	return LocalRPC{handler}
}

func (c LocalRPC) FilesystemRequest(r FilesystemRequest) (roots []Filesystem, err error) {
	return c.handler.HandleFilesystemRequest(r)
}

func (c LocalRPC) InitialTransferRequest(r InitialTransferRequest) (io.Reader, error) {
	return c.handler.HandleInitialTransferRequest(r)
}

func (c LocalRPC) IncrementalTransferRequest(r IncrementalTransferRequest) (reader io.Reader, err error) {
	reader, err = c.handler.HandleIncrementalTransferRequest(r)
	return
}
<|endoftext|>